1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21
22
23
24
25
26 :type PY_SOURCE_EXTS: tuple(str)
27 :var PY_SOURCE_EXTS: list of possible python source file extension
28
29 :type STD_LIB_DIR: str
30 :var STD_LIB_DIR: directory where standard modules are located
31
32 :type BUILTIN_MODULES: dict
33 :var BUILTIN_MODULES: dictionary with builtin module names has key
34 """
35 __docformat__ = "restructuredtext en"
36
37 import sys
38 import os
39 from os.path import walk, splitext, join, abspath, isdir, dirname, exists
40 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
41
42 try:
43 import zipimport
44 except ImportError:
45 zipimport = None
46
47 ZIPFILE = object()
48
49 from logilab.common import STD_BLACKLIST
50
51 if sys.platform.startswith('win'):
52 PY_SOURCE_EXTS = ('py', 'pyw')
53 PY_COMPILED_EXTS = ('dll', 'pyd')
54 STD_LIB_DIR = join(sys.prefix, 'lib')
55 else:
56 PY_SOURCE_EXTS = ('py',)
57 PY_COMPILED_EXTS = ('so',)
58 STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3])
59
60 BUILTIN_MODULES = dict(zip(sys.builtin_module_names,
61 [1]*len(sys.builtin_module_names)))
62
63
65 """exception raised when we are not able to get a python
66 source file for a precompiled file
67 """
68
71 self.module = module
72 self.obj = obj
73 self._imported = None
74
76 if self._imported is None:
77 self._imported = getattr(load_module_from_name(self.module),
78 self.obj)
79 return self._imported
80
82 try:
83 return super(LazyObject, self).__getattribute__(attr)
84 except AttributeError, ex:
85 return getattr(self.__getobj(), attr)
86
88 return self.__getobj()(*args, **kwargs)
89
90
92 """Load a Python module from it's name.
93
94 :type dotted_name: str
95 :param dotted_name: python name of a module or package
96
97 :type path: list or None
98 :param path:
99 optional list of path where the module or package should be
100 searched (use sys.path if nothing or None is given)
101
102 :type use_sys: bool
103 :param use_sys:
104 boolean indicating whether the sys.modules dictionary should be
105 used or not
106
107
108 :raise ImportError: if the module or package is not found
109
110 :rtype: module
111 :return: the loaded module
112 """
113 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
114
115
117 """Load a python module from it's splitted name.
118
119 :type parts: list(str) or tuple(str)
120 :param parts:
121 python name of a module or package splitted on '.'
122
123 :type path: list or None
124 :param path:
125 optional list of path where the module or package should be
126 searched (use sys.path if nothing or None is given)
127
128 :type use_sys: bool
129 :param use_sys:
130 boolean indicating whether the sys.modules dictionary should be used or not
131
132 :raise ImportError: if the module or package is not found
133
134 :rtype: module
135 :return: the loaded module
136 """
137 if use_sys:
138 try:
139 return sys.modules['.'.join(parts)]
140 except KeyError:
141 pass
142 modpath = []
143 prevmodule = None
144 for part in parts:
145 modpath.append(part)
146 curname = ".".join(modpath)
147 module = None
148 if len(modpath) != len(parts):
149
150 module = sys.modules.get(curname)
151 if module is None:
152 mp_file, mp_filename, mp_desc = find_module(part, path)
153 module = load_module(curname, mp_file, mp_filename, mp_desc)
154 if prevmodule:
155 setattr(prevmodule, part, module)
156 _file = getattr(module, "__file__", "")
157 if not _file and len(modpath) != len(parts):
158 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) )
159 path = [dirname( _file )]
160 prevmodule = module
161 return module
162
163
165 """Load a Python module from it's path.
166
167 :type filepath: str
168 :param filepath: path to the python module or package
169
170 :type path: list or None
171 :param path:
172 optional list of path where the module or package should be
173 searched (use sys.path if nothing or None is given)
174
175 :type use_sys: bool
176 :param use_sys:
177 boolean indicating whether the sys.modules dictionary should be
178 used or not
179
180
181 :raise ImportError: if the module or package is not found
182
183 :rtype: module
184 :return: the loaded module
185 """
186 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
187
188
190 """check there are some __init__.py all along the way"""
191 for part in mod_path:
192 path = join(path, part)
193 if not _has_init(path):
194 return False
195 return True
196
197
199 """given a file path return the corresponding splitted module's name
200 (i.e name of a module or package splitted on '.')
201
202 :type filename: str
203 :param filename: file's path for which we want the module's name
204
205 :type extrapath: dict
206 :param extrapath:
207 optional extra search path, with path as key and package name for the path
208 as value. This is usually useful to handle package splitted in multiple
209 directories using __path__ trick.
210
211
212 :raise ImportError:
213 if the corresponding module's name has not been found
214
215 :rtype: list(str)
216 :return: the corresponding splitted module's name
217 """
218 base = splitext(abspath(filename))[0]
219 if extrapath is not None:
220 for path_ in extrapath:
221 path = abspath(path_)
222 if path and base[:len(path)] == path:
223 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
224 if pkg]
225 if _check_init(path, submodpath[:-1]):
226 return extrapath[path_].split('.') + submodpath
227 for path in sys.path:
228 path = abspath(path)
229 if path and base[:len(path)] == path:
230 if filename.find('site-packages') != -1 and \
231 path.find('site-packages') == -1:
232 continue
233 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
234 if _check_init(path, modpath[:-1]):
235 return modpath
236 raise ImportError('Unable to find module for %s in %s' % (
237 filename, ', \n'.join(sys.path)))
238
239
240
242 """given a mod path (i.e. splitted module / package name), return the
243 corresponding file, giving priority to source file over precompiled
244 file if it exists
245
246 :type modpath: list or tuple
247 :param modpath:
248 splitted module's name (i.e name of a module or package splitted
249 on '.')
250 (this means explicit relative imports that start with dots have
251 empty strings in this list!)
252
253 :type path: list or None
254 :param path:
255 optional list of path where the module or package should be
256 searched (use sys.path if nothing or None is given)
257
258 :type context_file: str or None
259 :param context_file:
260 context file to consider, necessary if the identifier has been
261 introduced using a relative import unresolvable in the actual
262 context (i.e. modutils)
263
264 :raise ImportError: if there is no such module in the directory
265
266 :rtype: str or None
267 :return:
268 the path to the module's file or None if it's an integrated
269 builtin module such as 'sys'
270 """
271 if context_file is not None:
272 context = dirname(context_file)
273 else:
274 context = context_file
275 if modpath[0] == 'xml':
276
277 try:
278 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
279 except ImportError:
280 return _file_from_modpath(modpath, path, context)
281 elif modpath == ['os', 'path']:
282
283 return os.path.__file__
284 return _file_from_modpath(modpath, path, context)
285
286
287
289 """given a dotted name return the module part of the name :
290
291 >>> get_module_part('logilab.common.modutils.get_module_part')
292 'logilab.common.modutils'
293
294
295 :type dotted_name: str
296 :param dotted_name: full name of the identifier we are interested in
297
298 :type context_file: str or None
299 :param context_file:
300 context file to consider, necessary if the identifier has been
301 introduced using a relative import unresolvable in the actual
302 context (i.e. modutils)
303
304
305 :raise ImportError: if there is no such module in the directory
306
307 :rtype: str or None
308 :return:
309 the module part of the name or None if we have not been able at
310 all to import the given name
311
312 XXX: deprecated, since it doesn't handle package precedence over module
313 (see #10066)
314 """
315
316 if dotted_name.startswith('os.path'):
317 return 'os.path'
318 parts = dotted_name.split('.')
319 if context_file is not None:
320
321
322 if parts[0] in BUILTIN_MODULES:
323 if len(parts) > 2:
324 raise ImportError(dotted_name)
325 return parts[0]
326
327 path = None
328 starti = 0
329 if parts[0] == '':
330 assert context_file is not None, \
331 'explicit relative import, but no context_file?'
332 path = []
333 starti = 1
334 while parts[starti] == '':
335 starti += 1
336 context_file = dirname(context_file)
337 for i in range(starti, len(parts)):
338 try:
339 file_from_modpath(parts[starti:i+1],
340 path=path, context_file=context_file)
341 except ImportError:
342 if not i >= max(1, len(parts) - 2):
343 raise
344 return '.'.join(parts[:i])
345 return dotted_name
346
347
348
350 """given a package directory return a list of all available python
351 modules in the package and its subpackages
352
353 :type package: str
354 :param package: the python name for the package
355
356 :type src_directory: str
357 :param src_directory:
358 path of the directory corresponding to the package
359
360 :type blacklist: list or tuple
361 :param blacklist:
362 optional list of files or directory to ignore, default to
363 the value of `logilab.common.STD_BLACKLIST`
364
365 :rtype: list
366 :return:
367 the list of all available python modules in the package and its
368 subpackages
369 """
370 def func(modules, directory, fnames):
371 """walk handler"""
372
373 for norecurs in blacklist:
374 try:
375 fnames.remove(norecurs)
376 except ValueError:
377 continue
378
379 if not '__init__.py' in fnames:
380 while fnames:
381 fnames.pop()
382 elif directory != src_directory:
383
384 dir_package = directory[len(src_directory):].replace(os.sep, '.')
385 modules.append(package + dir_package)
386 for filename in fnames:
387 src = join(directory, filename)
388 if isdir(src):
389 continue
390 if _is_python_file(filename) and filename != '__init__.py':
391 module = package + src[len(src_directory):-3]
392 modules.append(module.replace(os.sep, '.'))
393 modules = []
394 walk(src_directory, func, modules)
395 return modules
396
397
398
400 """given a package directory return a list of all available python
401 module's files in the package and its subpackages
402
403 :type src_directory: str
404 :param src_directory:
405 path of the directory corresponding to the package
406
407 :type blacklist: list or tuple
408 :param blacklist:
409 optional list of files or directory to ignore, default to the value of
410 `logilab.common.STD_BLACKLIST`
411
412 :rtype: list
413 :return:
414 the list of all available python module's files in the package and
415 its subpackages
416 """
417 def func(files, directory, fnames):
418 """walk handler"""
419
420 for norecurs in blacklist:
421 try:
422 fnames.remove(norecurs)
423 except ValueError:
424 continue
425
426 if not '__init__.py' in fnames:
427 while fnames:
428 fnames.pop()
429 for filename in fnames:
430 src = join(directory, filename)
431 if isdir(src):
432 continue
433 if _is_python_file(filename):
434 files.append(src)
435 files = []
436 walk(src_directory, func, files)
437 return files
438
439
441 """given a python module's file name return the matching source file
442 name (the filename will be returned identically if it's a already an
443 absolute path to a python source file...)
444
445 :type filename: str
446 :param filename: python module's file name
447
448
449 :raise NoSourceFile: if no source file exists on the file system
450
451 :rtype: str
452 :return: the absolute path of the source file if it exists
453 """
454 base, orig_ext = splitext(abspath(filename))
455 for ext in PY_SOURCE_EXTS:
456 source_path = '%s.%s' % (base, ext)
457 if exists(source_path):
458 return source_path
459 if include_no_ext and not orig_ext and exists(base):
460 return base
461 raise NoSourceFile(filename)
462
463
465 """remove submodules of `directories` from `sys.modules`"""
466 for modname, module in sys.modules.items():
467 modfile = getattr(module, '__file__', None)
468 if modfile:
469 for directory in directories:
470 if modfile.startswith(directory):
471 del sys.modules[modname]
472 break
473
474
476 """
477 rtype: bool
478 return: True if the filename is a python source file
479 """
480 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
481
482
483
485 """try to guess if a module is a standard python module (by default,
486 see `std_path` parameter's description)
487
488 :type modname: str
489 :param modname: name of the module we are interested in
490
491 :type std_path: list(str) or tuple(str)
492 :param std_path: list of path considered has standard
493
494
495 :rtype: bool
496 :return:
497 true if the module:
498 - is located on the path listed in one of the directory in `std_path`
499 - is a built-in module
500 """
501 modname = modname.split('.')[0]
502 try:
503 filename = file_from_modpath([modname])
504 except ImportError, ex:
505
506
507 return 0
508
509
510 if filename is None:
511 return 1
512 filename = abspath(filename)
513 for path in std_path:
514 path = abspath(path)
515 if filename.startswith(path):
516 pfx_len = len(path)
517 if filename[pfx_len+1:pfx_len+14] != 'site-packages':
518 return 1
519 return 0
520 return False
521
522
523
525 """return true if the given module name is relative to the given
526 file name
527
528 :type modname: str
529 :param modname: name of the module we are interested in
530
531 :type from_file: str
532 :param from_file:
533 path of the module from which modname has been imported
534
535 :rtype: bool
536 :return:
537 true if the module has been imported relatively to `from_file`
538 """
539 if not isdir(from_file):
540 from_file = dirname(from_file)
541 if from_file in sys.path:
542 return False
543 try:
544 find_module(modname.split('.')[0], [from_file])
545 return True
546 except ImportError:
547 return False
548
549
550
551
553 """given a mod path (i.e. splitted module / package name), return the
554 corresponding file
555
556 this function is used internally, see `file_from_modpath`'s
557 documentation for more information
558 """
559 assert len(modpath) > 0
560 if context is not None:
561 try:
562 mtype, mp_filename = _module_file(modpath, [context])
563 except ImportError:
564 mtype, mp_filename = _module_file(modpath, path)
565 else:
566 mtype, mp_filename = _module_file(modpath, path)
567 if mtype == PY_COMPILED:
568 try:
569 return get_source_file(mp_filename)
570 except NoSourceFile:
571 return mp_filename
572 elif mtype == C_BUILTIN:
573
574 return None
575 elif mtype == PKG_DIRECTORY:
576 mp_filename = _has_init(mp_filename)
577 return mp_filename
578
580 for filepath, importer in pic.items():
581 if importer is not None:
582 if importer.find_module(modpath[0]):
583 if not importer.find_module('/'.join(modpath)):
584 raise ImportError('No module named %s in %s/%s' % (
585 '.'.join(modpath[1:]), file, modpath))
586 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
587 raise ImportError('No module named %s' % '.'.join(modpath))
588
590 """get a module type / file path
591
592 :type modpath: list or tuple
593 :param modpath:
594 splitted module's name (i.e name of a module or package splitted
595 on '.'), with leading empty strings for explicit relative import
596
597 :type path: list or None
598 :param path:
599 optional list of path where the module or package should be
600 searched (use sys.path if nothing or None is given)
601
602
603 :rtype: tuple(int, str)
604 :return: the module type flag and the file path for a module
605 """
606
607 try:
608 pic = sys.path_importer_cache
609 _path = (path is None and sys.path or path)
610 for __path in _path:
611 if not __path in pic:
612 try:
613 pic[__path] = zipimport.zipimporter(__path)
614 except zipimport.ZipImportError:
615 pic[__path] = None
616 checkeggs = True
617 except AttributeError:
618 checkeggs = False
619 imported = []
620 while modpath:
621 try:
622 _, mp_filename, mp_desc = find_module(modpath[0], path)
623 except ImportError:
624 if checkeggs:
625 return _search_zip(modpath, pic)[:2]
626 raise
627 else:
628 if checkeggs:
629 fullabspath = [abspath(x) for x in _path]
630 try:
631 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
632 emtype, emp_filename, zippath = _search_zip(modpath, pic)
633 if pathindex > _path.index(zippath):
634
635 return emtype, emp_filename
636 except ValueError:
637
638 pass
639 except ImportError:
640 pass
641 checkeggs = False
642 imported.append(modpath.pop(0))
643 mtype = mp_desc[2]
644 if modpath:
645 if mtype != PKG_DIRECTORY:
646 raise ImportError('No module %s in %s' % ('.'.join(modpath),
647 '.'.join(imported)))
648 path = [mp_filename]
649 return mtype, mp_filename
650
652 """return true if the given filename should be considered as a python file
653
654 .pyc and .pyo are ignored
655 """
656 for ext in ('.py', '.so', '.pyd', '.pyw'):
657 if filename.endswith(ext):
658 return True
659 return False
660
661
663 """if the given directory has a valid __init__ file, return its path,
664 else return None
665 """
666 mod_or_pack = join(directory, '__init__')
667 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
668 if exists(mod_or_pack + '.' + ext):
669 return mod_or_pack + '.' + ext
670 return None
671