Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21   
 22   
 23   
 24   
 25   
 26  :type PY_SOURCE_EXTS: tuple(str) 
 27  :var PY_SOURCE_EXTS: list of possible python source file extension 
 28   
 29  :type STD_LIB_DIR: str 
 30  :var STD_LIB_DIR: directory where standard modules are located 
 31   
 32  :type BUILTIN_MODULES: dict 
 33  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 34  """ 
 35  __docformat__ = "restructuredtext en" 
 36   
 37  import sys 
 38  import os 
 39  from os.path import walk, splitext, join, abspath, isdir, dirname, exists 
 40  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 41   
 42  try: 
 43      import zipimport 
 44  except ImportError: 
 45      zipimport = None 
 46   
 47  ZIPFILE = object() 
 48   
 49  from logilab.common import STD_BLACKLIST 
 50   
 51  if sys.platform.startswith('win'): 
 52      PY_SOURCE_EXTS = ('py', 'pyw') 
 53      PY_COMPILED_EXTS = ('dll', 'pyd') 
 54      STD_LIB_DIR = join(sys.prefix, 'lib') 
 55  else: 
 56      PY_SOURCE_EXTS = ('py',) 
 57      PY_COMPILED_EXTS = ('so',) 
 58      STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3]) 
 59   
 60  BUILTIN_MODULES = dict(zip(sys.builtin_module_names, 
 61                             [1]*len(sys.builtin_module_names))) 
 62   
 63   
64 -class NoSourceFile(Exception):
65 """exception raised when we are not able to get a python 66 source file for a precompiled file 67 """
68
69 -class LazyObject(object):
70 - def __init__(self, module, obj):
71 self.module = module 72 self.obj = obj 73 self._imported = None
74
75 - def __getobj(self):
76 if self._imported is None: 77 self._imported = getattr(load_module_from_name(self.module), 78 self.obj) 79 return self._imported
80
81 - def __getattribute__(self, attr):
82 try: 83 return super(LazyObject, self).__getattribute__(attr) 84 except AttributeError, ex: 85 return getattr(self.__getobj(), attr)
86
87 - def __call__(self, *args, **kwargs):
88 return self.__getobj()(*args, **kwargs)
89 90
91 -def load_module_from_name(dotted_name, path=None, use_sys=1):
92 """Load a Python module from it's name. 93 94 :type dotted_name: str 95 :param dotted_name: python name of a module or package 96 97 :type path: list or None 98 :param path: 99 optional list of path where the module or package should be 100 searched (use sys.path if nothing or None is given) 101 102 :type use_sys: bool 103 :param use_sys: 104 boolean indicating whether the sys.modules dictionary should be 105 used or not 106 107 108 :raise ImportError: if the module or package is not found 109 110 :rtype: module 111 :return: the loaded module 112 """ 113 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
114 115
116 -def load_module_from_modpath(parts, path=None, use_sys=1):
117 """Load a python module from it's splitted name. 118 119 :type parts: list(str) or tuple(str) 120 :param parts: 121 python name of a module or package splitted on '.' 122 123 :type path: list or None 124 :param path: 125 optional list of path where the module or package should be 126 searched (use sys.path if nothing or None is given) 127 128 :type use_sys: bool 129 :param use_sys: 130 boolean indicating whether the sys.modules dictionary should be used or not 131 132 :raise ImportError: if the module or package is not found 133 134 :rtype: module 135 :return: the loaded module 136 """ 137 if use_sys: 138 try: 139 return sys.modules['.'.join(parts)] 140 except KeyError: 141 pass 142 modpath = [] 143 prevmodule = None 144 for part in parts: 145 modpath.append(part) 146 curname = ".".join(modpath) 147 module = None 148 if len(modpath) != len(parts): 149 # even with use_sys=False, should try to get outer packages from sys.modules 150 module = sys.modules.get(curname) 151 if module is None: 152 mp_file, mp_filename, mp_desc = find_module(part, path) 153 module = load_module(curname, mp_file, mp_filename, mp_desc) 154 if prevmodule: 155 setattr(prevmodule, part, module) 156 _file = getattr(module, "__file__", "") 157 if not _file and len(modpath) != len(parts): 158 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) ) 159 path = [dirname( _file )] 160 prevmodule = module 161 return module
162 163
164 -def load_module_from_file(filepath, path=None, use_sys=1):
165 """Load a Python module from it's path. 166 167 :type filepath: str 168 :param filepath: path to the python module or package 169 170 :type path: list or None 171 :param path: 172 optional list of path where the module or package should be 173 searched (use sys.path if nothing or None is given) 174 175 :type use_sys: bool 176 :param use_sys: 177 boolean indicating whether the sys.modules dictionary should be 178 used or not 179 180 181 :raise ImportError: if the module or package is not found 182 183 :rtype: module 184 :return: the loaded module 185 """ 186 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
187 188
189 -def _check_init(path, mod_path):
190 """check there are some __init__.py all along the way""" 191 for part in mod_path: 192 path = join(path, part) 193 if not _has_init(path): 194 return False 195 return True
196 197
198 -def modpath_from_file(filename, extrapath=None):
199 """given a file path return the corresponding splitted module's name 200 (i.e name of a module or package splitted on '.') 201 202 :type filename: str 203 :param filename: file's path for which we want the module's name 204 205 :type extrapath: dict 206 :param extrapath: 207 optional extra search path, with path as key and package name for the path 208 as value. This is usually useful to handle package splitted in multiple 209 directories using __path__ trick. 210 211 212 :raise ImportError: 213 if the corresponding module's name has not been found 214 215 :rtype: list(str) 216 :return: the corresponding splitted module's name 217 """ 218 base = splitext(abspath(filename))[0] 219 if extrapath is not None: 220 for path_ in extrapath: 221 path = abspath(path_) 222 if path and base[:len(path)] == path: 223 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 224 if pkg] 225 if _check_init(path, submodpath[:-1]): 226 return extrapath[path_].split('.') + submodpath 227 for path in sys.path: 228 path = abspath(path) 229 if path and base[:len(path)] == path: 230 if filename.find('site-packages') != -1 and \ 231 path.find('site-packages') == -1: 232 continue 233 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 234 if _check_init(path, modpath[:-1]): 235 return modpath 236 raise ImportError('Unable to find module for %s in %s' % ( 237 filename, ', \n'.join(sys.path)))
238 239 240
241 -def file_from_modpath(modpath, path=None, context_file=None):
242 """given a mod path (i.e. splitted module / package name), return the 243 corresponding file, giving priority to source file over precompiled 244 file if it exists 245 246 :type modpath: list or tuple 247 :param modpath: 248 splitted module's name (i.e name of a module or package splitted 249 on '.') 250 (this means explicit relative imports that start with dots have 251 empty strings in this list!) 252 253 :type path: list or None 254 :param path: 255 optional list of path where the module or package should be 256 searched (use sys.path if nothing or None is given) 257 258 :type context_file: str or None 259 :param context_file: 260 context file to consider, necessary if the identifier has been 261 introduced using a relative import unresolvable in the actual 262 context (i.e. modutils) 263 264 :raise ImportError: if there is no such module in the directory 265 266 :rtype: str or None 267 :return: 268 the path to the module's file or None if it's an integrated 269 builtin module such as 'sys' 270 """ 271 if context_file is not None: 272 context = dirname(context_file) 273 else: 274 context = context_file 275 if modpath[0] == 'xml': 276 # handle _xmlplus 277 try: 278 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 279 except ImportError: 280 return _file_from_modpath(modpath, path, context) 281 elif modpath == ['os', 'path']: 282 # FIXME: currently ignoring search_path... 283 return os.path.__file__ 284 return _file_from_modpath(modpath, path, context)
285 286 287
288 -def get_module_part(dotted_name, context_file=None):
289 """given a dotted name return the module part of the name : 290 291 >>> get_module_part('logilab.common.modutils.get_module_part') 292 'logilab.common.modutils' 293 294 295 :type dotted_name: str 296 :param dotted_name: full name of the identifier we are interested in 297 298 :type context_file: str or None 299 :param context_file: 300 context file to consider, necessary if the identifier has been 301 introduced using a relative import unresolvable in the actual 302 context (i.e. modutils) 303 304 305 :raise ImportError: if there is no such module in the directory 306 307 :rtype: str or None 308 :return: 309 the module part of the name or None if we have not been able at 310 all to import the given name 311 312 XXX: deprecated, since it doesn't handle package precedence over module 313 (see #10066) 314 """ 315 # os.path trick 316 if dotted_name.startswith('os.path'): 317 return 'os.path' 318 parts = dotted_name.split('.') 319 if context_file is not None: 320 # first check for builtin module which won't be considered latter 321 # in that case (path != None) 322 if parts[0] in BUILTIN_MODULES: 323 if len(parts) > 2: 324 raise ImportError(dotted_name) 325 return parts[0] 326 # don't use += or insert, we want a new list to be created ! 327 path = None 328 starti = 0 329 if parts[0] == '': 330 assert context_file is not None, \ 331 'explicit relative import, but no context_file?' 332 path = [] # prevent resolving the import non-relatively 333 starti = 1 334 while parts[starti] == '': # for all further dots: change context 335 starti += 1 336 context_file = dirname(context_file) 337 for i in range(starti, len(parts)): 338 try: 339 file_from_modpath(parts[starti:i+1], 340 path=path, context_file=context_file) 341 except ImportError: 342 if not i >= max(1, len(parts) - 2): 343 raise 344 return '.'.join(parts[:i]) 345 return dotted_name
346 347 348
349 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
350 """given a package directory return a list of all available python 351 modules in the package and its subpackages 352 353 :type package: str 354 :param package: the python name for the package 355 356 :type src_directory: str 357 :param src_directory: 358 path of the directory corresponding to the package 359 360 :type blacklist: list or tuple 361 :param blacklist: 362 optional list of files or directory to ignore, default to 363 the value of `logilab.common.STD_BLACKLIST` 364 365 :rtype: list 366 :return: 367 the list of all available python modules in the package and its 368 subpackages 369 """ 370 def func(modules, directory, fnames): 371 """walk handler""" 372 # remove files/directories in the black list 373 for norecurs in blacklist: 374 try: 375 fnames.remove(norecurs) 376 except ValueError: 377 continue 378 # check for __init__.py 379 if not '__init__.py' in fnames: 380 while fnames: 381 fnames.pop() 382 elif directory != src_directory: 383 #src = join(directory, file) 384 dir_package = directory[len(src_directory):].replace(os.sep, '.') 385 modules.append(package + dir_package) 386 for filename in fnames: 387 src = join(directory, filename) 388 if isdir(src): 389 continue 390 if _is_python_file(filename) and filename != '__init__.py': 391 module = package + src[len(src_directory):-3] 392 modules.append(module.replace(os.sep, '.'))
393 modules = [] 394 walk(src_directory, func, modules) 395 return modules 396 397 398
399 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
400 """given a package directory return a list of all available python 401 module's files in the package and its subpackages 402 403 :type src_directory: str 404 :param src_directory: 405 path of the directory corresponding to the package 406 407 :type blacklist: list or tuple 408 :param blacklist: 409 optional list of files or directory to ignore, default to the value of 410 `logilab.common.STD_BLACKLIST` 411 412 :rtype: list 413 :return: 414 the list of all available python module's files in the package and 415 its subpackages 416 """ 417 def func(files, directory, fnames): 418 """walk handler""" 419 # remove files/directories in the black list 420 for norecurs in blacklist: 421 try: 422 fnames.remove(norecurs) 423 except ValueError: 424 continue 425 # check for __init__.py 426 if not '__init__.py' in fnames: 427 while fnames: 428 fnames.pop() 429 for filename in fnames: 430 src = join(directory, filename) 431 if isdir(src): 432 continue 433 if _is_python_file(filename): 434 files.append(src)
435 files = [] 436 walk(src_directory, func, files) 437 return files 438 439
440 -def get_source_file(filename, include_no_ext=False):
441 """given a python module's file name return the matching source file 442 name (the filename will be returned identically if it's a already an 443 absolute path to a python source file...) 444 445 :type filename: str 446 :param filename: python module's file name 447 448 449 :raise NoSourceFile: if no source file exists on the file system 450 451 :rtype: str 452 :return: the absolute path of the source file if it exists 453 """ 454 base, orig_ext = splitext(abspath(filename)) 455 for ext in PY_SOURCE_EXTS: 456 source_path = '%s.%s' % (base, ext) 457 if exists(source_path): 458 return source_path 459 if include_no_ext and not orig_ext and exists(base): 460 return base 461 raise NoSourceFile(filename)
462 463
464 -def cleanup_sys_modules(directories):
465 """remove submodules of `directories` from `sys.modules`""" 466 for modname, module in sys.modules.items(): 467 modfile = getattr(module, '__file__', None) 468 if modfile: 469 for directory in directories: 470 if modfile.startswith(directory): 471 del sys.modules[modname] 472 break
473 474
475 -def is_python_source(filename):
476 """ 477 rtype: bool 478 return: True if the filename is a python source file 479 """ 480 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
481 482 483
484 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
485 """try to guess if a module is a standard python module (by default, 486 see `std_path` parameter's description) 487 488 :type modname: str 489 :param modname: name of the module we are interested in 490 491 :type std_path: list(str) or tuple(str) 492 :param std_path: list of path considered has standard 493 494 495 :rtype: bool 496 :return: 497 true if the module: 498 - is located on the path listed in one of the directory in `std_path` 499 - is a built-in module 500 """ 501 modname = modname.split('.')[0] 502 try: 503 filename = file_from_modpath([modname]) 504 except ImportError, ex: 505 # import failed, i'm probably not so wrong by supposing it's 506 # not standard... 507 return 0 508 # modules which are not living in a file are considered standard 509 # (sys and __builtin__ for instance) 510 if filename is None: 511 return 1 512 filename = abspath(filename) 513 for path in std_path: 514 path = abspath(path) 515 if filename.startswith(path): 516 pfx_len = len(path) 517 if filename[pfx_len+1:pfx_len+14] != 'site-packages': 518 return 1 519 return 0 520 return False
521 522 523
524 -def is_relative(modname, from_file):
525 """return true if the given module name is relative to the given 526 file name 527 528 :type modname: str 529 :param modname: name of the module we are interested in 530 531 :type from_file: str 532 :param from_file: 533 path of the module from which modname has been imported 534 535 :rtype: bool 536 :return: 537 true if the module has been imported relatively to `from_file` 538 """ 539 if not isdir(from_file): 540 from_file = dirname(from_file) 541 if from_file in sys.path: 542 return False 543 try: 544 find_module(modname.split('.')[0], [from_file]) 545 return True 546 except ImportError: 547 return False
548 549 550 # internal only functions ##################################################### 551
552 -def _file_from_modpath(modpath, path=None, context=None):
553 """given a mod path (i.e. splitted module / package name), return the 554 corresponding file 555 556 this function is used internally, see `file_from_modpath`'s 557 documentation for more information 558 """ 559 assert len(modpath) > 0 560 if context is not None: 561 try: 562 mtype, mp_filename = _module_file(modpath, [context]) 563 except ImportError: 564 mtype, mp_filename = _module_file(modpath, path) 565 else: 566 mtype, mp_filename = _module_file(modpath, path) 567 if mtype == PY_COMPILED: 568 try: 569 return get_source_file(mp_filename) 570 except NoSourceFile: 571 return mp_filename 572 elif mtype == C_BUILTIN: 573 # integrated builtin module 574 return None 575 elif mtype == PKG_DIRECTORY: 576 mp_filename = _has_init(mp_filename) 577 return mp_filename
578
579 -def _search_zip(modpath, pic):
580 for filepath, importer in pic.items(): 581 if importer is not None: 582 if importer.find_module(modpath[0]): 583 if not importer.find_module('/'.join(modpath)): 584 raise ImportError('No module named %s in %s/%s' % ( 585 '.'.join(modpath[1:]), file, modpath)) 586 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 587 raise ImportError('No module named %s' % '.'.join(modpath))
588
589 -def _module_file(modpath, path=None):
590 """get a module type / file path 591 592 :type modpath: list or tuple 593 :param modpath: 594 splitted module's name (i.e name of a module or package splitted 595 on '.'), with leading empty strings for explicit relative import 596 597 :type path: list or None 598 :param path: 599 optional list of path where the module or package should be 600 searched (use sys.path if nothing or None is given) 601 602 603 :rtype: tuple(int, str) 604 :return: the module type flag and the file path for a module 605 """ 606 # egg support compat 607 try: 608 pic = sys.path_importer_cache 609 _path = (path is None and sys.path or path) 610 for __path in _path: 611 if not __path in pic: 612 try: 613 pic[__path] = zipimport.zipimporter(__path) 614 except zipimport.ZipImportError: 615 pic[__path] = None 616 checkeggs = True 617 except AttributeError: 618 checkeggs = False 619 imported = [] 620 while modpath: 621 try: 622 _, mp_filename, mp_desc = find_module(modpath[0], path) 623 except ImportError: 624 if checkeggs: 625 return _search_zip(modpath, pic)[:2] 626 raise 627 else: 628 if checkeggs: 629 fullabspath = [abspath(x) for x in _path] 630 try: 631 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 632 emtype, emp_filename, zippath = _search_zip(modpath, pic) 633 if pathindex > _path.index(zippath): 634 # an egg takes priority 635 return emtype, emp_filename 636 except ValueError: 637 # XXX not in _path 638 pass 639 except ImportError: 640 pass 641 checkeggs = False 642 imported.append(modpath.pop(0)) 643 mtype = mp_desc[2] 644 if modpath: 645 if mtype != PKG_DIRECTORY: 646 raise ImportError('No module %s in %s' % ('.'.join(modpath), 647 '.'.join(imported))) 648 path = [mp_filename] 649 return mtype, mp_filename
650
651 -def _is_python_file(filename):
652 """return true if the given filename should be considered as a python file 653 654 .pyc and .pyo are ignored 655 """ 656 for ext in ('.py', '.so', '.pyd', '.pyw'): 657 if filename.endswith(ext): 658 return True 659 return False
660 661
662 -def _has_init(directory):
663 """if the given directory has a valid __init__ file, return its path, 664 else return None 665 """ 666 mod_or_pack = join(directory, '__init__') 667 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 668 if exists(mod_or_pack + '.' + ext): 669 return mod_or_pack + '.' + ext 670 return None
671