Package flumotion :: Package common :: Module package
[hide private]

Source Code for Module flumotion.common.package

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_package -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  objects and functions used in dealing with packages 
 24  """ 
 25   
 26  import ihooks 
 27  import os 
 28  import sys 
 29  import glob 
 30   
 31  from flumotion.common import log, common 
 32  from twisted.python import rebuild, reflect 
 33   
34 -class _PatchedModuleImporter(ihooks.ModuleImporter):
35 """ 36 I am overriding ihook's ModuleImporter's import_module() method to 37 accept (and ignore) the 'level' keyword argument that appeared in 38 the built-in __import__() function in python2.5. 39 40 While no built-in modules in python2.5 seem to use that keyword 41 argument, 'encodings' module in python2.6 does and so it breaks if 42 used together with ihooks. 43 44 I make no attempt to properly support the 'level' argument - 45 ihooks didn't make it into py3k, and the only use in python2.6 46 we've seen so far, in 'encodings', serves as a performance hint 47 and it seems that can be ignored with no difference in behaviour. 48 """ 49
50 - def import_module(self, name, globals=None, locals=None, fromlist=None, 51 level=-1):
52 # all we do is drop 'level' as ihooks don't support it, anyway 53 return ihooks.ModuleImporter.import_module(self, name, globals, 54 locals, fromlist)
55 56
57 -class PackageHooks(ihooks.Hooks):
58 """ 59 I am an import Hooks object that makes sure that every package that gets 60 loaded has every necessary path in the module's __path__ list. 61 62 @type packager: L{Packager} 63 """ 64 packager = None 65
66 - def load_package(self, name, filename, file=None):
67 # this is only ever called the first time a package is imported 68 log.log('packager', 'load_package %s' % name) 69 ret = ihooks.Hooks.load_package(self, name, filename, file) 70 71 m = sys.modules[name] 72 73 packagePaths = self.packager.getPathsForPackage(name) 74 if not packagePaths: 75 return ret 76 77 # get full paths to the package 78 paths = [os.path.join(path, name.replace('.', os.sep)) for path in packagePaths] 79 for path in paths: 80 if not path in m.__path__: 81 log.log('packager', 'adding path %s for package %s' % ( 82 path, name)) 83 m.__path__.append(path) 84 85 return ret
86
87 -class Packager(log.Loggable):
88 """ 89 I am an object through which package paths can be registered, to support 90 the partitioning of the module import namespace across bundles. 91 """ 92 93 logCategory = 'packager' 94
95 - def __init__(self):
96 self._paths = {} # key -> package path registered with that key 97 self._packages = {} # package name -> keys for that package 98 self.install()
99
100 - def install(self):
101 """ 102 Install our custom importer that uses bundled packages. 103 """ 104 self.debug('installing custom importer') 105 self._hooks = PackageHooks() 106 self._hooks.packager = self 107 if sys.version_info < (2, 6): 108 self._importer = ihooks.ModuleImporter() 109 else: 110 self.debug('python2.6 or later detected - using patched' 111 ' ModuleImporter') 112 self._importer = _PatchedModuleImporter() 113 self._importer.set_hooks(self._hooks) 114 self._importer.install()
115
116 - def getPathsForPackage(self, packageName):
117 """ 118 Return all absolute paths to the top level of a tree from which 119 (part of) the given package name can be imported. 120 """ 121 if not packageName in self._packages.keys(): 122 return None 123 124 return [self._paths[key] for key in self._packages[packageName]]
125
126 - def registerPackagePath(self, packagePath, key, prefix='flumotion'):
127 """ 128 Register a given path as a path that can be imported from. 129 Used to support partition of bundled code or import code from various 130 uninstalled location. 131 132 sys.path will also be changed to include this, and remove references 133 to older packagePath's for the same bundle. 134 135 @param packagePath: path to add under which the module namespaces live, 136 (ending in an md5sum, for flumotion purposes) 137 @type packagePath: string 138 @param key a unique id for the package being registered 139 @type key: string 140 @param prefix: prefix of the packages to be considered 141 @type prefix: string 142 """ 143 144 new = True 145 packagePath = os.path.abspath(packagePath) 146 if not os.path.exists(packagePath): 147 log.warning('bundle', 148 'registering a non-existing package path %s' % packagePath) 149 150 self.log('registering packagePath %s' % packagePath) 151 152 # check if a packagePath for this bundle was already registered 153 if key in self._paths.keys(): 154 oldPath = self._paths[key] 155 if packagePath == oldPath: 156 self.log('already registered %s for key %s' % ( 157 packagePath, key)) 158 return 159 new = False 160 161 # Find the packages in the path and sort them, 162 # the following algorithm only works if they're sorted. 163 # By sorting the list we can ensure that a parent package 164 # is always processed before one of its children 165 packageNames = _findPackageCandidates(packagePath, prefix) 166 167 if not packageNames: 168 log.log('bundle', 169 'packagePath %s does not have candidates starting with %s' % 170 (packagePath, prefix)) 171 return 172 packageNames.sort() 173 174 self.log('package candidates %r' % packageNames) 175 176 if not new: 177 # it already existed, and now it's a different path 178 log.log('bundle', 179 'replacing old path %s with new path %s for key %s' % ( 180 oldPath, packagePath, key)) 181 182 if oldPath in sys.path: 183 log.log('bundle', 184 'removing old packagePath %s from sys.path' % oldPath) 185 sys.path.remove(oldPath) 186 187 # clear this key from our name -> key cache 188 for keys in self._packages.values(): 189 if key in keys: 190 keys.remove(key) 191 192 self._paths[key] = packagePath 193 194 # put packagePath at the top of sys.path if not in there 195 if not packagePath in sys.path: 196 self.log('adding packagePath %s to sys.path' % packagePath) 197 sys.path.insert(0, packagePath) 198 199 # update our name->keys cache 200 for name in packageNames: 201 if not name in self._packages.keys(): 202 self._packages[name] = [key] 203 else: 204 self._packages[name].insert(0, key) 205 206 self.log('packagePath %s has packageNames %r' % ( 207 packagePath, packageNames)) 208 # since we want sub-modules to be fixed up before parent packages, 209 # we reverse the list 210 packageNames.reverse() 211 212 for packageName in packageNames: 213 if packageName not in sys.modules.keys(): 214 continue 215 self.log('fixing up %s ...' % packageName) 216 217 # the package is imported, so mess with __path__ and rebuild 218 package = sys.modules.get(packageName) 219 for path in package.__path__: 220 if not new and path.startswith(oldPath): 221 self.log('%s.__path__ before remove %r' % ( 222 packageName, package.__path__)) 223 self.log('removing old %s from %s.__path__' % ( 224 path, name)) 225 package.__path__.remove(path) 226 self.log('%s.__path__ after remove %r' % ( 227 packageName, package.__path__)) 228 229 # move the new path to the top 230 # insert at front because FLU_REGISTRY_PATH paths should override 231 # base components, and because subsequent reload() should prefer 232 # the latest registered path 233 newPath = os.path.join(packagePath, 234 packageName.replace('.', os.sep)) 235 236 # if path already at position 0, everything's fine 237 # if it's in there at another place, it needs to move to front 238 # if not in there, it needs to be put in front 239 if len(package.__path__) == 0: 240 # FIXME: this seems to happen to e.g. flumotion.component.base 241 # even when it was just rebuilt and had the __path__ set 242 # can be triggered by choosing a admin_gtk depending on 243 # the base admin_gtk where the base admin_gtk changes 244 self.debug('WARN: package %s does not have __path__ values' % ( 245 packageName)) 246 elif package.__path__[0] == newPath: 247 self.log('path %s already at start of %s.__path__' % ( 248 newPath, packageName)) 249 continue 250 251 if newPath in package.__path__: 252 package.__path__.remove(newPath) 253 self.log('moving %s to front of %s.__path__' % ( 254 newPath, packageName)) 255 else: 256 self.log('inserting new %s into %s.__path__' % ( 257 newPath, packageName)) 258 package.__path__.insert(0, newPath) 259 260 # Rebuilding these packages just to get __path__ fixed in 261 # seems not necessary - but re-enable it if it breaks 262 # self.log('rebuilding package %s from paths %r' % (packageName, 263 # package.__path__)) 264 # rebuild.rebuild(package) 265 # self.log('rebuilt package %s with paths %r' % (packageName, 266 # package.__path__)) 267 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__)) 268 269 # now rebuild all non-package modules in this packagePath if this 270 # is not a new package 271 if not new: 272 self.log('finding end module candidates') 273 moduleNames = findEndModuleCandidates(packagePath, prefix) 274 self.log('end module candidates to rebuild: %r' % moduleNames) 275 for name in moduleNames: 276 if name in sys.modules: 277 # fixme: isn't sys.modules[name] sufficient? 278 self.log("rebuilding non-package module %s" % name) 279 try: 280 module = reflect.namedAny(name) 281 except AttributeError: 282 log.warning('bundle', 283 "could not reflect non-package module %s" % name) 284 continue 285 286 if hasattr(module, '__path__'): 287 self.log('rebuilding module %s with paths %r' % (name, 288 module.__path__)) 289 rebuild.rebuild(module) 290 #if paths: 291 # module.__path__ = paths 292 293 self.log('registered packagePath %s for key %s' % (packagePath, key))
294
295 - def unregister(self):
296 """ 297 Unregister all previously registered package paths, and uninstall 298 the custom importer. 299 """ 300 for path in self._paths.values(): 301 if path in sys.path: 302 self.log('removing packagePath %s from sys.path' % path) 303 sys.path.remove(path) 304 self._paths = {} 305 self._packages = {} 306 self.debug('uninstalling custom importer') 307 self._importer.uninstall()
308
309 -def _listDirRecursively(path):
310 """ 311 I'm similar to os.listdir, but I work recursively and only return 312 directories containing python code. 313 314 @param path: the path 315 @type path: string 316 """ 317 retval = [] 318 # files are never returned, only directories 319 if not os.path.isdir(path): 320 return retval 321 322 try: 323 files = os.listdir(path) 324 except OSError: 325 pass 326 else: 327 for f in files: 328 # this only adds directories since files are not returned 329 retval += _listDirRecursively(os.path.join(path, f)) 330 331 if glob.glob(os.path.join(path, '*.py*')): 332 retval.append(path) 333 334 return retval
335
336 -def _listPyFileRecursively(path):
337 """ 338 I'm similar to os.listdir, but I work recursively and only return 339 files representing python non-package modules. 340 341 @param path: the path 342 @type path: string 343 344 @rtype: list 345 @returns: list of files underneath the given path containing python code 346 """ 347 retval = [] 348 349 # get all the dirs containing python code 350 dirs = _listDirRecursively(path) 351 352 for dir in dirs: 353 pyfiles = glob.glob(os.path.join(dir, '*.py*')) 354 dontkeep = glob.glob(os.path.join(dir, '*__init__.py*')) 355 for f in dontkeep: 356 if f in pyfiles: 357 pyfiles.remove(f) 358 359 retval.extend(pyfiles) 360 361 return retval
362
363 -def _findPackageCandidates(path, prefix='flumotion'):
364 """ 365 I take a directory and return a list of candidate python packages 366 under that directory that start with the given prefix. 367 A package is a module containing modules; typically the directory 368 with the same name as the package contains __init__.py 369 370 @param path: the path 371 @type path: string 372 """ 373 # this function also "guesses" candidate packages when __init__ is missing 374 # so a bundle with only a subpackage is also detected 375 dirs = _listDirRecursively(os.path.join(path, prefix)) 376 377 # chop off the base path to get a list of "relative" bundlespace paths 378 bundlePaths = [x[len(path) + 1:] for x in dirs] 379 380 # remove some common candidates, like .svn subdirs, or containing - 381 isNotSvn = lambda x: x.find('.svn') == -1 382 bundlePaths = filter(isNotSvn, bundlePaths) 383 isNotDashed = lambda x: x.find('-') == -1 384 bundlePaths = filter(isNotDashed, bundlePaths) 385 386 # convert paths to module namespace 387 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths] 388 389 # now make sure that all parent packages for each package are listed 390 # as well 391 packages = {} 392 for name in bundlePackages: 393 packages[name] = 1 394 parts = name.split(".") 395 build = None 396 for p in parts: 397 if not build: 398 build = p 399 else: 400 build = build + "." + p 401 packages[build] = 1 402 403 bundlePackages = packages.keys() 404 405 # sort them so that depending packages are after higher-up packages 406 bundlePackages.sort() 407 408 return bundlePackages
409
410 -def findEndModuleCandidates(path, prefix='flumotion'):
411 """ 412 I take a directory and return a list of candidate python end modules 413 (i.e., non-package modules) for the given module prefix. 414 415 @param path: the path under which to search for end modules 416 @type path: string 417 @param prefix: module prefix to check candidates under 418 @type prefix: string 419 """ 420 pathPrefix = "/".join(prefix.split(".")) 421 files = _listPyFileRecursively(os.path.join(path, pathPrefix)) 422 423 # chop off the base path to get a list of "relative" import space paths 424 importPaths = [x[len(path) + 1:] for x in files] 425 426 # remove some common candidates, like .svn subdirs, or containing - 427 isNotSvn = lambda x: x.find('.svn') == -1 428 importPaths = filter(isNotSvn, importPaths) 429 isNotDashed = lambda x: x.find('-') == -1 430 importPaths = filter(isNotDashed, importPaths) 431 432 # convert paths to module namespace 433 endModules = [common.pathToModuleName(x) for x in importPaths] 434 435 # remove all not starting with prefix 436 isInPrefix = lambda x: x and x.startswith(prefix) 437 endModules = filter(isInPrefix, endModules) 438 439 # sort them so that depending packages are after higher-up packages 440 endModules.sort() 441 442 # make unique 443 res = {} 444 for b in endModules: res[b] = 1 445 446 return res.keys()
447 448 # singleton factory function 449 __packager = None 450
451 -def getPackager():
452 """ 453 Return the (unique) packager. 454 455 @rtype: L{Packager} 456 """ 457 global __packager 458 if not __packager: 459 __packager = Packager() 460 461 return __packager
462