Package flumotion :: Package common :: Module package
[hide private]

Source Code for Module flumotion.common.package

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_package -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  objects and functions used in dealing with packages 
 24  """ 
 25   
 26  import ihooks 
 27  import os 
 28  import sys 
 29  import glob 
 30   
 31  from flumotion.common import log, common 
 32  from twisted.python import rebuild, reflect 
 33   
34 -class PackageHooks(ihooks.Hooks):
35 """ 36 I am an import Hooks object that makes sure that every package that gets 37 loaded has every necessary path in the module's __path__ list. 38 39 @type packager: L{Packager} 40 """ 41 packager = None 42
43 - def load_package(self, name, filename, file=None):
44 # this is only ever called the first time a package is imported 45 log.log('packager', 'load_package %s' % name) 46 ret = ihooks.Hooks.load_package(self, name, filename, file) 47 48 m = sys.modules[name] 49 50 packagePaths = self.packager.getPathsForPackage(name) 51 if not packagePaths: 52 return ret 53 54 # get full paths to the package 55 paths = [os.path.join(path, name.replace('.', os.sep)) for path in packagePaths] 56 for path in paths: 57 if not path in m.__path__: 58 log.log('packager', 'adding path %s for package %s' % ( 59 path, name)) 60 m.__path__.append(path) 61 62 return ret
63
64 -class Packager(log.Loggable):
65 """ 66 I am an object through which package paths can be registered, to support 67 the partitioning of the module import namespace across bundles. 68 """ 69 70 logCategory = 'packager' 71
72 - def __init__(self):
73 self._paths = {} # key -> package path registered with that key 74 self._packages = {} # package name -> keys for that package 75 self.install()
76
77 - def install(self):
78 """ 79 Install our custom importer that uses bundled packages. 80 """ 81 self.debug('installing custom importer') 82 self._hooks = PackageHooks() 83 self._hooks.packager = self 84 self._importer = ihooks.ModuleImporter() 85 self._importer.set_hooks(self._hooks) 86 self._importer.install()
87
88 - def getPathsForPackage(self, packageName):
89 """ 90 Return all absolute paths to the top level of a tree from which 91 (part of) the given package name can be imported. 92 """ 93 if packageName not in self._packages: 94 return None 95 96 return [self._paths[key] for key in self._packages[packageName]]
97
98 - def registerPackagePath(self, packagePath, key, prefix='flumotion'):
99 """ 100 Register a given path as a path that can be imported from. 101 Used to support partition of bundled code or import code from various 102 uninstalled location. 103 104 sys.path will also be changed to include this, and remove references 105 to older packagePath's for the same bundle. 106 107 @param packagePath: path to add under which the module namespaces live, 108 (ending in an md5sum, for flumotion purposes) 109 @type packagePath: string 110 @param key a unique id for the package being registered 111 @type key: string 112 @param prefix: prefix of the packages to be considered 113 @type prefix: string 114 """ 115 116 new = True 117 packagePath = os.path.abspath(packagePath) 118 if not os.path.exists(packagePath): 119 log.warning('bundle', 120 'registering a non-existing package path %s' % packagePath) 121 122 self.log('registering packagePath %s' % packagePath) 123 124 # check if a packagePath for this bundle was already registered 125 if key in self._paths: 126 oldPath = self._paths[key] 127 if packagePath == oldPath: 128 self.log('already registered %s for key %s' % ( 129 packagePath, key)) 130 return 131 new = False 132 133 # Find the packages in the path and sort them, 134 # the following algorithm only works if they're sorted. 135 # By sorting the list we can ensure that a parent package 136 # is always processed before one of its children 137 if not os.path.isdir(packagePath): 138 log.warning('bundle', 'package path not a dir: %s', 139 packagePath) 140 packageNames = [] 141 else: 142 packageNames = _findPackageCandidates(packagePath, prefix) 143 144 if not packageNames: 145 log.log('bundle', 146 'packagePath %s does not have candidates starting with %s' % 147 (packagePath, prefix)) 148 return 149 packageNames.sort() 150 151 self.log('package candidates %r' % packageNames) 152 153 if not new: 154 # it already existed, and now it's a different path 155 log.log('bundle', 156 'replacing old path %s with new path %s for key %s' % ( 157 oldPath, packagePath, key)) 158 159 if oldPath in sys.path: 160 log.log('bundle', 161 'removing old packagePath %s from sys.path' % oldPath) 162 sys.path.remove(oldPath) 163 164 # clear this key from our name -> key cache 165 for keys in self._packages.values(): 166 if key in keys: 167 keys.remove(key) 168 169 self._paths[key] = packagePath 170 171 # put packagePath at the top of sys.path if not in there 172 if not packagePath in sys.path: 173 self.log('adding packagePath %s to sys.path' % packagePath) 174 sys.path.insert(0, packagePath) 175 176 # update our name->keys cache 177 for name in packageNames: 178 if name not in self._packages: 179 self._packages[name] = [key] 180 else: 181 self._packages[name].insert(0, key) 182 183 self.log('packagePath %s has packageNames %r' % ( 184 packagePath, packageNames)) 185 # since we want sub-modules to be fixed up before parent packages, 186 # we reverse the list 187 packageNames.reverse() 188 189 for packageName in packageNames: 190 if packageName not in sys.modules: 191 continue 192 self.log('fixing up %s ...' % packageName) 193 194 # the package is imported, so mess with __path__ and rebuild 195 package = sys.modules.get(packageName) 196 for path in package.__path__: 197 if not new and path.startswith(oldPath): 198 self.log('%s.__path__ before remove %r' % ( 199 packageName, package.__path__)) 200 self.log('removing old %s from %s.__path__' % ( 201 path, name)) 202 package.__path__.remove(path) 203 self.log('%s.__path__ after remove %r' % ( 204 packageName, package.__path__)) 205 206 # move the new path to the top 207 # insert at front because FLU_REGISTRY_PATH paths should override 208 # base components, and because subsequent reload() should prefer 209 # the latest registered path 210 newPath = os.path.join(packagePath, 211 packageName.replace('.', os.sep)) 212 213 # if path already at position 0, everything's fine 214 # if it's in there at another place, it needs to move to front 215 # if not in there, it needs to be put in front 216 if len(package.__path__) == 0: 217 # FIXME: this seems to happen to e.g. flumotion.component.base 218 # even when it was just rebuilt and had the __path__ set 219 # can be triggered by choosing a admin_gtk depending on 220 # the base admin_gtk where the base admin_gtk changes 221 self.debug('WARN: package %s does not have __path__ values' % ( 222 packageName)) 223 elif package.__path__[0] == newPath: 224 self.log('path %s already at start of %s.__path__' % ( 225 newPath, packageName)) 226 continue 227 228 if newPath in package.__path__: 229 package.__path__.remove(newPath) 230 self.log('moving %s to front of %s.__path__' % ( 231 newPath, packageName)) 232 else: 233 self.log('inserting new %s into %s.__path__' % ( 234 newPath, packageName)) 235 package.__path__.insert(0, newPath) 236 237 # Rebuilding these packages just to get __path__ fixed in 238 # seems not necessary - but re-enable it if it breaks 239 # self.log('rebuilding package %s from paths %r' % (packageName, 240 # package.__path__)) 241 # rebuild.rebuild(package) 242 # self.log('rebuilt package %s with paths %r' % (packageName, 243 # package.__path__)) 244 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__)) 245 246 # now rebuild all non-package modules in this packagePath if this 247 # is not a new package 248 if not new: 249 self.log('finding end module candidates') 250 if not os.path.isdir(packagePath): 251 log.warning('bundle', 'package path not a dir: %s', 252 path) 253 moduleNames = [] 254 else: 255 moduleNames = findEndModuleCandidates(packagePath, prefix) 256 self.log('end module candidates to rebuild: %r' % moduleNames) 257 for name in moduleNames: 258 if name in sys.modules: 259 # fixme: isn't sys.modules[name] sufficient? 260 self.log("rebuilding non-package module %s" % name) 261 try: 262 module = reflect.namedAny(name) 263 except AttributeError: 264 log.warning('bundle', 265 "could not reflect non-package module %s" % name) 266 continue 267 268 if hasattr(module, '__path__'): 269 self.log('rebuilding module %s with paths %r' % (name, 270 module.__path__)) 271 rebuild.rebuild(module) 272 #if paths: 273 # module.__path__ = paths 274 275 self.log('registered packagePath %s for key %s' % (packagePath, key))
276
277 - def unregister(self):
278 """ 279 Unregister all previously registered package paths, and uninstall 280 the custom importer. 281 """ 282 for path in self._paths.values(): 283 if path in sys.path: 284 self.log('removing packagePath %s from sys.path' % path) 285 sys.path.remove(path) 286 self._paths = {} 287 self._packages = {} 288 self.debug('uninstalling custom importer') 289 self._importer.uninstall()
290
291 -def _listDirRecursively(path):
292 """ 293 I'm similar to os.listdir, but I work recursively and only return 294 directories containing python code. 295 296 @param path: the path 297 @type path: string 298 """ 299 retval = [] 300 try: 301 files = os.listdir(path) 302 except OSError: 303 pass 304 else: 305 for f in files: 306 # this only adds directories since files are not returned 307 p = os.path.join(path, f) 308 if os.path.isdir(p) and f != '.svn': 309 retval += _listDirRecursively(p) 310 311 if glob.glob(os.path.join(path, '*.py*')): 312 retval.append(path) 313 314 return retval
315
316 -def _listPyFileRecursively(path):
317 """ 318 I'm similar to os.listdir, but I work recursively and only return 319 files representing python non-package modules. 320 321 @param path: the path 322 @type path: string 323 324 @rtype: list 325 @returns: list of files underneath the given path containing python code 326 """ 327 retval = [] 328 329 # get all the dirs containing python code 330 dirs = _listDirRecursively(path) 331 332 for dir in dirs: 333 pyfiles = glob.glob(os.path.join(dir, '*.py*')) 334 dontkeep = glob.glob(os.path.join(dir, '*__init__.py*')) 335 for f in dontkeep: 336 if f in pyfiles: 337 pyfiles.remove(f) 338 339 retval.extend(pyfiles) 340 341 return retval
342
343 -def _findPackageCandidates(path, prefix='flumotion'):
344 """ 345 I take a directory and return a list of candidate python packages 346 under that directory that start with the given prefix. 347 A package is a module containing modules; typically the directory 348 with the same name as the package contains __init__.py 349 350 @param path: the path 351 @type path: string 352 """ 353 # this function also "guesses" candidate packages when __init__ is missing 354 # so a bundle with only a subpackage is also detected 355 dirs = _listDirRecursively(os.path.join(path, prefix)) 356 357 # chop off the base path to get a list of "relative" bundlespace paths 358 bundlePaths = [x[len(path) + 1:] for x in dirs] 359 360 # remove some common candidates, like .svn subdirs, or containing - 361 isNotSvn = lambda x: x.find('.svn') == -1 362 bundlePaths = filter(isNotSvn, bundlePaths) 363 isNotDashed = lambda x: x.find('-') == -1 364 bundlePaths = filter(isNotDashed, bundlePaths) 365 366 # convert paths to module namespace 367 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths] 368 369 # now make sure that all parent packages for each package are listed 370 # as well 371 packages = {} 372 for name in bundlePackages: 373 packages[name] = 1 374 parts = name.split(".") 375 build = None 376 for p in parts: 377 if not build: 378 build = p 379 else: 380 build = build + "." + p 381 packages[build] = 1 382 383 bundlePackages = packages.keys() 384 385 # sort them so that depending packages are after higher-up packages 386 bundlePackages.sort() 387 388 return bundlePackages
389
390 -def findEndModuleCandidates(path, prefix='flumotion'):
391 """ 392 I take a directory and return a list of candidate python end modules 393 (i.e., non-package modules) for the given module prefix. 394 395 @param path: the path under which to search for end modules 396 @type path: string 397 @param prefix: module prefix to check candidates under 398 @type prefix: string 399 """ 400 pathPrefix = "/".join(prefix.split(".")) 401 files = _listPyFileRecursively(os.path.join(path, pathPrefix)) 402 403 # chop off the base path to get a list of "relative" import space paths 404 importPaths = [x[len(path) + 1:] for x in files] 405 406 # remove some common candidates, like .svn subdirs, or containing - 407 isNotSvn = lambda x: x.find('.svn') == -1 408 importPaths = filter(isNotSvn, importPaths) 409 isNotDashed = lambda x: x.find('-') == -1 410 importPaths = filter(isNotDashed, importPaths) 411 412 # convert paths to module namespace 413 endModules = [common.pathToModuleName(x) for x in importPaths] 414 415 # remove all not starting with prefix 416 isInPrefix = lambda x: x and x.startswith(prefix) 417 endModules = filter(isInPrefix, endModules) 418 419 # sort them so that depending packages are after higher-up packages 420 endModules.sort() 421 422 # make unique 423 res = {} 424 for b in endModules: res[b] = 1 425 426 return res.keys()
427 428 # singleton factory function 429 __packager = None 430
431 -def getPackager():
432 """ 433 Return the (unique) packager. 434 435 @rtype: L{Packager} 436 """ 437 global __packager 438 if not __packager: 439 __packager = Packager() 440 441 return __packager
442