1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 objects and functions used in dealing with packages
24 """
25
26 import ihooks
27 import os
28 import sys
29 import glob
30
31 from flumotion.common import log, common
32 from twisted.python import rebuild, reflect
33
35 """
36 I am an import Hooks object that makes sure that every package that gets
37 loaded has every necessary path in the module's __path__ list.
38
39 @type packager: L{Packager}
40 """
41 packager = None
42
63
65 """
66 I am an object through which package paths can be registered, to support
67 the partitioning of the module import namespace across bundles.
68 """
69
70 logCategory = 'packager'
71
73 self._paths = {}
74 self._packages = {}
75 self.install()
76
78 """
79 Install our custom importer that uses bundled packages.
80 """
81 self.debug('installing custom importer')
82 self._hooks = PackageHooks()
83 self._hooks.packager = self
84 self._importer = ihooks.ModuleImporter()
85 self._importer.set_hooks(self._hooks)
86 self._importer.install()
87
89 """
90 Return all absolute paths to the top level of a tree from which
91 (part of) the given package name can be imported.
92 """
93 if packageName not in self._packages:
94 return None
95
96 return [self._paths[key] for key in self._packages[packageName]]
97
99 """
100 Register a given path as a path that can be imported from.
101 Used to support partition of bundled code or import code from various
102 uninstalled location.
103
104 sys.path will also be changed to include this, and remove references
105 to older packagePath's for the same bundle.
106
107 @param packagePath: path to add under which the module namespaces live,
108 (ending in an md5sum, for flumotion purposes)
109 @type packagePath: string
110 @param key a unique id for the package being registered
111 @type key: string
112 @param prefix: prefix of the packages to be considered
113 @type prefix: string
114 """
115
116 new = True
117 packagePath = os.path.abspath(packagePath)
118 if not os.path.exists(packagePath):
119 log.warning('bundle',
120 'registering a non-existing package path %s' % packagePath)
121
122 self.log('registering packagePath %s' % packagePath)
123
124
125 if key in self._paths:
126 oldPath = self._paths[key]
127 if packagePath == oldPath:
128 self.log('already registered %s for key %s' % (
129 packagePath, key))
130 return
131 new = False
132
133
134
135
136
137 if not os.path.isdir(packagePath):
138 log.warning('bundle', 'package path not a dir: %s',
139 packagePath)
140 packageNames = []
141 else:
142 packageNames = _findPackageCandidates(packagePath, prefix)
143
144 if not packageNames:
145 log.log('bundle',
146 'packagePath %s does not have candidates starting with %s' %
147 (packagePath, prefix))
148 return
149 packageNames.sort()
150
151 self.log('package candidates %r' % packageNames)
152
153 if not new:
154
155 log.log('bundle',
156 'replacing old path %s with new path %s for key %s' % (
157 oldPath, packagePath, key))
158
159 if oldPath in sys.path:
160 log.log('bundle',
161 'removing old packagePath %s from sys.path' % oldPath)
162 sys.path.remove(oldPath)
163
164
165 for keys in self._packages.values():
166 if key in keys:
167 keys.remove(key)
168
169 self._paths[key] = packagePath
170
171
172 if not packagePath in sys.path:
173 self.log('adding packagePath %s to sys.path' % packagePath)
174 sys.path.insert(0, packagePath)
175
176
177 for name in packageNames:
178 if name not in self._packages:
179 self._packages[name] = [key]
180 else:
181 self._packages[name].insert(0, key)
182
183 self.log('packagePath %s has packageNames %r' % (
184 packagePath, packageNames))
185
186
187 packageNames.reverse()
188
189 for packageName in packageNames:
190 if packageName not in sys.modules:
191 continue
192 self.log('fixing up %s ...' % packageName)
193
194
195 package = sys.modules.get(packageName)
196 for path in package.__path__:
197 if not new and path.startswith(oldPath):
198 self.log('%s.__path__ before remove %r' % (
199 packageName, package.__path__))
200 self.log('removing old %s from %s.__path__' % (
201 path, name))
202 package.__path__.remove(path)
203 self.log('%s.__path__ after remove %r' % (
204 packageName, package.__path__))
205
206
207
208
209
210 newPath = os.path.join(packagePath,
211 packageName.replace('.', os.sep))
212
213
214
215
216 if len(package.__path__) == 0:
217
218
219
220
221 self.debug('WARN: package %s does not have __path__ values' % (
222 packageName))
223 elif package.__path__[0] == newPath:
224 self.log('path %s already at start of %s.__path__' % (
225 newPath, packageName))
226 continue
227
228 if newPath in package.__path__:
229 package.__path__.remove(newPath)
230 self.log('moving %s to front of %s.__path__' % (
231 newPath, packageName))
232 else:
233 self.log('inserting new %s into %s.__path__' % (
234 newPath, packageName))
235 package.__path__.insert(0, newPath)
236
237
238
239
240
241
242
243
244 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__))
245
246
247
248 if not new:
249 self.log('finding end module candidates')
250 if not os.path.isdir(packagePath):
251 log.warning('bundle', 'package path not a dir: %s',
252 path)
253 moduleNames = []
254 else:
255 moduleNames = findEndModuleCandidates(packagePath, prefix)
256 self.log('end module candidates to rebuild: %r' % moduleNames)
257 for name in moduleNames:
258 if name in sys.modules:
259
260 self.log("rebuilding non-package module %s" % name)
261 try:
262 module = reflect.namedAny(name)
263 except AttributeError:
264 log.warning('bundle',
265 "could not reflect non-package module %s" % name)
266 continue
267
268 if hasattr(module, '__path__'):
269 self.log('rebuilding module %s with paths %r' % (name,
270 module.__path__))
271 rebuild.rebuild(module)
272
273
274
275 self.log('registered packagePath %s for key %s' % (packagePath, key))
276
278 """
279 Unregister all previously registered package paths, and uninstall
280 the custom importer.
281 """
282 for path in self._paths.values():
283 if path in sys.path:
284 self.log('removing packagePath %s from sys.path' % path)
285 sys.path.remove(path)
286 self._paths = {}
287 self._packages = {}
288 self.debug('uninstalling custom importer')
289 self._importer.uninstall()
290
292 """
293 I'm similar to os.listdir, but I work recursively and only return
294 directories containing python code.
295
296 @param path: the path
297 @type path: string
298 """
299 retval = []
300 try:
301 files = os.listdir(path)
302 except OSError:
303 pass
304 else:
305 for f in files:
306
307 p = os.path.join(path, f)
308 if os.path.isdir(p) and f != '.svn':
309 retval += _listDirRecursively(p)
310
311 if glob.glob(os.path.join(path, '*.py*')):
312 retval.append(path)
313
314 return retval
315
317 """
318 I'm similar to os.listdir, but I work recursively and only return
319 files representing python non-package modules.
320
321 @param path: the path
322 @type path: string
323
324 @rtype: list
325 @returns: list of files underneath the given path containing python code
326 """
327 retval = []
328
329
330 dirs = _listDirRecursively(path)
331
332 for dir in dirs:
333 pyfiles = glob.glob(os.path.join(dir, '*.py*'))
334 dontkeep = glob.glob(os.path.join(dir, '*__init__.py*'))
335 for f in dontkeep:
336 if f in pyfiles:
337 pyfiles.remove(f)
338
339 retval.extend(pyfiles)
340
341 return retval
342
344 """
345 I take a directory and return a list of candidate python packages
346 under that directory that start with the given prefix.
347 A package is a module containing modules; typically the directory
348 with the same name as the package contains __init__.py
349
350 @param path: the path
351 @type path: string
352 """
353
354
355 dirs = _listDirRecursively(os.path.join(path, prefix))
356
357
358 bundlePaths = [x[len(path) + 1:] for x in dirs]
359
360
361 isNotSvn = lambda x: x.find('.svn') == -1
362 bundlePaths = filter(isNotSvn, bundlePaths)
363 isNotDashed = lambda x: x.find('-') == -1
364 bundlePaths = filter(isNotDashed, bundlePaths)
365
366
367 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths]
368
369
370
371 packages = {}
372 for name in bundlePackages:
373 packages[name] = 1
374 parts = name.split(".")
375 build = None
376 for p in parts:
377 if not build:
378 build = p
379 else:
380 build = build + "." + p
381 packages[build] = 1
382
383 bundlePackages = packages.keys()
384
385
386 bundlePackages.sort()
387
388 return bundlePackages
389
391 """
392 I take a directory and return a list of candidate python end modules
393 (i.e., non-package modules) for the given module prefix.
394
395 @param path: the path under which to search for end modules
396 @type path: string
397 @param prefix: module prefix to check candidates under
398 @type prefix: string
399 """
400 pathPrefix = "/".join(prefix.split("."))
401 files = _listPyFileRecursively(os.path.join(path, pathPrefix))
402
403
404 importPaths = [x[len(path) + 1:] for x in files]
405
406
407 isNotSvn = lambda x: x.find('.svn') == -1
408 importPaths = filter(isNotSvn, importPaths)
409 isNotDashed = lambda x: x.find('-') == -1
410 importPaths = filter(isNotDashed, importPaths)
411
412
413 endModules = [common.pathToModuleName(x) for x in importPaths]
414
415
416 isInPrefix = lambda x: x and x.startswith(prefix)
417 endModules = filter(isInPrefix, endModules)
418
419
420 endModules.sort()
421
422
423 res = {}
424 for b in endModules: res[b] = 1
425
426 return res.keys()
427
428
429 __packager = None
430
442