1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 objects and functions used in dealing with packages
24 """
25
26 import ihooks
27 import os
28 import sys
29 import glob
30
31 from flumotion.common import log, common
32 from twisted.python import rebuild, reflect
33
35 """
36 I am overriding ihook's ModuleImporter's import_module() method to
37 accept (and ignore) the 'level' keyword argument that appeared in
38 the built-in __import__() function in python2.5.
39
40 While no built-in modules in python2.5 seem to use that keyword
41 argument, 'encodings' module in python2.6 does and so it breaks if
42 used together with ihooks.
43
44 I make no attempt to properly support the 'level' argument -
45 ihooks didn't make it into py3k, and the only use in python2.6
46 we've seen so far, in 'encodings', serves as a performance hint
47 and it seems that can be ignored with no difference in behaviour.
48 """
49
50 - def import_module(self, name, globals=None, locals=None, fromlist=None,
51 level=-1):
52
53 return ihooks.ModuleImporter.import_module(self, name, globals,
54 locals, fromlist)
55
56
58 """
59 I am an import Hooks object that makes sure that every package that gets
60 loaded has every necessary path in the module's __path__ list.
61
62 @type packager: L{Packager}
63 """
64 packager = None
65
86
88 """
89 I am an object through which package paths can be registered, to support
90 the partitioning of the module import namespace across bundles.
91 """
92
93 logCategory = 'packager'
94
96 self._paths = {}
97 self._packages = {}
98 self.install()
99
101 """
102 Install our custom importer that uses bundled packages.
103 """
104 self.debug('installing custom importer')
105 self._hooks = PackageHooks()
106 self._hooks.packager = self
107 if sys.version_info < (2, 6):
108 self._importer = ihooks.ModuleImporter()
109 else:
110 self.debug('python2.6 or later detected - using patched'
111 ' ModuleImporter')
112 self._importer = _PatchedModuleImporter()
113 self._importer.set_hooks(self._hooks)
114 self._importer.install()
115
117 """
118 Return all absolute paths to the top level of a tree from which
119 (part of) the given package name can be imported.
120 """
121 if not packageName in self._packages.keys():
122 return None
123
124 return [self._paths[key] for key in self._packages[packageName]]
125
127 """
128 Register a given path as a path that can be imported from.
129 Used to support partition of bundled code or import code from various
130 uninstalled location.
131
132 sys.path will also be changed to include this, and remove references
133 to older packagePath's for the same bundle.
134
135 @param packagePath: path to add under which the module namespaces live,
136 (ending in an md5sum, for flumotion purposes)
137 @type packagePath: string
138 @param key a unique id for the package being registered
139 @type key: string
140 @param prefix: prefix of the packages to be considered
141 @type prefix: string
142 """
143
144 new = True
145 packagePath = os.path.abspath(packagePath)
146 if not os.path.exists(packagePath):
147 log.warning('bundle',
148 'registering a non-existing package path %s' % packagePath)
149
150 self.log('registering packagePath %s' % packagePath)
151
152
153 if key in self._paths.keys():
154 oldPath = self._paths[key]
155 if packagePath == oldPath:
156 self.log('already registered %s for key %s' % (
157 packagePath, key))
158 return
159 new = False
160
161
162
163
164
165 packageNames = _findPackageCandidates(packagePath, prefix)
166
167 if not packageNames:
168 log.log('bundle',
169 'packagePath %s does not have candidates starting with %s' %
170 (packagePath, prefix))
171 return
172 packageNames.sort()
173
174 self.log('package candidates %r' % packageNames)
175
176 if not new:
177
178 log.log('bundle',
179 'replacing old path %s with new path %s for key %s' % (
180 oldPath, packagePath, key))
181
182 if oldPath in sys.path:
183 log.log('bundle',
184 'removing old packagePath %s from sys.path' % oldPath)
185 sys.path.remove(oldPath)
186
187
188 for keys in self._packages.values():
189 if key in keys:
190 keys.remove(key)
191
192 self._paths[key] = packagePath
193
194
195 if not packagePath in sys.path:
196 self.log('adding packagePath %s to sys.path' % packagePath)
197 sys.path.insert(0, packagePath)
198
199
200 for name in packageNames:
201 if not name in self._packages.keys():
202 self._packages[name] = [key]
203 else:
204 self._packages[name].insert(0, key)
205
206 self.log('packagePath %s has packageNames %r' % (
207 packagePath, packageNames))
208
209
210 packageNames.reverse()
211
212 for packageName in packageNames:
213 if packageName not in sys.modules.keys():
214 continue
215 self.log('fixing up %s ...' % packageName)
216
217
218 package = sys.modules.get(packageName)
219 for path in package.__path__:
220 if not new and path.startswith(oldPath):
221 self.log('%s.__path__ before remove %r' % (
222 packageName, package.__path__))
223 self.log('removing old %s from %s.__path__' % (
224 path, name))
225 package.__path__.remove(path)
226 self.log('%s.__path__ after remove %r' % (
227 packageName, package.__path__))
228
229
230
231
232
233 newPath = os.path.join(packagePath,
234 packageName.replace('.', os.sep))
235
236
237
238
239 if len(package.__path__) == 0:
240
241
242
243
244 self.debug('WARN: package %s does not have __path__ values' % (
245 packageName))
246 elif package.__path__[0] == newPath:
247 self.log('path %s already at start of %s.__path__' % (
248 newPath, packageName))
249 continue
250
251 if newPath in package.__path__:
252 package.__path__.remove(newPath)
253 self.log('moving %s to front of %s.__path__' % (
254 newPath, packageName))
255 else:
256 self.log('inserting new %s into %s.__path__' % (
257 newPath, packageName))
258 package.__path__.insert(0, newPath)
259
260
261
262
263
264
265
266
267 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__))
268
269
270
271 if not new:
272 self.log('finding end module candidates')
273 moduleNames = findEndModuleCandidates(packagePath, prefix)
274 self.log('end module candidates to rebuild: %r' % moduleNames)
275 for name in moduleNames:
276 if name in sys.modules:
277
278 self.log("rebuilding non-package module %s" % name)
279 try:
280 module = reflect.namedAny(name)
281 except AttributeError:
282 log.warning('bundle',
283 "could not reflect non-package module %s" % name)
284 continue
285
286 if hasattr(module, '__path__'):
287 self.log('rebuilding module %s with paths %r' % (name,
288 module.__path__))
289 rebuild.rebuild(module)
290
291
292
293 self.log('registered packagePath %s for key %s' % (packagePath, key))
294
296 """
297 Unregister all previously registered package paths, and uninstall
298 the custom importer.
299 """
300 for path in self._paths.values():
301 if path in sys.path:
302 self.log('removing packagePath %s from sys.path' % path)
303 sys.path.remove(path)
304 self._paths = {}
305 self._packages = {}
306 self.debug('uninstalling custom importer')
307 self._importer.uninstall()
308
310 """
311 I'm similar to os.listdir, but I work recursively and only return
312 directories containing python code.
313
314 @param path: the path
315 @type path: string
316 """
317 retval = []
318
319 if not os.path.isdir(path):
320 return retval
321
322 try:
323 files = os.listdir(path)
324 except OSError:
325 pass
326 else:
327 for f in files:
328
329 retval += _listDirRecursively(os.path.join(path, f))
330
331 if glob.glob(os.path.join(path, '*.py*')):
332 retval.append(path)
333
334 return retval
335
337 """
338 I'm similar to os.listdir, but I work recursively and only return
339 files representing python non-package modules.
340
341 @param path: the path
342 @type path: string
343
344 @rtype: list
345 @returns: list of files underneath the given path containing python code
346 """
347 retval = []
348
349
350 dirs = _listDirRecursively(path)
351
352 for dir in dirs:
353 pyfiles = glob.glob(os.path.join(dir, '*.py*'))
354 dontkeep = glob.glob(os.path.join(dir, '*__init__.py*'))
355 for f in dontkeep:
356 if f in pyfiles:
357 pyfiles.remove(f)
358
359 retval.extend(pyfiles)
360
361 return retval
362
364 """
365 I take a directory and return a list of candidate python packages
366 under that directory that start with the given prefix.
367 A package is a module containing modules; typically the directory
368 with the same name as the package contains __init__.py
369
370 @param path: the path
371 @type path: string
372 """
373
374
375 dirs = _listDirRecursively(os.path.join(path, prefix))
376
377
378 bundlePaths = [x[len(path) + 1:] for x in dirs]
379
380
381 isNotSvn = lambda x: x.find('.svn') == -1
382 bundlePaths = filter(isNotSvn, bundlePaths)
383 isNotDashed = lambda x: x.find('-') == -1
384 bundlePaths = filter(isNotDashed, bundlePaths)
385
386
387 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths]
388
389
390
391 packages = {}
392 for name in bundlePackages:
393 packages[name] = 1
394 parts = name.split(".")
395 build = None
396 for p in parts:
397 if not build:
398 build = p
399 else:
400 build = build + "." + p
401 packages[build] = 1
402
403 bundlePackages = packages.keys()
404
405
406 bundlePackages.sort()
407
408 return bundlePackages
409
411 """
412 I take a directory and return a list of candidate python end modules
413 (i.e., non-package modules) for the given module prefix.
414
415 @param path: the path under which to search for end modules
416 @type path: string
417 @param prefix: module prefix to check candidates under
418 @type prefix: string
419 """
420 pathPrefix = "/".join(prefix.split("."))
421 files = _listPyFileRecursively(os.path.join(path, pathPrefix))
422
423
424 importPaths = [x[len(path) + 1:] for x in files]
425
426
427 isNotSvn = lambda x: x.find('.svn') == -1
428 importPaths = filter(isNotSvn, importPaths)
429 isNotDashed = lambda x: x.find('-') == -1
430 importPaths = filter(isNotDashed, importPaths)
431
432
433 endModules = [common.pathToModuleName(x) for x in importPaths]
434
435
436 isInPrefix = lambda x: x and x.startswith(prefix)
437 endModules = filter(isInPrefix, endModules)
438
439
440 endModules.sort()
441
442
443 res = {}
444 for b in endModules: res[b] = 1
445
446 return res.keys()
447
448
449 __packager = None
450
462