1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 objects and functions used in dealing with packages
24 """
25
26 import ihooks
27 import os
28 import sys
29 import glob
30
31 from flumotion.common import log, common
32 from twisted.python import rebuild, reflect
33
35 """
36 I am an import Hooks object that makes sure that every package that gets
37 loaded has every necessary path in the module's __path__ list.
38
39 @type packager: L{Packager}
40 """
41 packager = None
42
63
65 """
66 I am an object through which package paths can be registered, to support
67 the partitioning of the module import namespace across bundles.
68 """
69
70 logCategory = 'packager'
71
73 self._paths = {}
74 self._packages = {}
75 self.install()
76
78 """
79 Install our custom importer that uses bundled packages.
80 """
81 self.debug('installing custom importer')
82 self._hooks = PackageHooks()
83 self._hooks.packager = self
84 self._importer = ihooks.ModuleImporter()
85 self._importer.set_hooks(self._hooks)
86 self._importer.install()
87
89 """
90 Return all absolute paths to the top level of a tree from which
91 (part of) the given package name can be imported.
92 """
93 if not packageName in self._packages.keys():
94 return None
95
96 return [self._paths[key] for key in self._packages[packageName]]
97
99 """
100 Register a given path as a path that can be imported from.
101 Used to support partition of bundled code or import code from various
102 uninstalled location.
103
104 sys.path will also be changed to include this, and remove references
105 to older packagePath's for the same bundle.
106
107 @param packagePath: path to add under which the module namespaces live,
108 (ending in an md5sum, for flumotion purposes)
109 @type packagePath: string
110 @param key a unique id for the package being registered
111 @type key: string
112 @param prefix: prefix of the packages to be considered
113 @type prefix: string
114 """
115
116 new = True
117 packagePath = os.path.abspath(packagePath)
118 if not os.path.exists(packagePath):
119 log.warning('bundle',
120 'registering a non-existing package path %s' % packagePath)
121
122 self.log('registering packagePath %s' % packagePath)
123
124
125 if key in self._paths.keys():
126 oldPath = self._paths[key]
127 if packagePath == oldPath:
128 self.log('already registered %s for key %s' % (
129 packagePath, key))
130 return
131 new = False
132
133
134
135
136
137 packageNames = _findPackageCandidates(packagePath, prefix)
138
139 if not packageNames:
140 log.log('bundle',
141 'packagePath %s does not have candidates starting with %s' %
142 (packagePath, prefix))
143 return
144 packageNames.sort()
145
146 self.log('package candidates %r' % packageNames)
147
148 if not new:
149
150 log.log('bundle',
151 'replacing old path %s with new path %s for key %s' % (
152 oldPath, packagePath, key))
153
154 if oldPath in sys.path:
155 log.log('bundle',
156 'removing old packagePath %s from sys.path' % oldPath)
157 sys.path.remove(oldPath)
158
159
160 for keys in self._packages.values():
161 if key in keys:
162 keys.remove(key)
163
164 self._paths[key] = packagePath
165
166
167 if not packagePath in sys.path:
168 self.log('adding packagePath %s to sys.path' % packagePath)
169 sys.path.insert(0, packagePath)
170
171
172 for name in packageNames:
173 if not name in self._packages.keys():
174 self._packages[name] = [key]
175 else:
176 self._packages[name].insert(0, key)
177
178 self.log('packagePath %s has packageNames %r' % (
179 packagePath, packageNames))
180
181
182 packageNames.reverse()
183
184 for packageName in packageNames:
185 if packageName not in sys.modules.keys():
186 continue
187 self.log('fixing up %s ...' % packageName)
188
189
190 package = sys.modules.get(packageName)
191 for path in package.__path__:
192 if not new and path.startswith(oldPath):
193 self.log('%s.__path__ before remove %r' % (
194 packageName, package.__path__))
195 self.log('removing old %s from %s.__path__' % (
196 path, name))
197 package.__path__.remove(path)
198 self.log('%s.__path__ after remove %r' % (
199 packageName, package.__path__))
200
201
202
203
204
205 newPath = os.path.join(packagePath,
206 packageName.replace('.', os.sep))
207
208
209
210
211 if len(package.__path__) == 0:
212
213
214
215
216 self.debug('WARN: package %s does not have __path__ values' % (
217 packageName))
218 elif package.__path__[0] == newPath:
219 self.log('path %s already at start of %s.__path__' % (
220 newPath, packageName))
221 continue
222
223 if newPath in package.__path__:
224 package.__path__.remove(newPath)
225 self.log('moving %s to front of %s.__path__' % (
226 newPath, packageName))
227 else:
228 self.log('inserting new %s into %s.__path__' % (
229 newPath, packageName))
230 package.__path__.insert(0, newPath)
231
232
233
234
235
236
237
238
239 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__))
240
241
242
243 if not new:
244 self.log('finding end module candidates')
245 moduleNames = findEndModuleCandidates(packagePath, prefix)
246 self.log('end module candidates to rebuild: %r' % moduleNames)
247 for name in moduleNames:
248 if name in sys.modules:
249
250 self.log("rebuilding non-package module %s" % name)
251 try:
252 module = reflect.namedAny(name)
253 except AttributeError:
254 log.warning('bundle',
255 "could not reflect non-package module %s" % name)
256 continue
257
258 if hasattr(module, '__path__'):
259 self.log('rebuilding module %s with paths %r' % (name,
260 module.__path__))
261 rebuild.rebuild(module)
262
263
264
265 self.log('registered packagePath %s for key %s' % (packagePath, key))
266
268 """
269 Unregister all previously registered package paths, and uninstall
270 the custom importer.
271 """
272 for path in self._paths.values():
273 if path in sys.path:
274 self.log('removing packagePath %s from sys.path' % path)
275 sys.path.remove(path)
276 self._paths = {}
277 self._packages = {}
278 self.debug('uninstalling custom importer')
279 self._importer.uninstall()
280
282 """
283 I'm similar to os.listdir, but I work recursively and only return
284 directories containing python code.
285
286 @param path: the path
287 @type path: string
288 """
289 retval = []
290
291 if not os.path.isdir(path):
292 return retval
293
294 try:
295 files = os.listdir(path)
296 except OSError:
297 pass
298 else:
299 for f in files:
300
301 retval += _listDirRecursively(os.path.join(path, f))
302
303 if glob.glob(os.path.join(path, '*.py*')):
304 retval.append(path)
305
306 return retval
307
309 """
310 I'm similar to os.listdir, but I work recursively and only return
311 files representing python non-package modules.
312
313 @param path: the path
314 @type path: string
315
316 @rtype: list
317 @returns: list of files underneath the given path containing python code
318 """
319 retval = []
320
321
322 dirs = _listDirRecursively(path)
323
324 for dir in dirs:
325 pyfiles = glob.glob(os.path.join(dir, '*.py*'))
326 dontkeep = glob.glob(os.path.join(dir, '*__init__.py*'))
327 for f in dontkeep:
328 if f in pyfiles:
329 pyfiles.remove(f)
330
331 retval.extend(pyfiles)
332
333 return retval
334
336 """
337 I take a directory and return a list of candidate python packages
338 under that directory that start with the given prefix.
339 A package is a module containing modules; typically the directory
340 with the same name as the package contains __init__.py
341
342 @param path: the path
343 @type path: string
344 """
345
346
347 dirs = _listDirRecursively(os.path.join(path, prefix))
348
349
350 bundlePaths = [x[len(path) + 1:] for x in dirs]
351
352
353 isNotSvn = lambda x: x.find('.svn') == -1
354 bundlePaths = filter(isNotSvn, bundlePaths)
355 isNotDashed = lambda x: x.find('-') == -1
356 bundlePaths = filter(isNotDashed, bundlePaths)
357
358
359 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths]
360
361
362
363 packages = {}
364 for name in bundlePackages:
365 packages[name] = 1
366 parts = name.split(".")
367 build = None
368 for p in parts:
369 if not build:
370 build = p
371 else:
372 build = build + "." + p
373 packages[build] = 1
374
375 bundlePackages = packages.keys()
376
377
378 bundlePackages.sort()
379
380 return bundlePackages
381
383 """
384 I take a directory and return a list of candidate python end modules
385 (i.e., non-package modules) for the given module prefix.
386
387 @param path: the path under which to search for end modules
388 @type path: string
389 @param prefix: module prefix to check candidates under
390 @type prefix: string
391 """
392 pathPrefix = "/".join(prefix.split("."))
393 files = _listPyFileRecursively(os.path.join(path, pathPrefix))
394
395
396 importPaths = [x[len(path) + 1:] for x in files]
397
398
399 isNotSvn = lambda x: x.find('.svn') == -1
400 importPaths = filter(isNotSvn, importPaths)
401 isNotDashed = lambda x: x.find('-') == -1
402 importPaths = filter(isNotDashed, importPaths)
403
404
405 endModules = [common.pathToModuleName(x) for x in importPaths]
406
407
408 isInPrefix = lambda x: x and x.startswith(prefix)
409 endModules = filter(isInPrefix, endModules)
410
411
412 endModules.sort()
413
414
415 res = {}
416 for b in endModules: res[b] = 1
417
418 return res.keys()
419
420
421 __packager = None
422
434