1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 bundles of files used to implement caching over the network
24 """
25
26 import errno
27 import md5
28 import os
29 import sys
30 import zipfile
31 import StringIO
32
33 from flumotion.common import errors, dag
34
35 __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket']
36
38 """
39 I represent one file as managed by a bundler.
40 """
41 - def __init__(self, source, destination):
42 self.source = source
43 self.destination = destination
44 self._last_md5sum = self.md5sum()
45 self._last_timestamp = self.timestamp()
46 self.zipped = False
47
49 """
50 Calculate the md5sum of the given file.
51
52 @returns: the md5 sum a 32 character string of hex characters.
53 """
54 data = open(self.source, "r").read()
55 return md5.new(data).hexdigest()
56
58 """
59 @returns: the last modified timestamp for the file.
60 """
61 return os.path.getmtime(self.source)
62
64 """
65 Check if the file has changed since it was last checked.
66
67 @rtype: boolean
68 """
69
70
71
72
73 if not self.zipped:
74 return True
75
76 timestamp = self.timestamp()
77
78 if timestamp <= self._last_timestamp:
79 return False
80 self._last_timestamp = timestamp
81
82
83 md5sum = self.md5sum()
84 if self._last_md5sum != md5sum:
85 self._last_md5sum = md5sum
86 return True
87
88 return False
89
91 """
92 I am a bundle of files, represented by a zip file and md5sum.
93 """
98
100 """
101 Set the bundle to the given data representation of the zip file.
102 """
103 self.zip = zip
104 self.md5sum = md5.new(self.zip).hexdigest()
105
107 """
108 Get the bundle's zip data.
109 """
110 return self.zip
111
113 """
114 I unbundle bundles by unpacking them in the given directory
115 under directories with the bundle's md5sum.
116 """
119
121 """
122 Return the full path where a bundle with the given name and md5sum
123 would be unbundled to.
124 """
125 return os.path.join(self._undir, name, md5sum)
126
132
134 """
135 Unbundle the given bundle.
136
137 @type bundle: L{flumotion.common.bundle.Bundle}
138
139 @rtype: string
140 @returns: the full path to the directory where it was unpacked
141 """
142 dir = self.unbundlePath(bundle)
143
144 filelike = StringIO.StringIO(bundle.getZip())
145 zip = zipfile.ZipFile(filelike, "r")
146 zip.testzip()
147
148 filepaths = zip.namelist()
149 for filepath in filepaths:
150 path = os.path.join(dir, filepath)
151 parent = os.path.split(path)[0]
152 try:
153 os.makedirs(parent)
154 except OSError, err:
155
156 if err.errno != errno.EEXIST or not os.path.isdir(parent):
157 raise
158 data = zip.read(filepath)
159 handle = open(path, 'wb')
160 handle.write(data)
161 handle.close()
162 return dir
163
165 """
166 I bundle files into a bundle so they can be cached remotely easily.
167 """
169 """
170 Create a new bundle.
171 """
172 self._files = {}
173 self.name = name
174 self._bundle = Bundle(name)
175
176 - def add(self, source, destination = None):
177 """
178 Add files to the bundle.
179
180 @param source: the path to the file to add to the bundle.
181 @param destination: a relative path to store this file in in the bundle.
182 If unspecified, this will be stored in the top level.
183
184 @returns: the path the file got stored as
185 """
186 if destination == None:
187 destination = os.path.split(source)[1]
188 self._files[source] = BundledFile(source, destination)
189 return destination
190
192 """
193 Bundle the files registered with the bundler.
194
195 @rtype: L{flumotion.common.bundle.Bundle}
196 """
197
198
199 if not self._bundle.getZip():
200 self._bundle.setZip(self._buildzip())
201 return self._bundle
202
203 update = False
204 for file in self._files.values():
205 if file.hasChanged():
206 update = True
207
208 if update:
209 self._bundle.setZip(self._buildzip())
210
211 return self._bundle
212
213
214
216 filelike = StringIO.StringIO()
217 zip = zipfile.ZipFile(filelike, "w")
218 for path in self._files.keys():
219 bf = self._files[path]
220 self._files[path].zipped = True
221 zip.write(bf.source, bf.destination)
222 zip.close()
223 data = filelike.getvalue()
224 filelike.close()
225 return data
226
228 """
229 I manage bundlers that are registered through me.
230 """
232 """
233 Create a new bundler basket.
234 """
235 self._bundlers = {}
236
237 self._files = {}
238 self._imports = {}
239
240 self._graph = dag.DAG()
241
242 - def add(self, bundleName, source, destination = None):
243 """
244 Add files to the bundler basket for the given bundle.
245
246 @param bundleName: the name of the bundle this file is a part of
247 @param source: the path to the file to add to the bundle
248 @param destination: a relative path to store this file in in the bundle.
249 If unspecified, this will be stored in the top level
250 """
251
252 if not bundleName in self._bundlers.keys():
253 bundler = Bundler(bundleName)
254 self._bundlers[bundleName] = bundler
255 else:
256 bundler = self._bundlers[bundleName]
257
258
259 location = bundler.add(source, destination)
260 if self._files.has_key(location):
261 raise Exception("Cannot add %s to bundle %s, already in %s" % (
262 location, bundleName, self._files[location]))
263 self._files[location] = bundleName
264
265
266 package = None
267 if location.endswith('.py'):
268 package = location[:-3]
269 elif location.endswith('.pyc'):
270 package = location[:-4]
271
272 if package:
273 if package.endswith('__init__'):
274 package = os.path.split(package)[0]
275
276 package = ".".join(package.split('/'))
277 if self._imports.has_key(package):
278 raise Exception("Bundler %s already has import %s" % (
279 bundleName, package))
280 self._imports[package] = bundleName
281
282 - def depend(self, depender, *dependencies):
283 """
284 Make the given bundle depend on the other given bundles.
285
286 @type depender: string
287 @type dependencies: list of strings
288 """
289
290 if not self._graph.hasNode(depender):
291 self._graph.addNode(depender)
292 for dep in dependencies:
293 if not self._graph.hasNode(dep):
294 self._graph.addNode(dep)
295 self._graph.addEdge(depender, dep)
296
298 """
299 Return names of all the dependencies of this bundle, including this
300 bundle itself.
301 The dependencies are returned in a correct depending order.
302 """
303 if not bundlerName in self._bundlers:
304 raise errors.NoBundleError('Unknown bundle %s' % bundlerName)
305 return [bundlerName,] + self._graph.getOffspring(bundlerName)
306
308 """
309 Return the bundle by name, or None if not found.
310 """
311 if self._bundlers.has_key(bundlerName):
312 return self._bundlers[bundlerName]
313 return None
314
316 """
317 Return the bundler name by import statement, or None if not found.
318 """
319 if self._imports.has_key(importString):
320 return self._imports[importString]
321 return None
322
324 """
325 Return the bundler name by filename, or None if not found.
326 """
327 if self._files.has_key(filename):
328 return self._files[filename]
329 return None
330
332 """
333 I am a bundler, with the extension that I can also bundle other
334 bundlers.
335
336 The effect is that when you call bundle() on a me, you get one
337 bundle with a union of all subbundlers' files, in addition to any
338 loose files that you added to me.
339 """
340 - def __init__(self, name='merged-bundle'):
343
345 """Add to me all of the files managed by another bundler.
346
347 @param bundler: The bundler whose files you want in this
348 bundler.
349 @type bundler: L{Bundler}
350 """
351 if bundler.name not in self._subbundlers:
352 self._subbundlers[bundler.name] = bundler
353 for bfile in bundler._files.values():
354 self.add(bfile.source, bfile.destination)
355
357 """
358 @returns: A list of all of the bundlers that have been added to
359 me.
360 """
361 return self._subbundlers.values()
362
364 """
365 Make a bundle from a subset of all loaded modules, also writing out
366 a registry file that can apply to that subset of the global
367 registry. Suitable for use as a FLU_ATEXIT handler.
368
369 @param outfile: The path to which a zip file will be written.
370 @type outfile: str
371 @param outreg: The path to which a registry file will be written.
372 @type outreg: str
373 @param prefixes: A list of prefixes to which to limit the export. If
374 not given, package up all modules. For example, "flumotion" would
375 limit the output to modules that start with "flumotion".
376 @type prefixes: list of str
377 """
378 from flumotion.common import registry, log
379 from twisted.python import reflect
380
381 def getUsedModules(prefixes):
382 ret = {}
383 for modname in sys.modules:
384 if prefixes and not filter(modname.startswith, prefixes):
385 continue
386 try:
387 module = reflect.namedModule(modname)
388 if hasattr(module, '__file__'):
389 ret[modname] = module
390 else:
391 log.info('makebundle', 'Module %s has no file', module)
392 except ImportError:
393 log.info('makebundle', 'Could not import %s', modname)
394 return ret
395
396 def calculateModuleBundleMap():
397 allbundles = registry.getRegistry().getBundles()
398 ret = {}
399 for bundle in allbundles:
400 for directory in bundle.getDirectories():
401 for file in directory.getFiles():
402 path = os.path.join(directory.getName(), file.getLocation())
403 parts = path.split(os.path.sep)
404 if parts[-1].startswith('__init__.py'):
405 parts.pop()
406 elif parts[-1].endswith('.py'):
407 parts[-1] = parts[-1][:-3]
408 else:
409
410 continue
411 modname = '.'.join(parts)
412 ret[modname] = bundle
413 return ret
414
415 def makeMergedBundler(modules, modulebundlemap):
416 ret = MergedBundler()
417 basket = registry.getRegistry().makeBundlerBasket()
418 for modname in modules:
419 modfilename = modules[modname].__file__
420 if modname in modulebundlemap:
421 bundleName = modulebundlemap[modname].getName()
422 for depBundleName in basket.getDependencies(bundleName):
423 ret.addBundler(basket.getBundlerByName(depBundleName))
424 else:
425 if modfilename.endswith('.pyc'):
426 modfilename = modfilename[:-1]
427 if os.path.isdir(modfilename):
428 with_init = os.path.join(modfilename, '__init__.py')
429 if os.path.exists(with_init):
430 modfilename = with_init
431 nparts = len(modname.split('.'))
432 if '__init__' in modfilename:
433 nparts += 1
434 relpath = os.path.join(*modfilename.split(os.path.sep)[-nparts:])
435 ret.add(modfilename, relpath)
436 return ret
437
438 modules = getUsedModules(prefixes)
439 modulebundlemap = calculateModuleBundleMap()
440 bundler = makeMergedBundler(modules, modulebundlemap)
441
442 print 'Writing bundle to', outfile
443 open(outfile, 'w').write(bundler.bundle().getZip())
444
445 print 'Writing registry to', outreg
446 bundlers_used = [b.name for b in bundler.getSubBundlers()]
447 regwriter = registry.RegistrySubsetWriter(onlyBundles=bundlers_used)
448 regwriter.dump(open(outreg, 'w'))
449