1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 parsing of configuration files
24 """
25
26
27
28
29
30 import os
31 from xml.dom import minidom, Node
32 from xml.parsers import expat
33
34 from twisted.python import reflect
35
36 from flumotion.common import log, errors, common, registry, fxml
37 from flumotion.configure import configure
38
39 from errors import ConfigError, ComponentWorkerConfigError
40
41
42 BOOL_TRUE_VALUES = ['True', 'true', '1', 'Yes', 'yes']
43
44 -class ConfigEntryComponent(log.Loggable):
45 "I represent a <component> entry in a planet config file"
46 nice = 0
47 logCategory = 'config'
48 - def __init__(self, name, parent, type, config, defs, worker):
49 self.name = name
50 self.parent = parent
51 self.type = type
52 self.config = config
53 self.defs = defs
54 self.worker = worker
55
58
61
62 - def getParent(self):
64
65 - def getConfigDict(self):
67
68 - def getWorker(self):
70
72 "I represent a <flow> entry in a planet config file"
73 - def __init__(self, name):
74 self.name = name
75 self.components = {}
76
78 "I represent a <manager> entry in a planet config file"
79 - def __init__(self, name, host, port, transport, certificate, bouncer,
80 fludebug, plugs):
81 self.name = name
82 self.host = host
83 self.port = port
84 self.transport = transport
85 self.certificate = certificate
86 self.bouncer = bouncer
87 self.fludebug = fludebug
88 self.plugs = plugs
89
91 "I represent a <atmosphere> entry in a planet config file"
94
96 return len(self.components)
97
100 """
101 @param file: The file to parse, either as an open file object,
102 or as the name of a file to open.
103 @type file: str or file.
104 """
105 self.add(file)
106
107 - def add(self, file):
108 """
109 @param file: The file to parse, either as an open file object,
110 or as the name of a file to open.
111 @type file: str or file.
112 """
113 try:
114 self.path = os.path.split(file.name)[0]
115 except AttributeError:
116
117 self.path = None
118
119 try:
120 self.doc = self.getRoot(file)
121 except fxml.ParserError, e:
122 raise ConfigError(e.args[0])
123
126
128 return self.doc.toxml()
129
131 return [float(subnode.childNodes[0].data) for subnode in nodes]
132
134 return [int(subnode.childNodes[0].data) for subnode in nodes]
135
137 return [long(subnode.childNodes[0].data) for subnode in nodes]
138
142
144 values = []
145 for subnode in nodes:
146 try:
147 data = subnode.childNodes[0].data
148 except IndexError:
149 continue
150
151
152
153 try:
154 data = str(data)
155 except UnicodeEncodeError:
156 pass
157 if '\n' in data:
158 parts = [x.strip() for x in data.split('\n')]
159 data = ' '.join(parts)
160 values.append(data)
161
162 return values
163
165 values = []
166 for subnode in nodes:
167 try:
168 data = str(subnode.childNodes[0].data)
169 values.append(data)
170 except IndexError:
171 pass
172
173 string = "".join(values)
174 return [string, ]
175
177 def fraction_from_string(string):
178 parts = string.split('/')
179 if len(parts) == 2:
180 return (int(parts[0]), int(parts[1]))
181 elif len(parts) == 1:
182 return (int(parts[0]), 1)
183 else:
184 raise ConfigError("Invalid fraction: %s", string)
185 return [fraction_from_string(subnode.childNodes[0].data)
186 for subnode in nodes]
187
189 """
190 Parse a <property>-containing node in a configuration XML file.
191 Ignores any subnode not named <property>.
192
193 @param node: The <properties> XML node to parse.
194 @type node: L{xml.dom.Node}
195 @param properties: The set of valid properties.
196 @type properties: list of
197 L{flumotion.common.registry.RegistryEntryProperty}
198 @param error: An exception factory for parsing errors.
199 @type error: Callable that maps str => Exception.
200
201 @returns: The parsed properties, as a dict of name => value.
202 Absent optional properties will not appear in the dict.
203 """
204
205
206
207
208 import locale
209 locale.setlocale(locale.LC_NUMERIC, "C")
210
211 properties_given = {}
212 for subnode in node.childNodes:
213 if subnode.nodeName == 'property':
214 if not subnode.hasAttribute('name'):
215 raise error("<property> must have a name attribute")
216 name = subnode.getAttribute('name')
217 if not name in properties_given:
218 properties_given[name] = []
219 properties_given[name].append(subnode)
220
221 property_specs = dict([(x.name, x) for x in properties])
222
223 config = {}
224 for name, nodes in properties_given.items():
225 if not name in property_specs:
226 raise error("%s: unknown property" % name)
227
228 definition = property_specs[name]
229
230 if not definition.multiple and len(nodes) > 1:
231 raise error("%s: multiple value specified but not "
232 "allowed" % name)
233
234 parsers = {'string': self.get_string_values,
235 'rawstring': self.get_raw_string_values,
236 'int': self.get_int_values,
237 'long': self.get_long_values,
238 'bool': self.get_bool_values,
239 'float': self.get_float_values,
240 'fraction': self.get_fraction_values}
241
242 if not definition.type in parsers:
243 raise error("%s: invalid property type %s"
244 % (name, definition.type))
245
246 values = parsers[definition.type](nodes)
247
248 if values == []:
249 continue
250
251 if not definition.multiple:
252 values = values[0]
253
254 config[name] = values
255
256 for name, definition in property_specs.items():
257 if definition.isRequired() and not name in config:
258 raise error("%s: required but unspecified property"
259 % name)
260
261 return config
262
264
265
266 socket, type = self.parseAttributes(node, ('socket', 'type'))
267
268 try:
269 defs = registry.getRegistry().getPlug(type)
270 except KeyError:
271 raise ConfigError("unknown plug type: %s" % type)
272
273 possible_node_names = ['property']
274 for subnode in node.childNodes:
275 if (subnode.nodeType == Node.COMMENT_NODE
276 or subnode.nodeType == Node.TEXT_NODE):
277 continue
278 elif subnode.nodeName not in possible_node_names:
279 raise ConfigError("Invalid subnode of <plug>: %s"
280 % subnode.nodeName)
281
282 property_specs = defs.getProperties()
283 def err(str):
284 return ConfigError('%s: %s' % (type, str))
285 properties = self.parseProperties(node, property_specs, err)
286
287 return {'type':type, 'socket':socket, 'properties':properties}
288
290
291
292
293
294
295 plugs = {}
296 for socket in sockets:
297 plugs[socket] = []
298 def addplug(plug):
299 if plug['socket'] not in sockets:
300 raise ConfigError("Component does not support "
301 "sockets of type %s" % plug['socket'])
302 plugs[plug['socket']].append(plug)
303
304 parsers = {'plug': (self.parsePlug, addplug)}
305 self.parseFromTable(node, parsers)
306
307 return plugs
308
309
310
312 """
313 I represent a planet configuration file for Flumotion.
314
315 @ivar manager: A L{ConfigEntryManager} containing options for the manager
316 section, filled in at construction time.
317 @ivar atmosphere: A L{ConfigEntryAtmosphere}, filled in when parse() is
318 called.
319 @ivar flows: A list of L{ConfigEntryFlow}, filled in when parse() is
320 called.
321 """
322 logCategory = 'config'
323
334
335 - def parse(self, noRegistry=False):
336
337
338
339
340
341
342
343 root = self.doc.documentElement
344
345 if not root.nodeName == 'planet':
346 raise ConfigError("unexpected root node': %s" % root.nodeName)
347
348 for node in root.childNodes:
349 if node.nodeType != Node.ELEMENT_NODE:
350 continue
351
352 if noRegistry and node.nodeName != 'manager':
353 continue
354
355 if node.nodeName == 'atmosphere':
356 entry = self._parseAtmosphere(node)
357 self.atmosphere.components.update(entry)
358 elif node.nodeName == 'flow':
359 entry = self._parseFlow(node)
360 self.flows.append(entry)
361 elif node.nodeName == 'manager':
362 entry = self._parseManager(node, noRegistry)
363 self.manager = entry
364 else:
365 raise ConfigError("unexpected node under 'planet': %s" % node.nodeName)
366
368
369
370
371
372
373 ret = {}
374 for child in node.childNodes:
375 if (child.nodeType == Node.TEXT_NODE or
376 child.nodeType == Node.COMMENT_NODE):
377 continue
378
379 if child.nodeName == "component":
380 component = self._parseComponent(child, 'atmosphere')
381 else:
382 raise ConfigError("unexpected 'atmosphere' node: %s" % child.nodeName)
383
384 ret[component.name] = component
385 return ret
386
388 """
389 Parse a <component></component> block.
390
391 @rtype: L{ConfigEntryComponent}
392 """
393
394
395
396
397
398 if not node.hasAttribute('name'):
399 raise ConfigError("<component> must have a name attribute")
400 if not node.hasAttribute('type'):
401 raise ConfigError("<component> must have a type attribute")
402 if forManager:
403 if node.hasAttribute('worker'):
404 raise ComponentWorkerConfigError("components in manager"
405 "cannot have workers")
406 else:
407 if (not node.hasAttribute('worker')
408 or not node.getAttribute('worker')):
409
410 raise ComponentWorkerConfigError("<component> must have a"
411 " worker attribute")
412 version = None
413 if node.hasAttribute('version'):
414 versionString = node.getAttribute("version")
415 try:
416 versionList = map(int, versionString.split('.'))
417 if len(versionList) == 3:
418 version = tuple(versionList) + (0,)
419 elif len(versionList) == 4:
420 version = tuple(versionList)
421 except:
422 raise ComponentWorkerConfigError("<component> version not"
423 " parseable")
424
425
426 if not version:
427 version = configure.versionTuple
428
429 type = str(node.getAttribute('type'))
430 name = str(node.getAttribute('name'))
431 if forManager:
432 worker = None
433 else:
434 worker = str(node.getAttribute('worker'))
435
436
437
438
439
440 config = { 'name': name,
441 'parent': parent,
442 'type': type,
443 'avatarId': common.componentId(parent, name),
444 'version': version
445 }
446
447 try:
448 defs = registry.getRegistry().getComponent(type)
449 except KeyError:
450 raise errors.UnknownComponentError(
451 "unknown component type: %s" % type)
452
453 possible_node_names = ['source', 'clock-master', 'property',
454 'plugs']
455 for subnode in node.childNodes:
456 if subnode.nodeType == Node.COMMENT_NODE:
457 continue
458 elif subnode.nodeType == Node.TEXT_NODE:
459
460
461 continue
462 elif subnode.nodeName not in possible_node_names:
463 raise ConfigError("Invalid subnode of <component>: %s"
464 % subnode.nodeName)
465
466
467 config['feed'] = defs.getFeeders()
468
469 sources = self._parseSources(node, defs)
470 if sources:
471 config['source'] = sources
472
473 config['clock-master'] = self._parseClockMaster(node)
474 config['plugs'] = self._parsePlugs(node, defs.getSockets())
475
476 properties = defs.getProperties()
477
478 self.debug('Parsing component: %s' % name)
479 def err(str):
480 return ConfigError('%s: %s' % (name, str))
481 config['properties'] = self.parseProperties(node, properties, err)
482
483
484
485 return ConfigEntryComponent(name, parent, type, config, defs, worker)
486
488
489
490
491
492
493
494 if not node.hasAttribute('name'):
495 raise ConfigError("<flow> must have a name attribute")
496
497 name = str(node.getAttribute('name'))
498 if name == 'atmosphere':
499 raise ConfigError("<flow> cannot have 'atmosphere' as name")
500 if name == 'manager':
501 raise ConfigError("<flow> cannot have 'manager' as name")
502
503 flow = ConfigEntryFlow(name)
504
505 for child in node.childNodes:
506 if (child.nodeType == Node.TEXT_NODE or
507 child.nodeType == Node.COMMENT_NODE):
508 continue
509
510 if child.nodeName == "component":
511 component = self._parseComponent(child, name)
512 else:
513 raise ConfigError("unexpected 'flow' node: %s" % child.nodeName)
514
515 flow.components[component.name] = component
516
517
518 masters = [x for x in flow.components.values()
519 if x.config['clock-master']]
520 if len(masters) > 1:
521 raise ConfigError("Multiple clock masters in flow %s: %r"
522 % (name, masters))
523
524 need_sync = [(x.defs.getClockPriority(), x)
525 for x in flow.components.values()
526 if x.defs.getNeedsSynchronization()]
527 need_sync.sort()
528 need_sync = [x[1] for x in need_sync]
529
530 if need_sync:
531 if masters:
532 master = masters[0]
533 else:
534 master = need_sync[-1]
535
536 masterAvatarId = master.config['avatarId']
537 self.info("Setting %s as clock master" % masterAvatarId)
538
539 for c in need_sync:
540 c.config['clock-master'] = masterAvatarId
541 elif masters:
542 self.info('master clock specified, but no synchronization '
543 'necessary -- ignoring')
544 masters[0].config['clock-master'] = None
545
546 return flow
547
549
550
551
552
553
554 name = None
555 host = None
556 port = None
557 transport = None
558 certificate = None
559 bouncer = None
560 fludebug = None
561 plugs = {}
562
563 manager_sockets = \
564 ['flumotion.component.plugs.adminaction.AdminAction',
565 'flumotion.component.plugs.lifecycle.ManagerLifecycle',
566 'flumotion.component.plugs.identity.IdentityProvider']
567 for k in manager_sockets:
568 plugs[k] = []
569
570 if node.hasAttribute('name'):
571 name = str(node.getAttribute('name'))
572
573 for child in node.childNodes:
574 if (child.nodeType == Node.TEXT_NODE or
575 child.nodeType == Node.COMMENT_NODE):
576 continue
577
578 if child.nodeName == "host":
579 host = self._nodeGetString("host", child)
580 elif child.nodeName == "port":
581 port = self._nodeGetInt("port", child)
582 elif child.nodeName == "transport":
583 transport = self._nodeGetString("transport", child)
584 if not transport in ('tcp', 'ssl'):
585 raise ConfigError("<transport> must be ssl or tcp")
586 elif child.nodeName == "certificate":
587 certificate = self._nodeGetString("certificate", child)
588 elif child.nodeName == "component":
589 if noRegistry:
590 continue
591
592 if bouncer:
593 raise ConfigError(
594 "<manager> section can only have one <component>")
595 bouncer = self._parseComponent(child, 'manager',
596 forManager=True)
597 elif child.nodeName == "plugs":
598 if noRegistry:
599 continue
600
601 for k, v in self._parsePlugs(node, manager_sockets).items():
602 plugs[k].extend(v)
603 elif child.nodeName == "debug":
604 fludebug = self._nodeGetString("debug", child)
605 else:
606 raise ConfigError("unexpected '%s' node: %s" % (
607 node.nodeName, child.nodeName))
608
609
610
611 return ConfigEntryManager(name, host, port, transport, certificate,
612 bouncer, fludebug, plugs)
613
615 try:
616 value = int(node.firstChild.nodeValue)
617 except ValueError:
618 raise ConfigError("<%s> value must be an integer" % name)
619 except AttributeError:
620 raise ConfigError("<%s> value not specified" % name)
621 return value
622
624 try:
625 value = str(node.firstChild.nodeValue)
626 except AttributeError:
627 raise ConfigError("<%s> value not specified" % name)
628 return value
629
631
632 eaters = dict([(x.getName(), x) for x in defs.getEaters()])
633
634 nodes = []
635 for subnode in node.childNodes:
636 if subnode.nodeName == 'source':
637 nodes.append(subnode)
638 strings = self.get_string_values(nodes)
639
640
641
642
643 required = [x for x in eaters.values() if x.getRequired()]
644 multiple = [x for x in eaters.values() if x.getMultiple()]
645
646 if len(strings) == 0 and required:
647 raise ConfigError("Component %s wants to eat on %s, but no "
648 "source specified"
649 % (node.nodeName, eaters.keys()[0]))
650 elif len(strings) > 1 and not multiple:
651 raise ConfigError("Component %s does not support multiple "
652 "sources feeding %s (%r)"
653 % (node.nodeName, eaters.keys()[0], strings))
654
655 return strings
656
658 nodes = []
659 for subnode in node.childNodes:
660 if subnode.nodeName == 'clock-master':
661 nodes.append(subnode)
662 bools = self.get_bool_values(nodes)
663
664 if len(bools) > 1:
665 raise ConfigError("Only one <clock-master> node allowed")
666
667 if bools and bools[0]:
668 return True
669 else:
670 return None
671
673 plugs = {}
674 for socket in sockets:
675 plugs[socket] = []
676 for subnode in node.childNodes:
677 if subnode.nodeName == 'plugs':
678 newplugs = self.parsePlugs(subnode, sockets)
679 for socket in sockets:
680 plugs[socket].extend(newplugs[socket])
681 return plugs
682
683
685 """
686 Get all component entries from both atmosphere and all flows
687 from the configuration.
688
689 @rtype: dictionary of /parent/name -> L{ConfigEntryComponent}
690 """
691 entries = {}
692 if self.atmosphere and self.atmosphere.components:
693 for c in self.atmosphere.components.values():
694 path = common.componentId('atmosphere', c.name)
695 entries[path] = c
696
697 for flowEntry in self.flows:
698 for c in flowEntry.components.values():
699 path = common.componentId(c.parent, c.name)
700 entries[path] = c
701
702 return entries
703
705 """
706 Admin configuration file parser.
707 """
708 logCategory = 'config'
709
711 """
712 @param file: The file to parse, either as an open file object,
713 or as the name of a file to open.
714 @type file: str or file.
715 """
716 self.plugs = {}
717 for socket in sockets:
718 self.plugs[socket] = []
719
720
721 BaseConfigParser.__init__(self, file)
722
724
725
726 root = self.doc.documentElement
727 if not root.nodeName == 'admin':
728 raise ConfigError("unexpected root node': %s" % root.nodeName)
729
730 def parseplugs(node):
731 return self.parsePlugs(node, self.plugs.keys())
732 def addplugs(plugs):
733 for socket in plugs:
734 try:
735 self.plugs[socket].extend(plugs[socket])
736 except KeyError:
737 raise ConfigError("Admin does not support "
738 "sockets of type %s" % socket)
739 parsers = {'plugs': (parseplugs, addplugs)}
740
741 self.parseFromTable(root, parsers)
742
743 - def add(self, file):
744 """
745 @param file: The file to parse, either as an open file object,
746 or as the name of a file to open.
747 @type file: str or file.
748 """
749 BaseConfigParser.add(self, file)
750 self._parse()
751