Module configuration
[hide private]
[frames] | no frames]

Source Code for Module configuration

  1  #============================================================================  
  2  #Name        : configuration.py  
  3  #Part of     : Helium  
  4  # 
  5  #Partly Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
  6  #All rights reserved. 
  7  #This component and the accompanying materials are made available 
  8  #under the terms of the License "Eclipse Public License v1.0" 
  9  #which accompanies this distribution, and is available 
 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
 11  # 
 12  #Initial Contributors: 
 13  #Nokia Corporation - initial contribution. 
 14  # 
 15  #Contributors: 
 16  # 
 17  #Description:  
 18  #  
 19  #============================================================================== 
 20   
 21  """Defines an interface for accessing configurations, typically for SW builds. 
 22   
 23  This interface is generally based on the Jakarta Commons Configuration API. A 
 24  configuration is a collection of properties. Builders create Configuration 
 25  objects from various source inputs. 
 26   
 27  """ 
 28   
 29   
 30  import copy 
 31  import logging 
 32  import os 
 33  import re 
 34  import sys 
 35  import time 
 36  import types 
 37  import UserDict 
 38  import xml.dom.minidom 
 39   
 40   
 41  _logger = logging.getLogger('configuration') 
 42  logging.basicConfig(level=logging.INFO) 
 43   
44 -class Configuration(object, UserDict.DictMixin):
45 """ Base Configuration objects. """ 46 47 key_re = re.compile(r'\${(?P<name>[._a-zA-Z0-9]+)}', re.M) 48
49 - def __init__(self, data=None):
50 """ Initialization. """ 51 if data == None: 52 data = {} 53 self.name = None 54 self.data = data
55
56 - def __getitem__(self, key, interpolate=True):
57 """ Get an item from the configuration via dictionary interface. """ 58 if interpolate: 59 return self.interpolate(self.data[key]) 60 return self.data[key]
61
62 - def __setitem__(self, key, item):
63 """ Set an item from the configuration via dictionary interface. """ 64 self.data[key] = item
65
66 - def __delitem__(self, key):
67 """ Remove an item from the configuration via dictionary interface. """ 68 del self.data[key]
69
70 - def keys(self):
71 """ Get the list of item keys. """ 72 return self.data.keys()
73
74 - def match_name(self, name):
75 """ See if the given name matches the name of this configuration. """ 76 return self.name == name
77
78 - def get(self, key, default_value):
79 """ Get an item from the configuration. """ 80 try: 81 return self.__getitem__(key) 82 except KeyError: 83 return default_value
84
85 - def get_list(self, key, default_value):
86 try: 87 itemlist = self.__getitem__(key) 88 if not isinstance(itemlist, types.ListType): 89 itemlist = [itemlist] 90 return itemlist 91 except KeyError: 92 return default_value
93
94 - def get_int(self, key, default_value):
95 try: 96 value = self.__getitem__(key) 97 return int(value) 98 except KeyError: 99 return default_value
100
101 - def get_boolean(self, key, default_value):
102 try: 103 value = self.__getitem__(key) 104 return value == "true" or value == "yes" or value == "1" 105 except KeyError: 106 return default_value
107
108 - def interpolate(self, value):
109 """ Search for patterns of the form '${..}' and insert matching values. """ 110 if isinstance(value, types.ListType): 111 value = [self.interpolate(elem) for elem in value] 112 else: 113 if isinstance(value, types.StringType) or \ 114 isinstance(value, types.UnicodeType) or \ 115 isinstance(value, types.ListType): 116 for match in self.key_re.finditer(value): 117 for property_name in match.groups(): 118 if self.has_key(property_name): 119 property_value = self.__getitem__(property_name) 120 if isinstance(property_value, types.ListType): 121 property_value = ",".join(property_value) 122 else: 123 property_value = re.sub(r'\\', r'\\\\', property_value, re.M) 124 value = re.sub('\${' + property_name + '}', property_value, value, re.M) 125 return value
126
127 - def __str__(self):
128 return self.__class__.__name__ + '[' + str(self.name) + ']'
129 130
131 -class PropertiesConfiguration(Configuration):
132 """ A Configuration that parses a plain text properties file. 133 134 This typically follows the java.util.Properties format. 135 136 Note: This code is mostly based on this recipe 137 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496795. 138 Copyright (c) Anand Balachandran Pillai 139 """
140 - def __init__(self, stream=None, data=None):
141 Configuration.__init__(self, data) 142 143 self.othercharre = re.compile(r'(?<!\\)(\s*\=)|(?<!\\)(\s*\:)') 144 self.othercharre2 = re.compile(r'(\s*\=)|(\s*\:)') 145 self.bspacere = re.compile(r'\\(?!\s$)') 146 147 if stream is not None: 148 self.load(stream)
149 150 # def __str__(self): 151 # s='{' 152 # for key,value in self.data.items(): 153 # s = ''.join((s,key,'=',value,', ')) 154 # 155 # s=''.join((s[:-2],'}')) 156 # return s 157
158 - def __parse(self, lines):
159 """ Parse a list of lines and create 160 an internal property dictionary """ 161 162 # Every line in the file must consist of either a comment 163 # or a key-value pair. A key-value pair is a line consisting 164 # of a key which is a combination of non-white space characters 165 # The separator character between key-value pairs is a '=', 166 # ':' or a whitespace character not including the newline. 167 # If the '=' or ':' characters are found, in the line, even 168 # keys containing whitespace chars are allowed. 169 170 # A line with only a key according to the rules above is also 171 # fine. In such case, the value is considered as the empty string. 172 # In order to include characters '=' or ':' in a key or value, 173 # they have to be properly escaped using the backslash character. 174 175 # Some examples of valid key-value pairs: 176 # 177 # key value 178 # key=value 179 # key:value 180 # key value1,value2,value3 181 # key value1,value2,value3 \ 182 # value4, value5 183 # key 184 # This key= this value 185 # key = value1 value2 value3 186 187 # Any line that starts with a '#' is considerered a comment 188 # and skipped. Also any trailing or preceding whitespaces 189 # are removed from the key/value. 190 191 # This is a line parser. It parses the 192 # contents like by line. 193 194 lineno = 0 195 i = iter(lines) 196 197 for line in i: 198 lineno += 1 199 line = line.strip() 200 # Skip null lines 201 if not line: continue 202 # Skip lines which are comments 203 if line[0] == '#': continue 204 205 # Position of first separation char 206 sepidx = -1 207 # A flag for performing wspace re check 208 #flag = 0 209 # Check for valid space separation 210 # First obtain the max index to which we 211 # can search. 212 m = self.othercharre.search(line) 213 if m: 214 first, last = m.span() 215 start, end = 0, first 216 #flag = 1 217 wspacere = re.compile(r'(?<![\\\=\:])(\s)') 218 else: 219 if self.othercharre2.search(line): 220 # Check if either '=' or ':' is present 221 # in the line. If they are then it means 222 # they are preceded by a backslash. 223 224 # This means, we need to modify the 225 # wspacere a bit, not to look for 226 # : or = characters. 227 wspacere = re.compile(r'(?<![\\])(\s)') 228 start, end = 0, len(line) 229 230 m2 = wspacere.search(line, start, end) 231 if m2: 232 # print 'Space match=>',line 233 # Means we need to split by space. 234 first, last = m2.span() 235 sepidx = first 236 elif m: 237 # print 'Other match=>',line 238 # No matching wspace char found, need 239 # to split by either '=' or ':' 240 first, last = m.span() 241 sepidx = last - 1 242 # print line[sepidx] 243 244 245 # If the last character is a backslash 246 # it has to be preceded by a space in which 247 # case the next line is read as part of the 248 # same property 249 while line[-1] == '\\': 250 # Read next line 251 try: 252 nextline = i.next() 253 nextline = nextline.strip() 254 lineno += 1 255 # This line will become part of the value 256 line = line[:-1] + nextline 257 except StopIteration: 258 break 259 260 # Now split to key,value according to separation char 261 if sepidx != -1: 262 key, value = line[:sepidx], line[sepidx+1:] 263 else: 264 key, value = line,'' 265 266 self.processPair(key, value)
267
268 - def processPair(self, key, value):
269 """ Process a (key, value) pair """ 270 271 oldkey = key 272 oldvalue = value 273 274 # Create key intelligently 275 keyparts = self.bspacere.split(key) 276 # print keyparts 277 278 strippable = False 279 lastpart = keyparts[-1] 280 281 if lastpart.find('\\ ') != -1: 282 keyparts[-1] = lastpart.replace('\\','') 283 284 # If no backspace is found at the end, but empty 285 # space is found, strip it 286 elif lastpart and lastpart[-1] == ' ': 287 strippable = True 288 289 key = ''.join(keyparts) 290 if strippable: 291 key = key.strip() 292 oldkey = oldkey.strip() 293 294 oldvalue = self.unescape(oldvalue) 295 value = self.unescape(value) 296 297 self.data[key] = value.strip()
298 299 # # Check if an entry exists in pristine keys 300 # if self._keymap.has_key(key): 301 # oldkey = self._keymap.get(key) 302 # self._origprops[oldkey] = oldvalue.strip() 303 # else: 304 # self._origprops[oldkey] = oldvalue.strip() 305 # # Store entry in keymap 306 # self._keymap[key] = oldkey 307
308 - def escape(self, value):
309 310 # Java escapes the '=' and ':' in the value 311 # string with backslashes in the store method. 312 # So let us do the same. 313 newvalue = value.replace(':','\:') 314 newvalue = newvalue.replace('=','\=') 315 316 return newvalue
317
318 - def unescape(self, value):
319 320 # Reverse of escape 321 newvalue = value.replace('\:',':') 322 newvalue = newvalue.replace('\=','=') 323 324 return newvalue
325
326 - def load(self, stream):
327 """ Load properties from an open file stream """ 328 329 # For the time being only accept file input streams 330 if not(hasattr(stream, 'readlines') and callable(stream.readlines)): 331 raise TypeError,'Argument should be a file object!' 332 # Check for the opened mode 333 if hasattr(stream, 'mode') and stream.mode != 'r': 334 raise ValueError,'Stream should be opened in read-only mode!' 335 336 try: 337 lines = stream.readlines() 338 self.__parse(lines) 339 except IOError: 340 raise
341
342 - def store(self, out):
343 """ Serialize the properties back to a file. """ 344 345 if out.mode[0] != 'w': 346 raise ValueError, 'Stream should be opened in write mode!' 347 348 try: 349 # Write timestamp 350 out.write(''.join(('# ', time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime()), '\n'))) 351 352 # Write properties from the dictionary 353 for key in self.data.keys(): 354 value = self.data[key] 355 out.write(''.join((key, '=', self.escape(value), '\n'))) 356 357 out.close() 358 except IOError, e: 359 raise
360 361
362 -class NestedConfiguration(Configuration):
363 - def __init__(self):
364 """ Initialization. """ 365 Configuration.__init__(self, None) 366 self.parent = None 367 self.type = None 368 self.abstract = None
369
370 - def isBuildable(self):
371 return self.abstract == None
372
373 - def _addPropertyValue(self, key, value, parseList=True):
374 """Adds a property value to the configuration. 375 376 If the property does not exist, it is added without modification. 377 If there is already a single value matching the key, the value is replaced by a list 378 containing the original and new values. 379 If there is already a list, the new value is added to the list. 380 381 The value is first processed in case it also represents a list of values, 382 e.g. comma-separated values. 383 """ 384 if parseList and value.find(',') != -1: 385 value = value.split(',') 386 # Remove all whitespace 387 value = [v.strip() for v in value] 388 389 if key in self.data: 390 currentValue = self.data[key] 391 392 # Make sure current value is a list 393 if not isinstance(currentValue, types.ListType): 394 currentValue = [currentValue] 395 396 # Add new value(s) 397 if isinstance(value, types.ListType): 398 currentValue.extend(value) 399 else: 400 currentValue.append(value) 401 self.data[key] = currentValue 402 else: 403 self.data[key] = value
404
405 - def __getitem__(self, key, interpolate=True):
406 #print "__getitem__(%s, %s)" % (self.name, key) 407 if self.data.has_key(key): 408 value = super(NestedConfiguration, self).__getitem__(key, False) 409 if interpolate: 410 return self.interpolate(value) 411 return value 412 elif self.parent != None: 413 value = self.parent.__getitem__(key, False) 414 if interpolate: 415 return self.interpolate(value) 416 return value 417 raise KeyError('Cannot find key: ' + key)
418
419 - def __setitem__(self, key, item):
420 self.data[key] = item
421
422 - def __delitem__(self, key):
423 del self.data[key]
424
425 - def keys(self):
426 myKeys = self.data.keys() 427 if self.parent != None: 428 parentKeys = self.parent.keys() 429 for key in parentKeys: 430 if not key in myKeys: 431 myKeys.append(key) 432 return myKeys
433
434 - def match_name(self, name):
435 if self.name == name: 436 return True 437 if self.parent != None: 438 return self.parent.match_name(name) 439 return False
440 441
442 -class Specification(NestedConfiguration):
443 """ Deprecated. This should be removed in future, it adds no value. """ 444
445 - def __init__(self):
446 """ Initialization. """ 447 NestedConfiguration.__init__(self)
448 449
450 -class ConfigurationSet(Configuration):
451 """A ConfigurationSet represents a set of configurations. 452 453 Each configuration should be processed separately. This is matching 454 the Raptor model where a single XML file can contain definitions 455 of multiple specifications and configurations. 456 457 It is however somewhat different from the Commons Configuration classes 458 that combine configurations, e.g. CombinedConfiguration, 459 CompositeConfiguration. These act to combine configurations in a way 460 such that a single configuration interface is still presented to the 461 client. 462 """ 463
464 - def __init__(self, configs):
465 """ Initialization. """ 466 Configuration.__init__(self) 467 self._configs = configs
468
469 - def getConfigurations(self, name=None, type=None):
470 """ Return a list of configs that matches the name and type specified. 471 472 This can be queried multiple times to retrieve different named configurations. 473 """ 474 result = [] 475 for c in self._configs: 476 if ((name != None and c.match_name(name)) or name == None) and ((type != None and c.type == type) or type == None): 477 result.append(c) 478 return result
479 480
481 -class ConfigurationBuilder(object):
482 """ Base class for builders that can create Configuration objects. """ 483
484 - def getConfiguration(self):
485 """Returns a Configuration object.""" 486 raise NotImplementedError
487 488
489 -class NestedConfigurationBuilder(ConfigurationBuilder):
490 """ Builder for building Configuration objects from nested configurations. """ 491 492 _constructors = {'spec':Specification, 'config':NestedConfiguration} 493
494 - def __init__(self, inputfile, configname=''):
495 """ Initialization. """ 496 self.inputfile = inputfile 497 self.configname = configname 498 self._warn_on_deprecated_spec = False
499
500 - def getConfiguration(self):
501 """ Returns a ConfigurationSet object. 502 503 A ConfigurationSet represents a number of Configuration objects 504 that all may need to be processed. 505 """ 506 try: 507 dom = xml.dom.minidom.parse(self.inputfile) 508 except Exception, exc: 509 raise Exception("XML file '%s' cannot be parsed properly: %s" % (self.inputfile, exc)) 510 511 # The root element is typically <build> but can be anything 512 self.rootNode = dom.documentElement 513 configs = [] 514 515 # Create a flat list of buildable configurations 516 for child in self.rootNode.childNodes: 517 if child.nodeType == xml.dom.Node.ELEMENT_NODE: 518 _logger.debug('Parsing children') 519 self.parseConfiguration(child, configs) 520 521 # Add configuration references 522 references = [] 523 for reference in self.getReferences(): 524 for conf in configs: 525 if conf.match_name(reference[1]): 526 newConf = copy.deepcopy(conf) 527 newConf.name = reference[0] 528 references.append(newConf) 529 530 configs = configs + references 531 532 dom.unlink() 533 _logger.debug('Set of configs: ' + str(configs)) 534 535 if self._warn_on_deprecated_spec: 536 _logger.warning("Use of deprecated 'spec' element name in this configuration. Please rename to config") 537 return ConfigurationSet(configs)
538
539 - def getConfigurations(self, name=None, type=None):
540 """ Get a list of the individual configurations. 541 542 Once read a new builder must be opened to retrieve a differently filtered set of configurations. 543 """ 544 config_set = self.getConfiguration() 545 return config_set.getConfigurations(name, type)
546
547 - def getReferences(self):
548 references = [] 549 for rootNode in self.rootNode.childNodes: 550 if rootNode.nodeType == xml.dom.Node.ELEMENT_NODE: 551 for child in rootNode.childNodes: 552 if child.nodeType == xml.dom.Node.ELEMENT_NODE: 553 for conf in child.childNodes: 554 if conf.nodeName == 'specRef': 555 for ref in conf.getAttribute('ref').split(','): 556 if not ( child.getAttribute('abstract') and str(self.configname) == '' ): 557 references.append([child.getAttribute('name'), ref]) 558 return references
559
560 - def parseConfiguration(self, configNode, configs, parentConfig=None):
561 """ Parse an individual nested configuration. """ 562 # Create appropriate config object 563 if configNode.nodeName == 'spec': 564 self._warn_on_deprecated_spec = True 565 constructor = self._constructors[configNode.nodeName] 566 config = constructor() 567 _logger.debug('Configuration created: ' + str(config)) 568 if parentConfig != None: 569 config.parent = parentConfig 570 #config.data.update(parentConfig.data) 571 572 # Add any attribute properties 573 for i in range(configNode.attributes.length): 574 attribute = configNode.attributes.item(i) 575 if hasattr(config, attribute.name): 576 _logger.debug('Adding config attribute: ' + str(attribute.name)) 577 setattr(config, str(attribute.name), attribute.nodeValue) 578 else: 579 raise Exception('Invalid attribute for configuration: ' + attribute.name) 580 581 # Process the config element's children 582 configChildNodes = [] 583 584 for child in configNode.childNodes: 585 if child.nodeType == xml.dom.Node.ELEMENT_NODE: 586 # <append> directives should add to parent values. In 587 # this case initially set the value to the parent value. 588 if child.nodeName == 'append': 589 name = child.getAttribute('name') 590 if parentConfig != None and parentConfig.has_key(name): 591 parent_value = parentConfig.__getitem__(name, False) 592 if not isinstance(parent_value, types.ListType): 593 parent_value = [parent_value] 594 for value in parent_value: 595 config._addPropertyValue(name, value) 596 597 if child.nodeName == 'set' or child.nodeName == 'append': 598 name = child.getAttribute('name') 599 if child.hasAttribute('value'): 600 value = child.getAttribute('value') 601 config._addPropertyValue(name, value) 602 elif child.hasChildNodes(): 603 value = "" 604 for textchild in child.childNodes: 605 value += textchild.data 606 config._addPropertyValue(name, value, False) 607 elif child.nodeName == 'specRef': 608 for ref in child.getAttribute('ref').split(','): 609 node = self.getNodeByReference(ref) 610 if not node: 611 raise Exception('Referenced spec not found: ' + ref) 612 elif self._constructors.has_key(child.nodeName): 613 configChildNodes.append(child) 614 else: 615 raise Exception('Bad configuration xml element: ' + child.nodeName) 616 617 for childConfigNode in configChildNodes: 618 self.parseConfiguration(childConfigNode, configs, config) 619 620 # Only save the buildable configurations 621 if config.isBuildable(): 622 _logger.debug('Adding config to buildable set: ' + str(config)) 623 configs.append(config)
624
625 - def getNodeByReference(self, refName):
626 """ Find a node based on a reference to it. """ 627 for child in self.rootNode.childNodes: 628 if child.nodeType == xml.dom.Node.ELEMENT_NODE: 629 for conf in child.childNodes: 630 if conf.nodeName == 'spec': 631 if refName == conf.getAttribute('name'): 632 return conf
633 634
635 -class HierarchicalConfiguration(Configuration):
636 """ Represents hierarchical configurations such as XML documents. """ 637
638 - def __init__(self):
639 """ Initialization. """ 640 Configuration.__init__(self, None) 641 self._root = None
642
643 - def __getitem__(self, key, interpolate=True):
644 """ Get an item as a dict. """ 645 elements = self._root.xpath(_Key(key).to_xpath()) 646 values = [element.text for element in elements] 647 value = ','.join(values) 648 if interpolate: 649 value = self.interpolate(value) 650 return value
651 652
653 -class _Key(object):
654 """ A hierarchical configuration key. """ 655
656 - def __init__(self, string):
657 """ Initialization. """ 658 self.string = string
659
660 - def to_xpath(self):
661 return self.string.replace('.', '/')
662 663
664 -class XMLConfiguration(HierarchicalConfiguration):
665 """ A XML-based hierarchical configuration. """ 666
667 - def __init__(self, file_):
668 """ Initialization. """ 669 from lxml import etree 670 HierarchicalConfiguration.__init__(self) 671 672 self._root = etree.parse(file_)
673