Package build :: Module model
[hide private]
[frames] | no frames]

Source Code for Module build.model

  1  #============================================================================  
  2  #Name        : model.py  
  3  #Part of     : Helium  
  4   
  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
  6  #All rights reserved. 
  7  #This component and the accompanying materials are made available 
  8  #under the terms of the License "Eclipse Public License v1.0" 
  9  #which accompanies this distribution, and is available 
 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
 11  # 
 12  #Initial Contributors: 
 13  #Nokia Corporation - initial contribution. 
 14  # 
 15  #Contributors: 
 16  # 
 17  #Description: 
 18  #=============================================================================== 
 19   
 20  """ Models the concepts and objects that exist in a software build. """ 
 21   
 22  import itertools 
 23  import logging 
 24  import re 
 25  import os 
 26   
 27  from amara import bindery 
 28  import amara 
 29   
 30  import ccm 
 31  import ccm.extra 
 32  import configuration 
 33  import nokia.nokiaccm 
 34  from xmlhelper import node_scan, recursive_node_scan 
 35  import symrec 
 36   
 37  # Uncomment this line to enable logging in this module, or configure logging elsewhere 
 38  _logger = logging.getLogger("bom") 
 39  #_logger.setLevel(logging.DEBUG) 
 40  logging.basicConfig(level=logging.DEBUG) 
 41   
 42   
43 -class SessionCreator(object):
44 """ Session Creator object. """
45 - def __init__(self, username=None, password=None, provider=None):
46 """ Init the SessionCreator object.""" 47 self.__provider = provider 48 self.__username = username 49 self.__password = password
50
51 - def session(self, database):
52 """ Get a session for a database. If no session exists just create a new one.""" 53 _logger.info("Creating session for %s" % database) 54 return self.__provider.get(username=self.__username, password=self.__password, database=database)
55
56 - def close(self):
57 self.__provider = None
58 59
60 -class BOM(object):
61 """ The Bill of Materials for a build. """
62 - def __init__(self, config, ccm_project=None, username=None, password=None, provider=None):
63 """ Initialization. 64 65 :param config: The build configuration properties. 66 :param ccm_project: The Synergy project used for reading the BOM. 67 """ 68 self.config = config 69 self._sessioncreator = SessionCreator(username=username, password=password, provider=provider) 70 self.ccm_project = ccm_project 71 self.build = "" 72 self._projects = [] 73 if self.ccm_project != None: 74 self._projects = [Project(ccm_project, config)] 75 self._icd_icfs = [] 76 self._flags = [] 77 78 self._capture_projects() 79 self._capture_icd_icfs() 80 self._capture_flags()
81
82 - def _capture_projects(self):
83 # grab data from new format of delivery.xml 84 configBuilder = configuration.NestedConfigurationBuilder(open(self.config['delivery'], 'r')) 85 for config in configBuilder.getConfiguration().getConfigurations(): 86 _logger.debug('Importing project %s from delivery config.' % str(config.name)) 87 ccm_project = self._sessioncreator.session(config['database']).create(config.name) 88 project = Project(self.__find_project(ccm_project, config), config) 89 self._projects.append(project)
90
91 - def __find_project(self, project, config):
92 if (os.path.exists(os.path.join(config['dir'], project.name, "project.version"))): 93 return project 94 95 path = os.path.join(config['dir'], project.name, project.name) 96 if (not os.path.exists(path)): 97 return project 98 try: 99 result = project.session.get_workarea_info(path) 100 return result['project'] 101 except ccm.CCMException: 102 return project
103
104 - def _capture_icd_icfs(self):
105 prep_xml_path = self.config['prep.xml'] 106 if prep_xml_path is not None and os.path.exists(prep_xml_path): 107 prep_doc = amara.parse(open(prep_xml_path,'r')) 108 if hasattr(prep_doc.prepSpec, u'source'): 109 for source in prep_doc.prepSpec.source: 110 if hasattr(source, u'unzipicds'): 111 for unzipicds in source.unzipicds: 112 if hasattr(unzipicds, u'location'): 113 for location in unzipicds.location: 114 excludes = [] 115 excluded = False 116 if hasattr(location, 'exclude'): 117 for exclude in location.exclude: 118 _logger.debug('Exclude added: %s' % str(exclude.name)) 119 excludes.append(str(exclude.name)) 120 excluded = False 121 path = str(location.name) 122 if os.path.exists(path): 123 files = os.listdir(str(location.name)) 124 for file_ in files: 125 for exclude in excludes: 126 if file_.endswith(exclude): 127 excluded = True 128 if file_.endswith('.zip') and not excluded: 129 self._icd_icfs.append(file_) 130 self._icd_icfs.sort(key=str)
131
132 - def _capture_flags(self):
133 pass
134
135 - def _getprojects(self):
136 return self._projects
137 138 projects = property(_getprojects) 139
140 - def all_baselines(self):
141 baselines = {} 142 for project in self._projects: 143 for baseline, baseline_attrs in project.baselines.iteritems(): 144 baselines[baseline] = baseline_attrs 145 return baselines
146
147 - def all_tasks(self):
148 tasks = [] 149 for project in self._projects: 150 tasks.extend(project.all_tasks()) 151 tasks.sort(key=str) 152 return tasks
153
154 - def __str__(self):
155 return str(self._projects)
156
157 - def close(self):
158 self._sessioncreator.close()
159 160
161 -class Project(object):
162 """ An SCM project. 163 164 An input to the build area, typically copied from an SCM work area. 165 """
166 - def __init__(self, ccm_project, config, action=None):
167 """ Initialisation. """ 168 self._ccm_project = ccm_project 169 self._baselines = {} 170 #TODO : could querying release attribute return the ccm object? Or add a release attribute to Project 171 # class 172 release = self._ccm_project['release'] 173 _logger.debug("Project release: '%s'" % release) 174 self._ccm_release = None 175 if release != '': 176 self._ccm_project.session.create(release) 177 178 # capturing the frozen baseline. 179 _logger.debug('Capture baselines') 180 project_status = self._ccm_project['status'] 181 bproject = self._get_toplevel_baselines(self._ccm_project).pop() 182 if bproject != None: 183 self._baselines[unicode(bproject)] = {u'overridden':u'true'} 184 # This section finds the baselines of all of the checked out projects 185 if project_status == "prep" or project_status == "working": 186 for subproject in self._ccm_project.subprojects: 187 overridden = u'false' 188 subprojbaseline = subproject.baseline 189 if config.has_key('subbaselines'): 190 for subbaseline in config['subbaselines']: 191 if str(subbaseline) == str(subprojbaseline): 192 overridden = u'true' 193 194 if subprojbaseline != None: 195 self._baselines[unicode(subprojbaseline)] = {u'overridden': overridden} 196 # When a project is a snapshot, the baselines are the projects themselves 197 else: 198 for subproject in bproject.subprojects: 199 self._baselines[unicode(subproject)] = {u'overridden':u'false'} 200 201 self._tasks = [] 202 self._folders = [] 203 204 # Get Synergy reconfigure properties for folders and tasks 205 if action == None: 206 self._import_baseline_config() 207 # Get tasks from Synergy if using reconfigure template 208 if config.get_boolean("use.reconfigure.template", False): 209 self._tasks = self._ccm_project.tasks 210 self._folders = self._ccm_project.folders 211 212 # Or get folders and tasks defined in configuration file 213 elif action != None and action.nodeName == "checkout": 214 if not config.get_boolean("use.reconfigure.template", False): 215 for task_node in action.xml_xpath(u'./task[@id]'): 216 for task in [x.strip() for x in task_node.id.split(',')]: 217 self._tasks.append(ccm_project.session.create("Task %s" % task)) 218 for folder_node in action.xml_xpath(u'./folder[@id]'): 219 for folder in [x.strip() for x in folder_node.id.split(',')]: 220 self._folders.append(ccm_project.session.create("Folder %s" % folder)) 221 else: 222 self._tasks = self._ccm_project.tasks 223 self._folders = self._ccm_project.folders 224 self._import_baseline_config()
225
226 - def _import_baseline_config(self):
227 """ Import the baseline folders and tasks. """ 228 baselines = self._get_toplevel_baselines(self._ccm_project) 229 baselines.pop() 230 for baseline in baselines: 231 for task in baseline.tasks: 232 if task not in self._tasks: 233 self._tasks.append(task) 234 for folder in baseline.folders: 235 if folder not in self._folders: 236 self._folders.append(folder)
237
238 - def _get_toplevel_baselines(self, project):
239 if project == None: 240 return [] 241 project_status = project['status'] 242 if project_status == "prep" or project_status == "working": 243 result = [project] 244 baseline = project.baseline 245 if baseline != None: 246 result.extend(self._get_toplevel_baselines(baseline)) 247 return result 248 else: 249 return [project]
250
251 - def _getbaselines(self):
252 return self._baselines
253 254 baselines = property(_getbaselines) 255
256 - def _getfolders(self):
257 return self._folders
258 259 folders = property(_getfolders) 260
261 - def all_tasks(self):
262 """ Get all the tasks (individual and folder based). """ 263 tasks = [Task(ccm_task) for ccm_task in self._tasks] 264 for folder in self._folders: 265 [tasks.append(Task(ccm_task)) for ccm_task in folder.tasks] 266 tasks.sort(key=str) 267 return tasks
268
269 - def _gettasks(self):
270 return [Task(ccm_task) for ccm_task in self._tasks]
271 272 tasks = property(_gettasks) 273
274 - def _getsupplier(self):
275 if self._ccm_release != None: 276 component = self._ccm_release.component 277 comparisons = {'MC': '^mc', 278 'S60': 'S60', 279 'SPP/NCP': '^spp_config|spp_psw|spp_tools|ncp_sw$', 280 'IBUSAL': '^IBUSAL'} 281 for supplier, regexp in comparisons.iteritems(): 282 if re.search(regexp, component) != None: 283 return supplier 284 return "Unknown"
285 286 supplier = property(_getsupplier) 287
288 - def __repr__(self):
289 """ Object representation. """ 290 return str(self._ccm_project)
291
292 - def __str__(self):
293 """ String representation. """ 294 return str(self._ccm_project)
295 296
297 -class Fix(object):
298 """ A generic fix. """
299 - def __init__(self, description):
300 """ Initialisation. """ 301 self._description = description
302
303 - def __str__(self):
304 """ String representation. """ 305 return str(self._description)
306 307
308 -class TSWError(Fix):
309 """ A TSW database error. """ 310 regex = '([A-Z]{4}-[A-Z0-9]{6})' 311 groupname = 'TSW Errors' 312
313 - def __init__(self, description):
314 """ Initialisation. """ 315 Fix.__init__(self, description)
316 317
318 -class PCPError(Fix):
319 """ A PCP database error. """ 320 regex = '([A-Z]{2}-[0-9]{11})' 321 groupname = 'PCP Errors' 322
323 - def __init__(self, description):
324 """ Initialisation. """ 325 Fix.__init__(self, description)
326 327
328 -class TAChange(Fix):
329 """ A Type Approval change. """ 330 regex = '^_TA:(\s*)(.*?)(\s*)$' 331 groupname = 'TA Changes' 332
333 - def __init__(self, description):
334 """ Initialisation. """ 335 Fix.__init__(self, description)
336 337
338 -class Task(object):
339 """ A task or unit of change from the SCM system. """ 340 fix_types = [TSWError, PCPError, TAChange] 341
342 - def __init__(self, ccm_task):
343 """ Initialisation. """ 344 self.ccm_task = ccm_task
345
346 - def __getitem__(self, name):
347 """ Dictionary of tasks support. """ 348 return self.ccm_task[name]
349
350 - def has_fixed(self):
351 """ Returns an object representing what this task fixed, if anything. """ 352 text = str(self.ccm_task) 353 fix_object = None 354 for fix_type in self.fix_types: 355 match = re.search(fix_type.regex, str(self.ccm_task)) 356 if match != None: 357 fix_object = fix_type(text) 358 break 359 return fix_object
360
361 - def __cmp__(self, other):
362 """ Compare tasks based on their task number only. """ 363 self_task = str(self.ccm_task) 364 other_task = str(other.ccm_task) 365 return cmp(self_task[:self_task.find(':')], other_task[:other_task.find(':')])
366
367 - def __hash__(self):
368 """ Hash support. """ 369 self_task = str(self.ccm_task) 370 return hash(self_task[:self_task.find(':')])
371
372 - def __repr__(self):
373 """ Object representation. """ 374 self_task = repr(self.ccm_task) 375 return self_task[:self_task.find(':')]
376
377 - def __str__(self):
378 """ String representation. """ 379 return str(self.ccm_task)
380 381
382 -class ICD_ICF(object):
383 """ A ICD or ICF patch zip file provided by Symbian. """ 384 pass
385 386
387 -class Flag(object):
388 """ A compilation flag. """ 389 pass
390 391
392 -class BOMDeltaXMLWriter(object):
393 - def __init__(self, bom, bom_log):
394 """ Initialisation. """ 395 self._bom = bom 396 self._bom_log = bom_log
397
398 - def write(self, path):
399 """ Write the BOM delta information to an XML file. """ 400 bom_log = amara.parse(open(self._bom_log, 'r')) 401 doc = amara.create_document(u'bomDelta') 402 # pylint: disable-msg=E1101 403 doc.bomDelta.xml_append(doc.xml_create_element(u'buildFrom', content=unicode(bom_log.bom.build))) 404 doc.bomDelta.xml_append(doc.xml_create_element(u'buildTo', content=unicode(self._bom.config['build.id']))) 405 content_node = doc.xml_create_element(u'content') 406 doc.bomDelta.xml_append(content_node) 407 408 old_baselines = {} 409 baselines = {} 410 old_folders = {} 411 folders = {} 412 old_tasks = {} 413 tasks = {} 414 if hasattr(bom_log.bom.content, 'project'): 415 for project in bom_log.bom.content.project: 416 if hasattr(project, 'baseline'): 417 for baseline in project.baseline: 418 if not old_baselines.has_key(unicode(baseline)): 419 old_baselines[unicode(baseline)] = {} 420 if hasattr(baseline, 'xml_attributes'): 421 _logger.debug('baseline.xml_attributes: %s' % baseline.xml_attributes) 422 for attr_name, junk_tuple in sorted(baseline.xml_attributes.iteritems()): 423 _logger.debug('attr_name: %s' % attr_name) 424 old_baselines[unicode(baseline)][unicode(attr_name)] = unicode(getattr(baseline, attr_name)) 425 if hasattr(project, 'folder'): 426 for folder in project.folder: 427 if hasattr(folder, 'name'): 428 for name in folder.name: 429 folder_name = unicode(name) 430 _logger.debug('folder_name: %s' % folder_name) 431 if not old_folders.has_key(unicode(folder_name)): 432 old_folders[unicode(folder_name)] = {} 433 if hasattr(name, 'xml_attributes'): 434 for attr_name, junk_tuple in sorted(name.xml_attributes.iteritems()): 435 _logger.debug('attr_name: %s' % attr_name) 436 old_folders[unicode(folder_name)][unicode(attr_name)] = unicode(getattr(name, attr_name)) 437 for task in recursive_node_scan(bom_log.bom.content, u'task'): 438 _logger.debug('task: %s' % task) 439 _logger.debug('task: %s' % task.id) 440 _logger.debug('task: %s' % task.synopsis) 441 task_id = u"%s: %s" % (task.id, task.synopsis) 442 if not old_tasks.has_key(task_id): 443 old_tasks[task_id] = {} 444 if hasattr(task, 'xml_attributes'): 445 for attr_name, junk_tuple in sorted(task.xml_attributes.iteritems()): 446 _logger.debug('attr_name: %s' % attr_name) 447 old_tasks[task_id][unicode(attr_name)] = unicode(getattr(task, attr_name)) 448 for project in self._bom.projects: 449 for folder in project.folders: 450 folders[unicode(folder.instance + "#" + folder.name + ": " + folder.description)] = {u'overridden':u'true'} 451 for task in folder.tasks: 452 _logger.debug("task_bom:'%s'" % unicode(task)) 453 tasks[unicode(task)] = {u'overridden':u'false'} 454 for task in project.tasks: 455 _logger.debug("task_bom:'%s'" % unicode(task)) 456 tasks[unicode(task)] = {u'overridden':u'true'} 457 458 baselines = self._bom.all_baselines() 459 460 self._write_items_with_attributes(content_node, u'baseline', baselines, old_baselines) 461 self._write_items_with_attributes(content_node, u'folder', folders, old_folders) 462 self._write_items_with_attributes(content_node, u'task', tasks, old_tasks) 463 464 out = open(path, 'w') 465 doc.xml(out, indent='yes') 466 out.close()
467
468 - def _write_items(self, node, item_name, items, older_items):
469 items = frozenset(items) 470 older_items = frozenset(older_items) 471 472 items_added = list(items.difference(older_items)) 473 items_added.sort() 474 for item in items_added: 475 node.xml_append(node.xml_create_element(item_name, \ 476 attributes={u'status': u'added'}, content=unicode(item))) 477 478 items_deleted = list(older_items.difference(items)) 479 items_deleted.sort() 480 for item in items_deleted: 481 node.xml_append(node.xml_create_element(item_name, \ 482 attributes={u'status': u'deleted'}, content=unicode(item)))
483 484 # This method takes dictionaries as input to pass along attributes
485 - def _write_items_with_attributes(self, node, item_name, items, older_items):
486 fr_items = frozenset(items) 487 fr_older_items = frozenset(older_items) 488 489 items_added = list(fr_items.difference(fr_older_items)) 490 items_added.sort() 491 for item in items_added: 492 item_attributes = {u'status': u'added'} 493 for attr_name, attr_value in sorted(items[item].iteritems()): 494 _logger.debug('item: %s' % item) 495 _logger.debug('attr_name: %s' % attr_name) 496 _logger.debug('attr_value: %s' % attr_value) 497 item_attributes[attr_name] = attr_value 498 node.xml_append(node.xml_create_element(item_name, \ 499 attributes=item_attributes, content=unicode(item))) 500 501 items_deleted = list(fr_older_items.difference(fr_items)) 502 items_deleted.sort() 503 for item in items_deleted: 504 item_attributes = {u'status': u'deleted'} 505 for attr_name, attr_value in sorted(older_items[item].iteritems()): 506 _logger.debug('item: %s' % item) 507 _logger.debug('attr_name: %s' % attr_name) 508 _logger.debug('attr_value: %s' % attr_value) 509 item_attributes[attr_name] = attr_value 510 node.xml_append(node.xml_create_element(item_name, \ 511 attributes=item_attributes, content=unicode(item)))
512 513
514 -class BOMXMLWriter(object):
515 - def __init__(self, bom):
516 """ Initialisation. """ 517 self._bom = bom
518
519 - def write(self, path):
520 """ Write the BOM information to an XML file. """ 521 doc = amara.create_document(u'bom') 522 # pylint: disable-msg=E1101 523 doc.bom.xml_append(doc.xml_create_element(u'build', content=unicode(self._bom.config['build.id']))) 524 doc.bom.xml_append(doc.xml_create_element(u'content')) 525 for project in self._bom.projects: 526 project_node = doc.xml_create_element(u'project') 527 project_node.xml_append(doc.xml_create_element(u'name', content=unicode(project))) 528 project_node.xml_append(doc.xml_create_element(u'database', content=unicode(self._bom.config['ccm.database']))) 529 doc.bom.content.xml_append(project_node) 530 _logger.debug('baselines dictionary: %s' % project.baselines) 531 for baseline, baseline_attrs in sorted(project.baselines.iteritems()): 532 _logger.debug('baseline: %s' % baseline) 533 _logger.debug('baseline_attrs: %s' % baseline_attrs) 534 project_node.xml_append(doc.xml_create_element(u'baseline', content=unicode(baseline), attributes=baseline_attrs)) 535 for folder in project.folders: 536 folder_node = doc.xml_create_element(u'folder') 537 folder_node.xml_append(doc.xml_create_element(u'name', content=unicode(folder.instance + "#" + folder.name + ": " + folder.description), \ 538 attributes={u'overridden':u'true'})) 539 project_node.xml_append(folder_node) 540 for task in folder.tasks: 541 task_node = doc.xml_create_element(u'task', attributes={u'overridden':u'false'}) 542 task_node.xml_append(doc.xml_create_element(u'id', content=(unicode(task['displayname'])))) 543 task_node.xml_append(doc.xml_create_element(u'synopsis', content=(unicode(task['task_synopsis'])))) 544 task_node.xml_append(doc.xml_create_element(u'owner', content=(unicode(task['owner'])))) 545 #task_node.xml_append(doc.xml_create_element(u'completed', content=(unicode(self.parse_status_log(task['status_log']))))) 546 folder_node.xml_append(task_node) 547 for task in project.tasks: 548 task_node = doc.xml_create_element(u'task', attributes={u'overridden':u'true'}) 549 task_node.xml_append(doc.xml_create_element(u'id', content=(unicode(task['displayname'])))) 550 task_node.xml_append(doc.xml_create_element(u'synopsis', content=(unicode(task['task_synopsis'])))) 551 task_node.xml_append(doc.xml_create_element(u'owner', content=(unicode(task['owner'])))) 552 #task_node.xml_append(doc.xml_create_element(u'completed', content=(unicode(self.parse_status_log(task['status_log']))))) 553 project_node.xml_append(task_node) 554 555 fix = task.has_fixed() 556 if fix != None: 557 fix_node = doc.xml_create_element(u'fix', content=(unicode(task)), attributes = {u'type': unicode(fix.__class__.__name__)}) 558 project_node.xml_append(fix_node) 559 560 # Add ICD info to BOM 561 doc.bom.content.xml_append(doc.xml_create_element(u'input')) 562 563 # Add default values to unused fields so icds are visible in the BOM 564 empty_bom_str = u'N/A' 565 empty_bom_tm = u'0' 566 doc.bom.content.input.xml_append(doc.xml_create_element(u'name', content=(unicode(empty_bom_str)))) 567 doc.bom.content.input.xml_append(doc.xml_create_element(u'year', content=(unicode(empty_bom_tm)))) 568 doc.bom.content.input.xml_append(doc.xml_create_element(u'week', content=(unicode(empty_bom_tm)))) 569 doc.bom.content.input.xml_append(doc.xml_create_element(u'version', content=(unicode(empty_bom_str)))) 570 571 doc.bom.content.input.xml_append(doc.xml_create_element(u'icds')) 572 573 # pylint: disable-msg=R0914 574 for i, icd in enumerate(self._bom._icd_icfs): 575 doc.bom.content.input.icds.xml_append(doc.xml_create_element(u'icd')) 576 doc.bom.content.input.icds.icd[i].xml_append(doc.xml_create_element(u'name', content=(unicode(icd)))) 577 #If currentRelease.xml exists then send s60 <input> tag to diamonds 578 current_release_xml_path = self._bom.config['currentRelease.xml'] 579 if current_release_xml_path is not None and os.path.exists(current_release_xml_path): 580 metadata = symrec.ReleaseMetadata(current_release_xml_path) 581 service = metadata.service 582 product = metadata.product 583 release = metadata.release 584 # Get name, year, week and version from baseline configuration 585 s60_input_node = doc.xml_create_element(u'input') 586 s60_version = self._bom.config['s60_version'] 587 s60_release = self._bom.config['s60_release'] 588 if s60_version != None: 589 s60_year = s60_version[0:4] 590 s60_week = s60_version[4:] 591 else: 592 s60_year = u'0' 593 s60_week = u'0' 594 s60_input_node.xml_append(doc.xml_create_element(u'name', content=(unicode("s60")))) 595 s60_input_node.xml_append(doc.xml_create_element(u'year', content=(unicode(s60_year)))) 596 s60_input_node.xml_append(doc.xml_create_element(u'week', content=(unicode(s60_week)))) 597 s60_input_node.xml_append(doc.xml_create_element(u'version', content=(unicode(s60_release)))) 598 599 s60_input_source = s60_input_node.xml_create_element(u'source') 600 s60_input_source.xml_append(doc.xml_create_element(u'type', content=(unicode("grace")))) 601 s60_input_source.xml_append(doc.xml_create_element(u'service', content=(unicode(service)))) 602 s60_input_source.xml_append(doc.xml_create_element(u'product', content=(unicode(product)))) 603 s60_input_source.xml_append(doc.xml_create_element(u'release', content=(unicode(release)))) 604 s60_input_node.xml_append(s60_input_source) 605 doc.bom.content.xml_append(s60_input_node) 606 out = open(path, 'w') 607 doc.xml(out, indent='yes') 608 out.close()
609
610 - def parse_status_log(self, log):
611 _log_array = log.split('\r') 612 if(len(_log_array) == 3 and log.find('completed') > 0): 613 _completed_line = _log_array[2] 614 return _completed_line[:_completed_line.rfind(':')].strip() 615 else: 616 return u'None'
617