1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 """ Models the concepts and objects that exist in a software build. """
21
22 import itertools
23 import logging
24 import re
25 import os
26
27 from amara import bindery
28 import amara
29
30 import ccm
31 import ccm.extra
32 import configuration
33 import nokia.nokiaccm
34 from xmlhelper import node_scan, recursive_node_scan
35 import symrec
36
37
38 _logger = logging.getLogger("bom")
39
40 logging.basicConfig(level=logging.DEBUG)
41
42
44 """ Session Creator object. """
45 - def __init__(self, username=None, password=None, provider=None):
46 """ Init the SessionCreator object."""
47 self.__provider = provider
48 self.__username = username
49 self.__password = password
50
52 """ Get a session for a database. If no session exists just create a new one."""
53 _logger.info("Creating session for %s" % database)
54 return self.__provider.get(username=self.__username, password=self.__password, database=database)
55
57 self.__provider = None
58
59
61 """ The Bill of Materials for a build. """
62 - def __init__(self, config, ccm_project=None, username=None, password=None, provider=None):
63 """ Initialization.
64
65 :param config: The build configuration properties.
66 :param ccm_project: The Synergy project used for reading the BOM.
67 """
68 self.config = config
69 self._sessioncreator = SessionCreator(username=username, password=password, provider=provider)
70 self.ccm_project = ccm_project
71 self.build = ""
72 self._projects = []
73 if self.ccm_project != None:
74 self._projects = [Project(ccm_project, config)]
75 self._icd_icfs = []
76 self._flags = []
77
78 self._capture_projects()
79 self._capture_icd_icfs()
80 self._capture_flags()
81
90
103
105 prep_xml_path = self.config['prep.xml']
106 if prep_xml_path is not None and os.path.exists(prep_xml_path):
107 prep_doc = amara.parse(open(prep_xml_path,'r'))
108 if hasattr(prep_doc.prepSpec, u'source'):
109 for source in prep_doc.prepSpec.source:
110 if hasattr(source, u'unzipicds'):
111 for unzipicds in source.unzipicds:
112 if hasattr(unzipicds, u'location'):
113 for location in unzipicds.location:
114 excludes = []
115 excluded = False
116 if hasattr(location, 'exclude'):
117 for exclude in location.exclude:
118 _logger.debug('Exclude added: %s' % str(exclude.name))
119 excludes.append(str(exclude.name))
120 excluded = False
121 path = str(location.name)
122 if os.path.exists(path):
123 files = os.listdir(str(location.name))
124 for file_ in files:
125 for exclude in excludes:
126 if file_.endswith(exclude):
127 excluded = True
128 if file_.endswith('.zip') and not excluded:
129 self._icd_icfs.append(file_)
130 self._icd_icfs.sort(key=str)
131
134
136 return self._projects
137
138 projects = property(_getprojects)
139
146
153
155 return str(self._projects)
156
158 self._sessioncreator.close()
159
160
162 """ An SCM project.
163
164 An input to the build area, typically copied from an SCM work area.
165 """
166 - def __init__(self, ccm_project, config, action=None):
167 """ Initialisation. """
168 self._ccm_project = ccm_project
169 self._baselines = {}
170
171
172 release = self._ccm_project['release']
173 _logger.debug("Project release: '%s'" % release)
174 self._ccm_release = None
175 if release != '':
176 self._ccm_project.session.create(release)
177
178
179 _logger.debug('Capture baselines')
180 project_status = self._ccm_project['status']
181 bproject = self._get_toplevel_baselines(self._ccm_project).pop()
182 if bproject != None:
183 self._baselines[unicode(bproject)] = {u'overridden':u'true'}
184
185 if project_status == "prep" or project_status == "working":
186 for subproject in self._ccm_project.subprojects:
187 overridden = u'false'
188 subprojbaseline = subproject.baseline
189 if config.has_key('subbaselines'):
190 for subbaseline in config['subbaselines']:
191 if str(subbaseline) == str(subprojbaseline):
192 overridden = u'true'
193
194 if subprojbaseline != None:
195 self._baselines[unicode(subprojbaseline)] = {u'overridden': overridden}
196
197 else:
198 for subproject in bproject.subprojects:
199 self._baselines[unicode(subproject)] = {u'overridden':u'false'}
200
201 self._tasks = []
202 self._folders = []
203
204
205 if action == None:
206 self._import_baseline_config()
207
208 if config.get_boolean("use.reconfigure.template", False):
209 self._tasks = self._ccm_project.tasks
210 self._folders = self._ccm_project.folders
211
212
213 elif action != None and action.nodeName == "checkout":
214 if not config.get_boolean("use.reconfigure.template", False):
215 for task_node in action.xml_xpath(u'./task[@id]'):
216 for task in [x.strip() for x in task_node.id.split(',')]:
217 self._tasks.append(ccm_project.session.create("Task %s" % task))
218 for folder_node in action.xml_xpath(u'./folder[@id]'):
219 for folder in [x.strip() for x in folder_node.id.split(',')]:
220 self._folders.append(ccm_project.session.create("Folder %s" % folder))
221 else:
222 self._tasks = self._ccm_project.tasks
223 self._folders = self._ccm_project.folders
224 self._import_baseline_config()
225
237
250
252 return self._baselines
253
254 baselines = property(_getbaselines)
255
258
259 folders = property(_getfolders)
260
262 """ Get all the tasks (individual and folder based). """
263 tasks = [Task(ccm_task) for ccm_task in self._tasks]
264 for folder in self._folders:
265 [tasks.append(Task(ccm_task)) for ccm_task in folder.tasks]
266 tasks.sort(key=str)
267 return tasks
268
270 return [Task(ccm_task) for ccm_task in self._tasks]
271
272 tasks = property(_gettasks)
273
275 if self._ccm_release != None:
276 component = self._ccm_release.component
277 comparisons = {'MC': '^mc',
278 'S60': 'S60',
279 'SPP/NCP': '^spp_config|spp_psw|spp_tools|ncp_sw$',
280 'IBUSAL': '^IBUSAL'}
281 for supplier, regexp in comparisons.iteritems():
282 if re.search(regexp, component) != None:
283 return supplier
284 return "Unknown"
285
286 supplier = property(_getsupplier)
287
289 """ Object representation. """
290 return str(self._ccm_project)
291
293 """ String representation. """
294 return str(self._ccm_project)
295
296
298 """ A generic fix. """
300 """ Initialisation. """
301 self._description = description
302
304 """ String representation. """
305 return str(self._description)
306
307
309 """ A TSW database error. """
310 regex = '([A-Z]{4}-[A-Z0-9]{6})'
311 groupname = 'TSW Errors'
312
316
317
319 """ A PCP database error. """
320 regex = '([A-Z]{2}-[0-9]{11})'
321 groupname = 'PCP Errors'
322
326
327
329 """ A Type Approval change. """
330 regex = '^_TA:(\s*)(.*?)(\s*)$'
331 groupname = 'TA Changes'
332
336
337
339 """ A task or unit of change from the SCM system. """
340 fix_types = [TSWError, PCPError, TAChange]
341
343 """ Initialisation. """
344 self.ccm_task = ccm_task
345
347 """ Dictionary of tasks support. """
348 return self.ccm_task[name]
349
351 """ Returns an object representing what this task fixed, if anything. """
352 text = str(self.ccm_task)
353 fix_object = None
354 for fix_type in self.fix_types:
355 match = re.search(fix_type.regex, str(self.ccm_task))
356 if match != None:
357 fix_object = fix_type(text)
358 break
359 return fix_object
360
362 """ Compare tasks based on their task number only. """
363 self_task = str(self.ccm_task)
364 other_task = str(other.ccm_task)
365 return cmp(self_task[:self_task.find(':')], other_task[:other_task.find(':')])
366
368 """ Hash support. """
369 self_task = str(self.ccm_task)
370 return hash(self_task[:self_task.find(':')])
371
373 """ Object representation. """
374 self_task = repr(self.ccm_task)
375 return self_task[:self_task.find(':')]
376
378 """ String representation. """
379 return str(self.ccm_task)
380
381
383 """ A ICD or ICF patch zip file provided by Symbian. """
384 pass
385
386
388 """ A compilation flag. """
389 pass
390
391
394 """ Initialisation. """
395 self._bom = bom
396 self._bom_log = bom_log
397
399 """ Write the BOM delta information to an XML file. """
400 bom_log = amara.parse(open(self._bom_log, 'r'))
401 doc = amara.create_document(u'bomDelta')
402
403 doc.bomDelta.xml_append(doc.xml_create_element(u'buildFrom', content=unicode(bom_log.bom.build)))
404 doc.bomDelta.xml_append(doc.xml_create_element(u'buildTo', content=unicode(self._bom.config['build.id'])))
405 content_node = doc.xml_create_element(u'content')
406 doc.bomDelta.xml_append(content_node)
407
408 old_baselines = {}
409 baselines = {}
410 old_folders = {}
411 folders = {}
412 old_tasks = {}
413 tasks = {}
414 if hasattr(bom_log.bom.content, 'project'):
415 for project in bom_log.bom.content.project:
416 if hasattr(project, 'baseline'):
417 for baseline in project.baseline:
418 if not old_baselines.has_key(unicode(baseline)):
419 old_baselines[unicode(baseline)] = {}
420 if hasattr(baseline, 'xml_attributes'):
421 _logger.debug('baseline.xml_attributes: %s' % baseline.xml_attributes)
422 for attr_name, junk_tuple in sorted(baseline.xml_attributes.iteritems()):
423 _logger.debug('attr_name: %s' % attr_name)
424 old_baselines[unicode(baseline)][unicode(attr_name)] = unicode(getattr(baseline, attr_name))
425 if hasattr(project, 'folder'):
426 for folder in project.folder:
427 if hasattr(folder, 'name'):
428 for name in folder.name:
429 folder_name = unicode(name)
430 _logger.debug('folder_name: %s' % folder_name)
431 if not old_folders.has_key(unicode(folder_name)):
432 old_folders[unicode(folder_name)] = {}
433 if hasattr(name, 'xml_attributes'):
434 for attr_name, junk_tuple in sorted(name.xml_attributes.iteritems()):
435 _logger.debug('attr_name: %s' % attr_name)
436 old_folders[unicode(folder_name)][unicode(attr_name)] = unicode(getattr(name, attr_name))
437 for task in recursive_node_scan(bom_log.bom.content, u'task'):
438 _logger.debug('task: %s' % task)
439 _logger.debug('task: %s' % task.id)
440 _logger.debug('task: %s' % task.synopsis)
441 task_id = u"%s: %s" % (task.id, task.synopsis)
442 if not old_tasks.has_key(task_id):
443 old_tasks[task_id] = {}
444 if hasattr(task, 'xml_attributes'):
445 for attr_name, junk_tuple in sorted(task.xml_attributes.iteritems()):
446 _logger.debug('attr_name: %s' % attr_name)
447 old_tasks[task_id][unicode(attr_name)] = unicode(getattr(task, attr_name))
448 for project in self._bom.projects:
449 for folder in project.folders:
450 folders[unicode(folder.instance + "#" + folder.name + ": " + folder.description)] = {u'overridden':u'true'}
451 for task in folder.tasks:
452 _logger.debug("task_bom:'%s'" % unicode(task))
453 tasks[unicode(task)] = {u'overridden':u'false'}
454 for task in project.tasks:
455 _logger.debug("task_bom:'%s'" % unicode(task))
456 tasks[unicode(task)] = {u'overridden':u'true'}
457
458 baselines = self._bom.all_baselines()
459
460 self._write_items_with_attributes(content_node, u'baseline', baselines, old_baselines)
461 self._write_items_with_attributes(content_node, u'folder', folders, old_folders)
462 self._write_items_with_attributes(content_node, u'task', tasks, old_tasks)
463
464 out = open(path, 'w')
465 doc.xml(out, indent='yes')
466 out.close()
467
468 - def _write_items(self, node, item_name, items, older_items):
469 items = frozenset(items)
470 older_items = frozenset(older_items)
471
472 items_added = list(items.difference(older_items))
473 items_added.sort()
474 for item in items_added:
475 node.xml_append(node.xml_create_element(item_name, \
476 attributes={u'status': u'added'}, content=unicode(item)))
477
478 items_deleted = list(older_items.difference(items))
479 items_deleted.sort()
480 for item in items_deleted:
481 node.xml_append(node.xml_create_element(item_name, \
482 attributes={u'status': u'deleted'}, content=unicode(item)))
483
484
486 fr_items = frozenset(items)
487 fr_older_items = frozenset(older_items)
488
489 items_added = list(fr_items.difference(fr_older_items))
490 items_added.sort()
491 for item in items_added:
492 item_attributes = {u'status': u'added'}
493 for attr_name, attr_value in sorted(items[item].iteritems()):
494 _logger.debug('item: %s' % item)
495 _logger.debug('attr_name: %s' % attr_name)
496 _logger.debug('attr_value: %s' % attr_value)
497 item_attributes[attr_name] = attr_value
498 node.xml_append(node.xml_create_element(item_name, \
499 attributes=item_attributes, content=unicode(item)))
500
501 items_deleted = list(fr_older_items.difference(fr_items))
502 items_deleted.sort()
503 for item in items_deleted:
504 item_attributes = {u'status': u'deleted'}
505 for attr_name, attr_value in sorted(older_items[item].iteritems()):
506 _logger.debug('item: %s' % item)
507 _logger.debug('attr_name: %s' % attr_name)
508 _logger.debug('attr_value: %s' % attr_value)
509 item_attributes[attr_name] = attr_value
510 node.xml_append(node.xml_create_element(item_name, \
511 attributes=item_attributes, content=unicode(item)))
512
513
516 """ Initialisation. """
517 self._bom = bom
518
520 """ Write the BOM information to an XML file. """
521 doc = amara.create_document(u'bom')
522
523 doc.bom.xml_append(doc.xml_create_element(u'build', content=unicode(self._bom.config['build.id'])))
524 doc.bom.xml_append(doc.xml_create_element(u'content'))
525 for project in self._bom.projects:
526 project_node = doc.xml_create_element(u'project')
527 project_node.xml_append(doc.xml_create_element(u'name', content=unicode(project)))
528 project_node.xml_append(doc.xml_create_element(u'database', content=unicode(self._bom.config['ccm.database'])))
529 doc.bom.content.xml_append(project_node)
530 _logger.debug('baselines dictionary: %s' % project.baselines)
531 for baseline, baseline_attrs in sorted(project.baselines.iteritems()):
532 _logger.debug('baseline: %s' % baseline)
533 _logger.debug('baseline_attrs: %s' % baseline_attrs)
534 project_node.xml_append(doc.xml_create_element(u'baseline', content=unicode(baseline), attributes=baseline_attrs))
535 for folder in project.folders:
536 folder_node = doc.xml_create_element(u'folder')
537 folder_node.xml_append(doc.xml_create_element(u'name', content=unicode(folder.instance + "#" + folder.name + ": " + folder.description), \
538 attributes={u'overridden':u'true'}))
539 project_node.xml_append(folder_node)
540 for task in folder.tasks:
541 task_node = doc.xml_create_element(u'task', attributes={u'overridden':u'false'})
542 task_node.xml_append(doc.xml_create_element(u'id', content=(unicode(task['displayname']))))
543 task_node.xml_append(doc.xml_create_element(u'synopsis', content=(unicode(task['task_synopsis']))))
544 task_node.xml_append(doc.xml_create_element(u'owner', content=(unicode(task['owner']))))
545
546 folder_node.xml_append(task_node)
547 for task in project.tasks:
548 task_node = doc.xml_create_element(u'task', attributes={u'overridden':u'true'})
549 task_node.xml_append(doc.xml_create_element(u'id', content=(unicode(task['displayname']))))
550 task_node.xml_append(doc.xml_create_element(u'synopsis', content=(unicode(task['task_synopsis']))))
551 task_node.xml_append(doc.xml_create_element(u'owner', content=(unicode(task['owner']))))
552
553 project_node.xml_append(task_node)
554
555 fix = task.has_fixed()
556 if fix != None:
557 fix_node = doc.xml_create_element(u'fix', content=(unicode(task)), attributes = {u'type': unicode(fix.__class__.__name__)})
558 project_node.xml_append(fix_node)
559
560
561 doc.bom.content.xml_append(doc.xml_create_element(u'input'))
562
563
564 empty_bom_str = u'N/A'
565 empty_bom_tm = u'0'
566 doc.bom.content.input.xml_append(doc.xml_create_element(u'name', content=(unicode(empty_bom_str))))
567 doc.bom.content.input.xml_append(doc.xml_create_element(u'year', content=(unicode(empty_bom_tm))))
568 doc.bom.content.input.xml_append(doc.xml_create_element(u'week', content=(unicode(empty_bom_tm))))
569 doc.bom.content.input.xml_append(doc.xml_create_element(u'version', content=(unicode(empty_bom_str))))
570
571 doc.bom.content.input.xml_append(doc.xml_create_element(u'icds'))
572
573
574 for i, icd in enumerate(self._bom._icd_icfs):
575 doc.bom.content.input.icds.xml_append(doc.xml_create_element(u'icd'))
576 doc.bom.content.input.icds.icd[i].xml_append(doc.xml_create_element(u'name', content=(unicode(icd))))
577
578 current_release_xml_path = self._bom.config['currentRelease.xml']
579 if current_release_xml_path is not None and os.path.exists(current_release_xml_path):
580 metadata = symrec.ReleaseMetadata(current_release_xml_path)
581 service = metadata.service
582 product = metadata.product
583 release = metadata.release
584
585 s60_input_node = doc.xml_create_element(u'input')
586 s60_version = self._bom.config['s60_version']
587 s60_release = self._bom.config['s60_release']
588 if s60_version != None:
589 s60_year = s60_version[0:4]
590 s60_week = s60_version[4:]
591 else:
592 s60_year = u'0'
593 s60_week = u'0'
594 s60_input_node.xml_append(doc.xml_create_element(u'name', content=(unicode("s60"))))
595 s60_input_node.xml_append(doc.xml_create_element(u'year', content=(unicode(s60_year))))
596 s60_input_node.xml_append(doc.xml_create_element(u'week', content=(unicode(s60_week))))
597 s60_input_node.xml_append(doc.xml_create_element(u'version', content=(unicode(s60_release))))
598
599 s60_input_source = s60_input_node.xml_create_element(u'source')
600 s60_input_source.xml_append(doc.xml_create_element(u'type', content=(unicode("grace"))))
601 s60_input_source.xml_append(doc.xml_create_element(u'service', content=(unicode(service))))
602 s60_input_source.xml_append(doc.xml_create_element(u'product', content=(unicode(product))))
603 s60_input_source.xml_append(doc.xml_create_element(u'release', content=(unicode(release))))
604 s60_input_node.xml_append(s60_input_source)
605 doc.bom.content.xml_append(s60_input_node)
606 out = open(path, 'w')
607 doc.xml(out, indent='yes')
608 out.close()
609
611 _log_array = log.split('\r')
612 if(len(_log_array) == 3 and log.find('completed') > 0):
613 _completed_line = _log_array[2]
614 return _completed_line[:_completed_line.rfind(':')].strip()
615 else:
616 return u'None'
617