Module preparation
[hide private]
[frames] | no frames]

Source Code for Module preparation

  1  #============================================================================  
  2  #Name        : preparation.py  
  3  #Part of     : Helium  
  4   
  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
  6  #All rights reserved. 
  7  #This component and the accompanying materials are made available 
  8  #under the terms of the License "Eclipse Public License v1.0" 
  9  #which accompanies this distribution, and is available 
 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
 11  # 
 12  #Initial Contributors: 
 13  #Nokia Corporation - initial contribution. 
 14  # 
 15  #Contributors: 
 16  # 
 17  #Description: 
 18  #=============================================================================== 
 19   
 20  """ This package implements the new update work area functionality. 
 21   
 22  """ 
 23   
 24  import logging 
 25  import os 
 26  import shutil 
 27  import time 
 28  import xml.dom.minidom 
 29   
 30  import ccm 
 31  import ccm.extra 
 32  import configuration 
 33  import fileutils 
 34   
 35   
 36  # Uncomment this line to enable logging in this module, or configure logging elsewhere 
 37  logging.basicConfig(level=logging.INFO) 
 38  _logger = logging.getLogger("preparation.ccmgetinput") 
 39   
 40   
 41  DEFAULT_THREADS = 1 
 42  THREADS_MIN_TOTAL = 1 
 43  THREADS_MAX_TOTAL = 10 
 44   
 45   
46 -def find(function, seq):
47 """Return first item in sequence where f(item) == True.""" 48 for item in seq: 49 if function(item): 50 return item 51 return None
52 53
54 -class PreparationAction(object):
55 """ Implements an abstract preparation function. """ 56
57 - def __init__(self, config, builder):
58 self._config = config 59 self._builder = builder
60
61 - def check(self):
62 """ Checks if project is available in synergy. """ 63 self._check_object(self._config.name)
64
65 - def _check_object(self, fpn):
66 """ Check if ccmobject exists in synergy database. """ 67 session = self.get_session() 68 ccm_object = session.create(fpn) 69 if ccm_object.exists(): 70 _logger.info("Checking '%s'...Ok" % fpn) 71 else: 72 _logger.info("Checking '%s'...Not Found!" % fpn) 73 raise Exception("Could not find object %s in the database." % fpn)
74
75 - def execute(self):
76 """ This method needs to be override by child class. 77 78 It should implement the action to achieve. 79 """ 80 pass
81
82 - def get_session(self):
83 """ Helper that retreive the session from the builder. Setting threads correctly. """ 84 if self._config.has_key('host'): 85 return self._builder.session(self._config['database'], self.get_threads(), self._config['host'], self._config['dbpath']) 86 return self._builder.session(self._config['database'], self.get_threads())
87
88 - def get_threads(self):
89 """ Returning the number of threads that should be used. """ 90 threads = self._config.get_int('threads', DEFAULT_THREADS) 91 if threads < THREADS_MIN_TOTAL: 92 threads = THREADS_MIN_TOTAL 93 if threads > THREADS_MAX_TOTAL: 94 threads = THREADS_MAX_TOTAL 95 return threads
96 97
98 -class PreparationSnapshot(PreparationAction):
99 """ Implements a Snapshot preparation function. 100 101 Support the parallel snapshotter. 102 """ 103
104 - def __init__(self, config, builder):
105 """ Initialization. """ 106 PreparationAction.__init__(self, config, builder)
107
108 - def execute(self):
109 """ Method that implements snapshoting of the project into a folder. """ 110 _logger.info("=== Stage=snapshot = %s" % self._config.name) 111 _logger.info("++ Started at %s" % time.strftime("%H:%M:%S", time.localtime())) 112 session = self.get_session() 113 project = session.create(self._config.name) 114 115 target_dir = os.path.join(self._config['dir'], project.name) 116 if not self._check_version(target_dir): 117 if (not os.path.exists(target_dir)): 118 _logger.info("Creating '%s'." % target_dir) 119 os.makedirs(target_dir) 120 else: 121 _logger.info("Project needs to be updated, so deleting '%s'." % target_dir) 122 fileutils.rmtree(target_dir) 123 124 try: 125 _logger.info("Snapshotting project.") 126 if self.get_threads() == 1: 127 _logger.info(project.snapshot(target_dir, True)) 128 else: 129 _logger.info(ccm.extra.FastSnapshot(project, target_dir, self.get_threads())) 130 131 # writing version file 132 _logger.info("Saving project version information.") 133 versionfile = open(os.path.join(self._config['dir'], project.name, 'project.version'), "w+") 134 versionfile.write(str(project)) 135 versionfile.close() 136 except Exception, exc: 137 if isinstance(exc, ccm.extra.CCMExtraException): 138 for sexc in exc.subexceptions: 139 _logger.info(sexc) 140 _logger.info("ERROR: snapshotting %s" % self._config.name) 141 _logger.info(exc) 142 raise exc 143 else: 144 _logger.info("Project snapshot is still up to date. Nothing to do.") 145 146 _logger.info("++ Finished at %s" % time.strftime("%H:%M:%S", time.localtime()))
147
148 - def _check_version(self, targetdir):
149 """ Check the version file for snaphot and identify if the project has to be snapshot or not. 150 Returns True if the content of the file matches the project to snapshot (nothing to do). 151 Returns falls either if the file is missing, or the content is different. 152 """ 153 versionfile = os.path.join(targetdir,'project.version') 154 if (os.path.exists(versionfile)): 155 file_ = open(versionfile, "r") 156 projectname = file_.read().strip() 157 file_.close() 158 if (projectname == self._config.name): 159 return True 160 return False
161 162
163 -class PreparationCheckout(PreparationAction):
164 """ Handle the checkout and update of project content. """
165 - def __init__(self, config, builder):
166 """ Initialization. """ 167 PreparationAction.__init__(self, config, builder)
168
169 - def check(self):
170 """ Checks if all synergy resources are available. """ 171 PreparationAction.check(self) 172 if self._config.has_key('release'): 173 self._check_object(str(self._config['release'])) 174 else: 175 raise Exception("'release' property is not defined for %s" % self._config.name) 176 177 for task in self.__get_tasks(): 178 self._check_object("Task %s" % task) 179 for folder in self.__get_folders(): 180 self._check_object("Folder %s" % folder) 181 182 for project in self.__get_subbaselines(): 183 self._check_object(project) 184 185 # checking if the purpose exists 186 if self._config.has_key('purpose'): 187 session = self.get_session() 188 purposes = session.purposes() 189 if purposes.has_key(str(self._config['purpose'])): 190 _logger.info("Checking purpose '%s'...Ok" % (self._config['purpose'])) 191 else: 192 _logger.info("Checking purpose '%s'...Not Found!" % (self._config['purpose'])) 193 raise Exception("Could not find purpose %s in the database." % self._config['purpose']) 194 195 role = session.role 196 co_role = ccm.get_role_for_purpose(session, str(self._config['purpose'])) 197 _logger.info("Try to switch user to role: %s" % co_role) 198 session.role = co_role 199 session.role = role
200
201 - def execute(self):
202 """ Creates a checkout of the project, or updates an existing checkout if one is found. 203 204 The work area is maintained as part of this. 205 """ 206 _logger.info("=== Stage=checkout = %s" % self._config.name) 207 _logger.info("++ Started at %s" % time.strftime("%H:%M:%S", time.localtime())) 208 session = self.get_session() 209 project = session.create(self._config.name) 210 211 session.home = self._config['dir'] 212 213 result = self.__find_project(project) 214 # for testing: result = session.create("ppd_sw-fa1f5132#wbernard2:project:sa1spp#1") 215 if (result != None): 216 _logger.info("Project found: '%s'" % result) 217 218 # setting up the project 219 self.__setup_project(project, result) 220 else: 221 _logger.info("Checking out from '%s'." % project) 222 223 purpose = None 224 if self._config.has_key('purpose'): 225 purpose = self._config['purpose'] 226 _logger.info("Using purpose: '%s'" % purpose) 227 228 version = None 229 if self._config.has_key('version'): 230 version = self._config['version'] 231 _logger.info("Using version: '%s'" % version) 232 233 try: 234 result = project.checkout(session.create(self._config['release']), version=version, purpose=purpose) 235 ccm.log_result(result, ccm.CHECKOUT_LOG_RULES, _logger) 236 except ccm.CCMException, exc: 237 ccm.log_result(exc.result, ccm.CHECKOUT_LOG_RULES, _logger) 238 raise exc 239 _logger.info('Checkout complete') 240 241 if result.project != None and result.project.exists(): 242 _logger.info("Project checked out: '%s'" % result.project) 243 244 245 _logger.info("Maintaining the workarea...") 246 if self.get_threads() == 1: 247 _logger.info(result.project.work_area(True, True, True, self._config['dir'], result.project.name)) 248 else: 249 #pool = self._builder.session(self._config['database'], self.get_threads()) 250 _logger.info(ccm.extra.FastMaintainWorkArea(result.project, self._config['dir'], result.project.name, self.get_threads())) 251 self.__setup_project(project, result.project) 252 else: 253 raise Exception("Error checking out '%s'" % project) 254 _logger.info("++ Finished at %s" % time.strftime("%H:%M:%S", time.localtime()))
255
256 - def __find_project(self, project):
257 """ Private method. """ 258 if (os.path.exists(os.path.join(self._config['dir'], project.name, "project.version"))): 259 _logger.info("Snapshot to checkout deleting '%s'." % os.path.join(self._config['dir'], project.name)) 260 fileutils.rmtree(os.path.join(self._config['dir'], project.name)) 261 return None 262 263 path = os.path.join(self._config['dir'], project.name, project.name) 264 try: 265 result = project.session.get_workarea_info(path) 266 if(result == None): 267 fileutils.rmtree(path) 268 return result 269 return result['project'] 270 except ccm.CCMException: 271 # Delete the project dir if found 272 if os.path.exists(os.path.dirname(path)): 273 fileutils.rmtree(os.path.dirname(path)) 274 return None
275
276 - def __setup_project(self, project, coproject):
277 """ Private method. """ 278 session = self.get_session() 279 role = session.role 280 if self._config.has_key('purpose'): 281 co_role = ccm.get_role_for_purpose(session, self._config['purpose']) 282 _logger.info("Switching user to role: %s" % co_role) 283 session.role = co_role 284 285 newprojs = [] 286 if not self._config.get_boolean('use.reconfigure.template', False): 287 _logger.info("Validating release") 288 self.__set_release(coproject) 289 _logger.info("Setting update properties to manual") 290 coproject.set_update_method('manual', True) 291 _logger.info("Setting the baseline to '%s'" % project) 292 coproject.set_baseline(project, True) 293 self.__set_subbaselines(coproject) 294 _logger.info("Cleaning up update properties") 295 self._clean_update_properties(coproject) 296 _logger.info("Setting update properties.") 297 self._set_tasks_and_folders(coproject) 298 _logger.info("Applying update properties.") 299 coproject.apply_update_properties(baseline=False) 300 else: 301 _logger.info("Validating release") 302 self.__set_release(coproject) 303 304 replace_subprojects = True 305 if not self._config.get_boolean('replace.subprojects', True): 306 _logger.info("NOT replacing subprojects") 307 replace_subprojects = False 308 update_keepgoing = True 309 if self._config.get_boolean('update.failonerror', False): 310 _logger.info("The build will fail with update errors") 311 update_keepgoing = False 312 _logger.info("Updating...") 313 result = coproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(coproject.session)) 314 315 if self._config.get_boolean('fix.missing.baselines', False) and replace_subprojects: 316 newprojs = self.__fix_baseline(coproject) 317 if len(newprojs) > 0: 318 result = coproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(coproject.session)) 319 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) 320 _logger.info("Detected additional projects into baseline - Maintaining the whole toplevel project again...") 321 coproject.work_area(True, True) 322 else: 323 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) 324 else: 325 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) 326 327 # Running sync 328 self._sync(coproject) 329 330 # Running check conflicts 331 self._check_conflicts(coproject) 332 333 _logger.info("Switching user to role: %s" % role) 334 session.role = role
335
336 - def _sync(self, coproject):
337 """ Run the sync if the 'sync' property is defined to true in the 338 configuration 339 """ 340 if self._config.get_boolean('sync', False): 341 _logger.info("Synchronizing...") 342 result = coproject.sync(True, True) 343 ccm.log_result(result, ccm.SYNC_LOG_RULES, _logger)
344 345
346 - def __set_release(self, project):
347 """ Update the release of the project hierarchy if required. """ 348 release = project.session.create(self._config['release']) 349 _logger.info("Current release: '%s'" % project.release) 350 _logger.info("Configuration release: '%s'" % release) 351 if project.release != release: 352 _logger.info("Updating release on the project hierarchy.") 353 for subp in [project] + project.subprojects: 354 subp.release = release
355
356 - def __fix_baseline(self, coproject):
357 """ Check for project in a different status, then check them out. """ 358 newprojs = [] 359 _logger.info("Looking for new projects in the check out.") 360 status = coproject['status'] 361 for subproj in coproject.subprojects: 362 if subproj['status'] == status: 363 continue 364 _logger.info("New project detected in the checkout '%s'" % subproj.objectname) 365 purpose = None 366 if self._config.has_key('purpose'): 367 purpose = self._config['purpose'] 368 _logger.info("Using purpose: '%s'" % purpose) 369 370 version = None 371 if self._config.has_key('version'): 372 version = self._config['version'] 373 _logger.info("Using version: '%s'" % version) 374 375 result = subproj.checkout(subproj.session.create(self._config['release']), version=version, purpose=purpose, subprojects=False) 376 _logger.info('Checkout complete') 377 if result.project != None and result.project.exists(): 378 newcop = result.project 379 newprojs.append(newcop) 380 381 _logger.info("Setting is_relative to true") 382 if "is_relative" in newcop.keys(): 383 newcop["is_relative"] = "TRUE" 384 else: 385 newcop.create_attribute("is_relative", "boolean", "TRUE") 386 387 if not self._config.get_boolean('use.reconfigure.template', False): 388 newcop.set_update_method('manual', False) 389 390 _logger.info("Setting the baseline to '%s'" % subproj) 391 newcop.set_baseline(subproj, True) 392 393 _logger.info("Cleaning up update properties") 394 self._clean_update_properties(newcop) 395 396 _logger.info("Setting update properties.") 397 self._set_tasks_and_folders(newcop) 398 return newprojs
399
400 - def _check_conflicts(self, coproject):
401 """ Private method. """ 402 if not self._config.get_boolean('show.conflicts', False): 403 return 404 405 result = coproject.conflicts(True, True) 406 ccm.log_result(result, ccm.CONFLICTS_LOG_RULES, _logger)
407 # for project in result.keys(): 408 # for error in result[project]: 409 # if 'object' in error: 410 # _logger.info("CONFLICTS: %s" % error['comment']) 411 # else: 412 # _logger.info("CONFLICTS: %s" % error['comment']) 413 414 @staticmethod
415 - def _clean_update_properties(project):
416 """ Private method. """ 417 for task in project.tasks: 418 project.remove_task(task) 419 for folder in project.folders: 420 project.remove_folder(folder)
421 422 @staticmethod
423 - def __find_subproject(subprojects, project):
424 """ Private method. """ 425 for subproj in subprojects: 426 if subproj.is_same_family(project): 427 return subproj 428 raise Exception("Error could not identify check out project for '%s'" % project)
429
430 - def __set_subbaselines(self, project):
431 """ Private method. """ 432 if len(self.__get_subbaselines()) > 0: 433 subprojects = project.subprojects 434 for subbaseline in self.__get_subbaselines(): 435 subbaseline = project.session.create(subbaseline) 436 subproj = self.__find_subproject(subprojects, subbaseline) 437 _logger.info("Setting subproject '%s' baseline to '%s'" % (subproj, subbaseline)) 438 subproj.set_baseline(subbaseline, True)
439
440 - def __get_array(self, key):
441 """ Private method. """ 442 result = [] 443 if (self._config.has_key(key)): 444 if isinstance(self._config[key], type([])): 445 for value in self._config[key]: 446 value = value.strip() 447 if len(value) > 0: 448 result.append(value) 449 else: 450 value = self._config[key].strip() 451 if len(value) > 0: 452 result.append(value) 453 return result
454
455 - def __get_subbaselines(self):
456 """ Private method. """ 457 return self.__get_array('subbaselines')
458
459 - def __get_tasks(self):
460 """ Private method. """ 461 return self.__get_array('tasks')
462
463 - def __get_folders(self):
464 """ Private method. """ 465 return self.__get_array('folders')
466
467 - def _set_tasks_and_folders(self, project):
468 """ Private method. """ 469 for task in self.__get_tasks(): 470 _logger.info("Adding task %s" % task) 471 project.add_task(project.session.create("Task %s" % task)) 472 for folder in self.__get_folders(): 473 _logger.info("Adding folder %s" % folder) 474 project.add_folder(project.session.create("Folder %s" % folder))
475
476 -class PreparationUpdate(PreparationCheckout):
477 """ Synergy project updater. """ 478
479 - def __init__(self, config, builder):
480 """ Initialization. """ 481 PreparationCheckout.__init__(self, config, builder)
482
483 - def check(self):
484 """ Checks if all synergy resources are available. """ 485 PreparationAction.check(self) 486 487 session = self.get_session() 488 ccm_object = session.create(self._config.name) 489 role = session.role 490 co_role = ccm.get_role_for_status(session, ccm_object['status']) 491 _logger.info("Try to switch user to role: %s" % co_role) 492 session.role = co_role 493 session.role = role
494
495 - def execute(self):
496 """ Updating the mentioned project. """ 497 498 session = self.get_session() 499 ccmproject = session.create(self._config.name) 500 role = session.role 501 502 status = ccmproject['status'] 503 co_role = ccm.get_role_for_status(session, status) 504 session.role = co_role 505 506 if not self._config.get_boolean('use.reconfigure.template', False): 507 _logger.info("Setting update properties to manual") 508 ccmproject.set_update_method('manual', True) 509 _logger.info("Cleaning up update properties") 510 self._clean_update_properties(ccmproject) 511 _logger.info("Setting update properties.") 512 self._set_tasks_and_folders(ccmproject) 513 _logger.info("Applying update properties.") 514 ccmproject.apply_update_properties(baseline=False) 515 replace_subprojects = True 516 if not self._config.get_boolean('replace.subprojects', True): 517 _logger.info("NOT replacing subprojects") 518 replace_subprojects = False 519 update_keepgoing = True 520 if self._config.get_boolean('update.failonerror', False): 521 _logger.info("The build will fail with update errors") 522 update_keepgoing = False 523 524 _logger.info("Updating %s..." % ccmproject.objectname) 525 result = ccmproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(ccmproject.session)) 526 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) 527 528 self._sync(ccmproject) 529 530 self._check_conflicts(ccmproject) 531 532 session.role = role
533
534 -class PreparationBuilder(object):
535 """ Creates an updated work area from a configuration. """
536 - def __init__(self, configs, username = None, password = None, cache=None):
537 """ Initialization. """ 538 self._configs = configs 539 self._sessions = {} 540 self._actions = [] 541 self.__username = username 542 self.__password = password 543 self.__provider = ccm.extra.CachedSessionProvider(cache=cache) 544 for config in self._configs: 545 if config.type == "snapshot": 546 self._actions.append(PreparationSnapshot(config, self)) 547 elif config.type == "checkout": 548 self._actions.append(PreparationCheckout(config, self)) 549 elif config.type == "update": 550 self._actions.append(PreparationUpdate(config, self))
551
552 - def check(self):
553 """ Check that all dependencies are there. """ 554 for action in self._actions: 555 action.check()
556
557 - def get_content(self):
558 """ Run the each action. """ 559 for action in self._actions: 560 action.execute()
561
562 - def session(self, database, size=1, engine=None, dbpath=None):
563 """ Handles pool rather that sessions. """ 564 assert size > 0, "The pool must contains at least one session!" 565 if self.__provider is None: 566 raise Exception("The builder has been closed.") 567 if not self._sessions.has_key(database): 568 _logger.info("Get a session for %s" % database) 569 session = ccm.SessionPool(self.__username, self.__password, engine, dbpath, database, size, opener=self.__provider.get) 570 self._sessions[database] = session 571 # be developer by default 572 session.role = "developer" 573 session = self._sessions[database] 574 if session.size < size: 575 _logger.info("Resizing the pool for database %s to %d" % (database, size)) 576 session.size = size 577 # be developer by default 578 session.role = "developer" 579 return session
580
581 - def close(self):
582 """ This is the preparation cleanup method. 583 It closes all opened sessions. 584 """ 585 _logger.debug("Closing sessions...") 586 dbs = self._sessions.keys() 587 while len(dbs) > 0: 588 session = self._sessions.pop(dbs.pop()) 589 session.close() 590 self.__provider.close() 591 self.__provider = None
592 593
594 - def __del__(self):
595 self.close()
596