diff -r 7685cec9fd3c -r f2ddfa555b0f doc/api/python/preparation-pysrc.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/api/python/preparation-pysrc.html Fri Sep 11 11:54:49 2009 +0100 @@ -0,0 +1,3556 @@ + + + + + preparation + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Module preparation + + + + + + +
[hide private]
[frames] | no frames]
+
+

Source Code for Module preparation

+
+  1  #============================================================================  
+  2  #Name        : preparation.py  
+  3  #Part of     : Helium  
+  4   
+  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
+  6  #All rights reserved. 
+  7  #This component and the accompanying materials are made available 
+  8  #under the terms of the License "Eclipse Public License v1.0" 
+  9  #which accompanies this distribution, and is available 
+ 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
+ 11  # 
+ 12  #Initial Contributors: 
+ 13  #Nokia Corporation - initial contribution. 
+ 14  # 
+ 15  #Contributors: 
+ 16  # 
+ 17  #Description: 
+ 18  #=============================================================================== 
+ 19   
+ 20  """ This package implements the new update work area functionality. 
+ 21   
+ 22  """ 
+ 23   
+ 24  import logging 
+ 25  import os 
+ 26  import shutil 
+ 27  import time 
+ 28  import xml.dom.minidom 
+ 29   
+ 30  import ccm 
+ 31  import ccm.extra 
+ 32  import configuration 
+ 33  import fileutils 
+ 34   
+ 35   
+ 36  # Uncomment this line to enable logging in this module, or configure logging elsewhere 
+ 37  logging.basicConfig(level=logging.INFO) 
+ 38  _logger = logging.getLogger("preparation.ccmgetinput") 
+ 39   
+ 40   
+ 41  DEFAULT_THREADS = 1 
+ 42  THREADS_MIN_TOTAL = 1 
+ 43  THREADS_MAX_TOTAL = 10 
+ 44   
+ 45   
+
46 -def find(function, seq): +
47 """Return first item in sequence where f(item) == True.""" + 48 for item in seq: + 49 if function(item): + 50 return item + 51 return None +
52 + 53 +
54 -class PreparationAction(object): +
55 """ Implements an abstract preparation function. """ + 56 +
57 - def __init__(self, config, builder): +
58 self._config = config + 59 self._builder = builder +
60 +
61 - def check(self): +
62 """ Checks if project is available in synergy. """ + 63 self._check_object(self._config.name) +
64 +
65 - def _check_object(self, fpn): +
66 """ Check if ccmobject exists in synergy database. """ + 67 session = self.get_session() + 68 ccm_object = session.create(fpn) + 69 if ccm_object.exists(): + 70 _logger.info("Checking '%s'...Ok" % fpn) + 71 else: + 72 _logger.info("Checking '%s'...Not Found!" % fpn) + 73 raise Exception("Could not find object %s in the database." % fpn) +
74 +
75 - def execute(self): +
76 """ This method needs to be override by child class. + 77 + 78 It should implement the action to achieve. + 79 """ + 80 pass +
81 +
82 - def get_session(self): +
83 """ Helper that retreive the session from the builder. Setting threads correctly. """ + 84 if self._config.has_key('host'): + 85 return self._builder.session(self._config['database'], self.get_threads(), self._config['host'], self._config['dbpath']) + 86 return self._builder.session(self._config['database'], self.get_threads()) +
87 +
88 - def get_threads(self): +
89 """ Returning the number of threads that should be used. """ + 90 threads = self._config.get_int('threads', DEFAULT_THREADS) + 91 if threads < THREADS_MIN_TOTAL: + 92 threads = THREADS_MIN_TOTAL + 93 if threads > THREADS_MAX_TOTAL: + 94 threads = THREADS_MAX_TOTAL + 95 return threads +
96 + 97 +
98 -class PreparationSnapshot(PreparationAction): +
99 """ Implements a Snapshot preparation function. +100 +101 Support the parallel snapshotter. +102 """ +103 +
104 - def __init__(self, config, builder): +
105 """ Initialization. """ +106 PreparationAction.__init__(self, config, builder) +
107 +
108 - def execute(self): +
109 """ Method that implements snapshoting of the project into a folder. """ +110 _logger.info("=== Stage=snapshot = %s" % self._config.name) +111 _logger.info("++ Started at %s" % time.strftime("%H:%M:%S", time.localtime())) +112 session = self.get_session() +113 project = session.create(self._config.name) +114 +115 target_dir = os.path.join(self._config['dir'], project.name) +116 if not self._check_version(target_dir): +117 if (not os.path.exists(target_dir)): +118 _logger.info("Creating '%s'." % target_dir) +119 os.makedirs(target_dir) +120 else: +121 _logger.info("Project needs to be updated, so deleting '%s'." % target_dir) +122 fileutils.rmtree(target_dir) +123 +124 try: +125 _logger.info("Snapshotting project.") +126 if self.get_threads() == 1: +127 _logger.info(project.snapshot(target_dir, True)) +128 else: +129 _logger.info(ccm.extra.FastSnapshot(project, target_dir, self.get_threads())) +130 +131 # writing version file +132 _logger.info("Saving project version information.") +133 versionfile = open(os.path.join(self._config['dir'], project.name, 'project.version'), "w+") +134 versionfile.write(str(project)) +135 versionfile.close() +136 except Exception, exc: +137 if isinstance(exc, ccm.extra.CCMExtraException): +138 for sexc in exc.subexceptions: +139 _logger.info(sexc) +140 _logger.info("ERROR: snapshotting %s" % self._config.name) +141 _logger.info(exc) +142 raise exc +143 else: +144 _logger.info("Project snapshot is still up to date. Nothing to do.") +145 +146 _logger.info("++ Finished at %s" % time.strftime("%H:%M:%S", time.localtime())) +
147 +
148 - def _check_version(self, targetdir): +
149 """ Check the version file for snaphot and identify if the project has to be snapshot or not. +150 Returns True if the content of the file matches the project to snapshot (nothing to do). +151 Returns falls either if the file is missing, or the content is different. +152 """ +153 versionfile = os.path.join(targetdir,'project.version') +154 if (os.path.exists(versionfile)): +155 file_ = open(versionfile, "r") +156 projectname = file_.read().strip() +157 file_.close() +158 if (projectname == self._config.name): +159 return True +160 return False +
161 +162 +
163 -class PreparationCheckout(PreparationAction): +
164 """ Handle the checkout and update of project content. """ +
165 - def __init__(self, config, builder): +
166 """ Initialization. """ +167 PreparationAction.__init__(self, config, builder) +
168 +
169 - def check(self): +
170 """ Checks if all synergy resources are available. """ +171 PreparationAction.check(self) +172 if self._config.has_key('release'): +173 self._check_object(str(self._config['release'])) +174 else: +175 raise Exception("'release' property is not defined for %s" % self._config.name) +176 +177 for task in self.__get_tasks(): +178 self._check_object("Task %s" % task) +179 for folder in self.__get_folders(): +180 self._check_object("Folder %s" % folder) +181 +182 for project in self.__get_subbaselines(): +183 self._check_object(project) +184 +185 # checking if the purpose exists +186 if self._config.has_key('purpose'): +187 session = self.get_session() +188 purposes = session.purposes() +189 if purposes.has_key(str(self._config['purpose'])): +190 _logger.info("Checking purpose '%s'...Ok" % (self._config['purpose'])) +191 else: +192 _logger.info("Checking purpose '%s'...Not Found!" % (self._config['purpose'])) +193 raise Exception("Could not find purpose %s in the database." % self._config['purpose']) +194 +195 role = session.role +196 co_role = ccm.get_role_for_purpose(session, str(self._config['purpose'])) +197 _logger.info("Try to switch user to role: %s" % co_role) +198 session.role = co_role +199 session.role = role +
200 +
201 - def execute(self): +
202 """ Creates a checkout of the project, or updates an existing checkout if one is found. +203 +204 The work area is maintained as part of this. +205 """ +206 _logger.info("=== Stage=checkout = %s" % self._config.name) +207 _logger.info("++ Started at %s" % time.strftime("%H:%M:%S", time.localtime())) +208 session = self.get_session() +209 project = session.create(self._config.name) +210 +211 session.home = self._config['dir'] +212 +213 result = self.__find_project(project) +214 # for testing: result = session.create("ppd_sw-fa1f5132#wbernard2:project:sa1spp#1") +215 if (result != None): +216 _logger.info("Project found: '%s'" % result) +217 +218 # setting up the project +219 self.__setup_project(project, result) +220 else: +221 _logger.info("Checking out from '%s'." % project) +222 +223 purpose = None +224 if self._config.has_key('purpose'): +225 purpose = self._config['purpose'] +226 _logger.info("Using purpose: '%s'" % purpose) +227 +228 version = None +229 if self._config.has_key('version'): +230 version = self._config['version'] +231 _logger.info("Using version: '%s'" % version) +232 +233 try: +234 result = project.checkout(session.create(self._config['release']), version=version, purpose=purpose) +235 ccm.log_result(result, ccm.CHECKOUT_LOG_RULES, _logger) +236 except ccm.CCMException, exc: +237 ccm.log_result(exc.result, ccm.CHECKOUT_LOG_RULES, _logger) +238 raise exc +239 _logger.info('Checkout complete') +240 +241 if result.project != None and result.project.exists(): +242 _logger.info("Project checked out: '%s'" % result.project) +243 +244 +245 _logger.info("Maintaining the workarea...") +246 if self.get_threads() == 1: +247 _logger.info(result.project.work_area(True, True, True, self._config['dir'], result.project.name)) +248 else: +249 #pool = self._builder.session(self._config['database'], self.get_threads()) +250 _logger.info(ccm.extra.FastMaintainWorkArea(result.project, self._config['dir'], result.project.name, self.get_threads())) +251 self.__setup_project(project, result.project) +252 else: +253 raise Exception("Error checking out '%s'" % project) +254 _logger.info("++ Finished at %s" % time.strftime("%H:%M:%S", time.localtime())) +
255 +
256 - def __find_project(self, project): +
257 """ Private method. """ +258 if (os.path.exists(os.path.join(self._config['dir'], project.name, "project.version"))): +259 _logger.info("Snapshot to checkout deleting '%s'." % os.path.join(self._config['dir'], project.name)) +260 fileutils.rmtree(os.path.join(self._config['dir'], project.name)) +261 return None +262 +263 path = os.path.join(self._config['dir'], project.name, project.name) +264 try: +265 result = project.session.get_workarea_info(path) +266 if(result == None): +267 fileutils.rmtree(path) +268 return result +269 return result['project'] +270 except ccm.CCMException: +271 # Delete the project dir if found +272 if os.path.exists(os.path.dirname(path)): +273 fileutils.rmtree(os.path.dirname(path)) +274 return None +
275 +
276 - def __setup_project(self, project, coproject): +
277 """ Private method. """ +278 session = self.get_session() +279 role = session.role +280 if self._config.has_key('purpose'): +281 co_role = ccm.get_role_for_purpose(session, self._config['purpose']) +282 _logger.info("Switching user to role: %s" % co_role) +283 session.role = co_role +284 +285 newprojs = [] +286 if not self._config.get_boolean('use.reconfigure.template', False): +287 _logger.info("Validating release") +288 self.__set_release(coproject) +289 _logger.info("Setting update properties to manual") +290 coproject.set_update_method('manual', True) +291 _logger.info("Setting the baseline to '%s'" % project) +292 coproject.set_baseline(project, True) +293 self.__set_subbaselines(coproject) +294 _logger.info("Cleaning up update properties") +295 self._clean_update_properties(coproject) +296 _logger.info("Setting update properties.") +297 self._set_tasks_and_folders(coproject) +298 _logger.info("Applying update properties.") +299 coproject.apply_update_properties(baseline=False) +300 else: +301 _logger.info("Validating release") +302 self.__set_release(coproject) +303 +304 replace_subprojects = True +305 if not self._config.get_boolean('replace.subprojects', True): +306 _logger.info("NOT replacing subprojects") +307 replace_subprojects = False +308 update_keepgoing = True +309 if self._config.get_boolean('update.failonerror', False): +310 _logger.info("The build will fail with update errors") +311 update_keepgoing = False +312 _logger.info("Updating...") +313 result = coproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(coproject.session)) +314 +315 if self._config.get_boolean('fix.missing.baselines', False) and replace_subprojects: +316 newprojs = self.__fix_baseline(coproject) +317 if len(newprojs) > 0: +318 result = coproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(coproject.session)) +319 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) +320 _logger.info("Detected additional projects into baseline - Maintaining the whole toplevel project again...") +321 coproject.work_area(True, True) +322 else: +323 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) +324 else: +325 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) +326 +327 # Running sync +328 self._sync(coproject) +329 +330 # Running check conflicts +331 self._check_conflicts(coproject) +332 +333 _logger.info("Switching user to role: %s" % role) +334 session.role = role +
335 +
336 - def _sync(self, coproject): +
337 """ Run the sync if the 'sync' property is defined to true in the +338 configuration +339 """ +340 if self._config.get_boolean('sync', False): +341 _logger.info("Synchronizing...") +342 result = coproject.sync(True, True) +343 ccm.log_result(result, ccm.SYNC_LOG_RULES, _logger) +
344 +345 +
346 - def __set_release(self, project): +
347 """ Update the release of the project hierarchy if required. """ +348 release = project.session.create(self._config['release']) +349 _logger.info("Current release: '%s'" % project.release) +350 _logger.info("Configuration release: '%s'" % release) +351 if project.release != release: +352 _logger.info("Updating release on the project hierarchy.") +353 for subp in [project] + project.subprojects: +354 subp.release = release +
355 +
356 - def __fix_baseline(self, coproject): +
357 """ Check for project in a different status, then check them out. """ +358 newprojs = [] +359 _logger.info("Looking for new projects in the check out.") +360 status = coproject['status'] +361 for subproj in coproject.subprojects: +362 if subproj['status'] == status: +363 continue +364 _logger.info("New project detected in the checkout '%s'" % subproj.objectname) +365 purpose = None +366 if self._config.has_key('purpose'): +367 purpose = self._config['purpose'] +368 _logger.info("Using purpose: '%s'" % purpose) +369 +370 version = None +371 if self._config.has_key('version'): +372 version = self._config['version'] +373 _logger.info("Using version: '%s'" % version) +374 +375 result = subproj.checkout(subproj.session.create(self._config['release']), version=version, purpose=purpose, subprojects=False) +376 _logger.info('Checkout complete') +377 if result.project != None and result.project.exists(): +378 newcop = result.project +379 newprojs.append(newcop) +380 +381 _logger.info("Setting is_relative to true") +382 if "is_relative" in newcop.keys(): +383 newcop["is_relative"] = "TRUE" +384 else: +385 newcop.create_attribute("is_relative", "boolean", "TRUE") +386 +387 if not self._config.get_boolean('use.reconfigure.template', False): +388 newcop.set_update_method('manual', False) +389 +390 _logger.info("Setting the baseline to '%s'" % subproj) +391 newcop.set_baseline(subproj, True) +392 +393 _logger.info("Cleaning up update properties") +394 self._clean_update_properties(newcop) +395 +396 _logger.info("Setting update properties.") +397 self._set_tasks_and_folders(newcop) +398 return newprojs +
399 +
400 - def _check_conflicts(self, coproject): +
401 """ Private method. """ +402 if not self._config.get_boolean('show.conflicts', False): +403 return +404 +405 result = coproject.conflicts(True, True) +406 ccm.log_result(result, ccm.CONFLICTS_LOG_RULES, _logger) +
407 # for project in result.keys(): +408 # for error in result[project]: +409 # if 'object' in error: +410 # _logger.info("CONFLICTS: %s" % error['comment']) +411 # else: +412 # _logger.info("CONFLICTS: %s" % error['comment']) +413 +414 @staticmethod +
415 - def _clean_update_properties(project): +
416 """ Private method. """ +417 for task in project.tasks: +418 project.remove_task(task) +419 for folder in project.folders: +420 project.remove_folder(folder) +
421 +422 @staticmethod +
423 - def __find_subproject(subprojects, project): +
424 """ Private method. """ +425 for subproj in subprojects: +426 if subproj.is_same_family(project): +427 return subproj +428 raise Exception("Error could not identify check out project for '%s'" % project) +
429 +
430 - def __set_subbaselines(self, project): +
431 """ Private method. """ +432 if len(self.__get_subbaselines()) > 0: +433 subprojects = project.subprojects +434 for subbaseline in self.__get_subbaselines(): +435 subbaseline = project.session.create(subbaseline) +436 subproj = self.__find_subproject(subprojects, subbaseline) +437 _logger.info("Setting subproject '%s' baseline to '%s'" % (subproj, subbaseline)) +438 subproj.set_baseline(subbaseline, True) +
439 +
440 - def __get_array(self, key): +
441 """ Private method. """ +442 result = [] +443 if (self._config.has_key(key)): +444 if isinstance(self._config[key], type([])): +445 for value in self._config[key]: +446 value = value.strip() +447 if len(value) > 0: +448 result.append(value) +449 else: +450 value = self._config[key].strip() +451 if len(value) > 0: +452 result.append(value) +453 return result +
454 +
455 - def __get_subbaselines(self): +
456 """ Private method. """ +457 return self.__get_array('subbaselines') +
458 +
459 - def __get_tasks(self): +
460 """ Private method. """ +461 return self.__get_array('tasks') +
462 +
463 - def __get_folders(self): +
464 """ Private method. """ +465 return self.__get_array('folders') +
466 +
467 - def _set_tasks_and_folders(self, project): +
468 """ Private method. """ +469 for task in self.__get_tasks(): +470 _logger.info("Adding task %s" % task) +471 project.add_task(project.session.create("Task %s" % task)) +472 for folder in self.__get_folders(): +473 _logger.info("Adding folder %s" % folder) +474 project.add_folder(project.session.create("Folder %s" % folder)) +
475 +
476 -class PreparationUpdate(PreparationCheckout): +
477 """ Synergy project updater. """ +478 +
479 - def __init__(self, config, builder): +
480 """ Initialization. """ +481 PreparationCheckout.__init__(self, config, builder) +
482 +
483 - def check(self): +
484 """ Checks if all synergy resources are available. """ +485 PreparationAction.check(self) +486 +487 session = self.get_session() +488 ccm_object = session.create(self._config.name) +489 role = session.role +490 co_role = ccm.get_role_for_status(session, ccm_object['status']) +491 _logger.info("Try to switch user to role: %s" % co_role) +492 session.role = co_role +493 session.role = role +
494 +
495 - def execute(self): +
496 """ Updating the mentioned project. """ +497 +498 session = self.get_session() +499 ccmproject = session.create(self._config.name) +500 role = session.role +501 +502 status = ccmproject['status'] +503 co_role = ccm.get_role_for_status(session, status) +504 session.role = co_role +505 +506 if not self._config.get_boolean('use.reconfigure.template', False): +507 _logger.info("Setting update properties to manual") +508 ccmproject.set_update_method('manual', True) +509 _logger.info("Cleaning up update properties") +510 self._clean_update_properties(ccmproject) +511 _logger.info("Setting update properties.") +512 self._set_tasks_and_folders(ccmproject) +513 _logger.info("Applying update properties.") +514 ccmproject.apply_update_properties(baseline=False) +515 replace_subprojects = True +516 if not self._config.get_boolean('replace.subprojects', True): +517 _logger.info("NOT replacing subprojects") +518 replace_subprojects = False +519 update_keepgoing = True +520 if self._config.get_boolean('update.failonerror', False): +521 _logger.info("The build will fail with update errors") +522 update_keepgoing = False +523 +524 _logger.info("Updating %s..." % ccmproject.objectname) +525 result = ccmproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(ccmproject.session)) +526 ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger) +527 +528 self._sync(ccmproject) +529 +530 self._check_conflicts(ccmproject) +531 +532 session.role = role +
533 +
534 -class PreparationBuilder(object): +
535 """ Creates an updated work area from a configuration. """ +
536 - def __init__(self, configs, username = None, password = None, cache=None): +
537 """ Initialization. """ +538 self._configs = configs +539 self._sessions = {} +540 self._actions = [] +541 self.__username = username +542 self.__password = password +543 self.__provider = ccm.extra.CachedSessionProvider(cache=cache) +544 for config in self._configs: +545 if config.type == "snapshot": +546 self._actions.append(PreparationSnapshot(config, self)) +547 elif config.type == "checkout": +548 self._actions.append(PreparationCheckout(config, self)) +549 elif config.type == "update": +550 self._actions.append(PreparationUpdate(config, self)) +
551 +
552 - def check(self): +
553 """ Check that all dependencies are there. """ +554 for action in self._actions: +555 action.check() +
556 +
557 - def get_content(self): +
558 """ Run the each action. """ +559 for action in self._actions: +560 action.execute() +
561 +
562 - def session(self, database, size=1, engine=None, dbpath=None): +
563 """ Handles pool rather that sessions. """ +564 assert size > 0, "The pool must contains at least one session!" +565 if self.__provider is None: +566 raise Exception("The builder has been closed.") +567 if not self._sessions.has_key(database): +568 _logger.info("Get a session for %s" % database) +569 session = ccm.SessionPool(self.__username, self.__password, engine, dbpath, database, size, opener=self.__provider.get) +570 self._sessions[database] = session +571 # be developer by default +572 session.role = "developer" +573 session = self._sessions[database] +574 if session.size < size: +575 _logger.info("Resizing the pool for database %s to %d" % (database, size)) +576 session.size = size +577 # be developer by default +578 session.role = "developer" +579 return session +
580 +
581 - def close(self): +
582 """ This is the preparation cleanup method. +583 It closes all opened sessions. +584 """ +585 _logger.debug("Closing sessions...") +586 dbs = self._sessions.keys() +587 while len(dbs) > 0: +588 session = self._sessions.pop(dbs.pop()) +589 session.close() +590 self.__provider.close() +591 self.__provider = None +
592 +593 +
594 - def __del__(self): +
595 self.close() +
596 +
+
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + +