Module preparation |
[hide private]
[frames] | no frames]

# Source Code for Module preparation

      1  #============================================================================
      2  #Name        : preparation.py
      3  #Part of     : Helium
      4
      5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
      6  #All rights reserved.
      7  #This component and the accompanying materials are made available
      8  #under the terms of the License "Eclipse Public License v1.0"
      9  #which accompanies this distribution, and is available
     10  #at the URL "http://www.eclipse.org/legal/epl-v10.html".
     11  #
     12  #Initial Contributors:
     13  #Nokia Corporation - initial contribution.
     14  #
     15  #Contributors:
     16  #
     17  #Description:
     18  #===============================================================================
     19
     20  """ This package implements the new update work area functionality.
     21
     22  """
     23
     24  import logging
     25  import os
     26  import shutil
     27  import time
     28  import xml.dom.minidom
     29
     30  import ccm
     31  import ccm.extra
     32  import configuration
     33  import fileutils
     34
     35
     36  # Uncomment this line to enable logging in this module, or configure logging elsewhere
     37  logging.basicConfig(level=logging.INFO)
     38  _logger = logging.getLogger("preparation.ccmgetinput")
     39
     40
     41  DEFAULT_THREADS = 1
     42  THREADS_MIN_TOTAL = 1
     43  THREADS_MAX_TOTAL = 10
     44
     45
     46 -def find(function, seq):
     47      """Return first item in sequence where f(item) == True."""
     48      for item in seq:
     49          if function(item):
     50              return item
     51      return None
     52
     53
     54 -class PreparationAction(object):
     55      """ Implements an abstract preparation function. """
     56
     57 -    def __init__(self, config, builder):
     58          self._config = config
     59          self._builder = builder
     60
     61 -    def check(self):
     62          """ Checks if project is available in synergy. """
     63          self._check_object(self._config.name)
     64
     65 -    def _check_object(self, fpn):
     66          """ Check if ccmobject exists in synergy database. """
     67          session = self.get_session()
     68          ccm_object = session.create(fpn)
     69          if ccm_object.exists():
     70              _logger.info("Checking '%s'...Ok" % fpn)
     71          else:
     72              _logger.info("Checking '%s'...Not Found!" % fpn)
     73              raise Exception("Could not find  object %s in the database." % fpn)
     74
     75 -    def execute(self):
     76          """ This method needs to be override by child class.
     77
     78          It should implement the action to achieve.
     79          """
     80          pass
     81
     82 -    def get_session(self):
     83          """ Helper that retreive the session from the builder. Setting threads correctly. """
     84          if self._config.has_key('host'):
     85              return self._builder.session(self._config['database'], self.get_threads(), self._config['host'], self._config['dbpath'])
     86          return self._builder.session(self._config['database'], self.get_threads())
     87
     88 -    def get_threads(self):
     89          """ Returning the number of threads that should be used. """
     90          threads = self._config.get_int('threads', DEFAULT_THREADS)
     91          if threads < THREADS_MIN_TOTAL:
     92              threads = THREADS_MIN_TOTAL
     93          if threads > THREADS_MAX_TOTAL:
     94              threads = THREADS_MAX_TOTAL
     95          return threads
     96
     97
     98 -class PreparationSnapshot(PreparationAction):
     99      """ Implements a Snapshot preparation function.
    100
    101      Support the parallel snapshotter.
    102      """
    103
    104 -    def __init__(self, config, builder):
    105          """ Initialization. """
    106          PreparationAction.__init__(self, config, builder)
    107
    108 -    def execute(self):
    109          """ Method that implements snapshoting of the project into a folder. """
    110          _logger.info("=== Stage=snapshot = %s" % self._config.name)
    111          _logger.info("++ Started at %s" % time.strftime("%H:%M:%S", time.localtime()))
    112          session = self.get_session()
    113          project = session.create(self._config.name)
    114
    115          target_dir = os.path.join(self._config['dir'], project.name)
    116          if not self._check_version(target_dir):
    117              if (not os.path.exists(target_dir)):
    118                  _logger.info("Creating '%s'." % target_dir)
    119                  os.makedirs(target_dir)
    120              else:
    121                  _logger.info("Project needs to be updated, so deleting '%s'." % target_dir)
    122                  fileutils.rmtree(target_dir)
    123
    124              try:
    125                  _logger.info("Snapshotting project.")
    126                  if self.get_threads() == 1:
    127                      _logger.info(project.snapshot(target_dir, True))
    128                  else:
    129                      _logger.info(ccm.extra.FastSnapshot(project, target_dir, self.get_threads()))
    130
    131                  # writing version file
    132                  _logger.info("Saving project version information.")
    133                  versionfile = open(os.path.join(self._config['dir'], project.name, 'project.version'), "w+")
    134                  versionfile.write(str(project))
    135                  versionfile.close()
    136              except Exception, exc:
    137                  if isinstance(exc, ccm.extra.CCMExtraException):
    138                      for sexc in exc.subexceptions:
    139                          _logger.info(sexc)
    140                  _logger.info("ERROR: snapshotting %s" % self._config.name)
    141                  _logger.info(exc)
    142                  raise exc
    143          else:
    144              _logger.info("Project snapshot is still up to date. Nothing to do.")
    145
    146          _logger.info("++ Finished at %s" % time.strftime("%H:%M:%S", time.localtime()))
    147
    148 -    def _check_version(self, targetdir):
    149          """ Check the version file for snaphot and identify if the project has to be snapshot or not.
    150              Returns True if the content of the file matches the project to snapshot (nothing to do).
    151              Returns falls either if the file is missing, or the content is different.
    152          """
    153          versionfile = os.path.join(targetdir,'project.version')
    154          if (os.path.exists(versionfile)):
    155              file_ = open(versionfile, "r")
    156              projectname = file_.read().strip()
    157              file_.close()
    158              if (projectname == self._config.name):
    159                  return True
    160          return False
    161
    162
    163 -class PreparationCheckout(PreparationAction):
    164      """ Handle the checkout and update of project content. """
    165 -    def __init__(self, config, builder):
    166          """ Initialization. """
    167          PreparationAction.__init__(self, config, builder)
    168
    169 -    def check(self):
    170          """ Checks if all synergy resources are available. """
    171          PreparationAction.check(self)
    172          if self._config.has_key('release'):
    173              self._check_object(str(self._config['release']))
    174          else:
    175              raise Exception("'release' property is not defined for %s" % self._config.name)
    176
    177          for task in self.__get_tasks():
    178              self._check_object("Task %s" % task)
    179          for folder in self.__get_folders():
    180              self._check_object("Folder %s" % folder)
    181
    182          for project in self.__get_subbaselines():
    183              self._check_object(project)
    184
    185          # checking if the purpose exists
    186          if self._config.has_key('purpose'):
    187              session = self.get_session()
    188              purposes = session.purposes()
    189              if purposes.has_key(str(self._config['purpose'])):
    190                  _logger.info("Checking purpose '%s'...Ok" % (self._config['purpose']))
    191              else:
    192                  _logger.info("Checking purpose '%s'...Not Found!" % (self._config['purpose']))
    193                  raise Exception("Could not find purpose %s in the database." % self._config['purpose'])
    194
    195              role = session.role
    196              co_role = ccm.get_role_for_purpose(session, str(self._config['purpose']))
    197              _logger.info("Try to switch user to role: %s" % co_role)
    198              session.role = co_role
    199              session.role = role
    200
    201 -    def execute(self):
    202          """ Creates a checkout of the project, or updates an existing checkout if one is found.
    203
    204          The work area is maintained as part of this.
    205          """
    206          _logger.info("=== Stage=checkout = %s" % self._config.name)
    207          _logger.info("++ Started at %s" % time.strftime("%H:%M:%S", time.localtime()))
    208          session = self.get_session()
    209          project = session.create(self._config.name)
    210
    211          session.home = self._config['dir']
    212
    213          result = self.__find_project(project)
    214          # for testing: result = session.create("ppd_sw-fa1f5132#wbernard2:project:sa1spp#1")
    215          if (result != None):
    216              _logger.info("Project found: '%s'" % result)
    217
    218              # setting up the project
    219              self.__setup_project(project, result)
    220          else:
    221              _logger.info("Checking out from '%s'." % project)
    222
    223              purpose = None
    224              if self._config.has_key('purpose'):
    225                  purpose = self._config['purpose']
    226                  _logger.info("Using purpose: '%s'" % purpose)
    227
    228              version = None
    229              if self._config.has_key('version'):
    230                  version = self._config['version']
    231                  _logger.info("Using version: '%s'" % version)
    232
    233              try:
    234                  result = project.checkout(session.create(self._config['release']), version=version, purpose=purpose)
    235                  ccm.log_result(result, ccm.CHECKOUT_LOG_RULES, _logger)
    236              except ccm.CCMException, exc:
    237                  ccm.log_result(exc.result, ccm.CHECKOUT_LOG_RULES, _logger)
    238                  raise exc
    239              _logger.info('Checkout complete')
    240
    241              if result.project != None and result.project.exists():
    242                  _logger.info("Project checked out: '%s'" % result.project)
    243
    244
    245                  _logger.info("Maintaining the workarea...")
    246                  if self.get_threads() == 1:
    247                      _logger.info(result.project.work_area(True, True, True, self._config['dir'], result.project.name))
    248                  else:
    249                      #pool = self._builder.session(self._config['database'], self.get_threads())
    250                      _logger.info(ccm.extra.FastMaintainWorkArea(result.project, self._config['dir'], result.project.name, self.get_threads()))
    251                  self.__setup_project(project, result.project)
    252              else:
    253                  raise Exception("Error checking out '%s'" % project)
    254          _logger.info("++ Finished at %s" % time.strftime("%H:%M:%S", time.localtime()))
    255
    256 -    def __find_project(self, project):
    257          """ Private method. """
    258          if (os.path.exists(os.path.join(self._config['dir'], project.name, "project.version"))):
    259              _logger.info("Snapshot to checkout deleting '%s'." % os.path.join(self._config['dir'], project.name))
    260              fileutils.rmtree(os.path.join(self._config['dir'], project.name))
    261              return None
    262
    263          path = os.path.join(self._config['dir'], project.name, project.name)
    264          try:
    265              result = project.session.get_workarea_info(path)
    266              if(result == None):
    267                  fileutils.rmtree(path)
    268                  return result
    269              return result['project']
    270          except ccm.CCMException:
    271              # Delete the project dir if found
    272              if os.path.exists(os.path.dirname(path)):
    273                  fileutils.rmtree(os.path.dirname(path))
    274              return None
    275
    276 -    def __setup_project(self, project, coproject):
    277          """ Private method. """
    278          session = self.get_session()
    279          role = session.role
    280          if self._config.has_key('purpose'):
    281              co_role = ccm.get_role_for_purpose(session, self._config['purpose'])
    282              _logger.info("Switching user to role: %s" % co_role)
    283              session.role = co_role
    284
    285          newprojs = []
    286          if not self._config.get_boolean('use.reconfigure.template', False):
    287              _logger.info("Validating release")
    288              self.__set_release(coproject)
    289              _logger.info("Setting update properties to manual")
    290              coproject.set_update_method('manual', True)
    291              _logger.info("Setting the baseline to '%s'" % project)
    292              coproject.set_baseline(project, True)
    293              self.__set_subbaselines(coproject)
    294              _logger.info("Cleaning up update properties")
    295              self._clean_update_properties(coproject)
    296              _logger.info("Setting update properties.")
    297              self._set_tasks_and_folders(coproject)
    298              _logger.info("Applying update properties.")
    299              coproject.apply_update_properties(baseline=False)
    300          else:
    301              _logger.info("Validating release")
    302              self.__set_release(coproject)
    303
    304          replace_subprojects = True
    305          if not self._config.get_boolean('replace.subprojects', True):
    306              _logger.info("NOT replacing subprojects")
    307              replace_subprojects = False
    308          update_keepgoing = True
    309          if self._config.get_boolean('update.failonerror', False):
    310              _logger.info("The build will fail with update errors")
    311              update_keepgoing = False
    312          _logger.info("Updating...")
    313          result = coproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(coproject.session))
    314
    315          if self._config.get_boolean('fix.missing.baselines', False) and replace_subprojects:
    316              newprojs = self.__fix_baseline(coproject)
    317              if len(newprojs) > 0:
    318                  result = coproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(coproject.session))
    319                  ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger)
    320                  _logger.info("Detected additional projects into baseline - Maintaining the whole toplevel project again...")
    321                  coproject.work_area(True, True)
    322              else:
    323                  ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger)
    324          else:
    325              ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger)
    326
    327          # Running sync
    328          self._sync(coproject)
    329
    330          # Running check conflicts
    331          self._check_conflicts(coproject)
    332
    333          _logger.info("Switching user to role: %s" % role)
    334          session.role = role
    335
    336 -    def _sync(self, coproject):
    337          """ Run the sync if the 'sync' property is defined to true in the
    338              configuration
    339          """
    340          if self._config.get_boolean('sync', False):
    341              _logger.info("Synchronizing...")
    342              result = coproject.sync(True, True)
    343              ccm.log_result(result, ccm.SYNC_LOG_RULES, _logger)
    344
    345
    346 -    def __set_release(self, project):
    347          """ Update the release of the project hierarchy if required. """
    348          release = project.session.create(self._config['release'])
    349          _logger.info("Current release: '%s'" % project.release)
    350          _logger.info("Configuration release: '%s'" % release)
    351          if project.release != release:
    352              _logger.info("Updating release on the project hierarchy.")
    353              for subp in [project] + project.subprojects:
    354                  subp.release = release
    355
    356 -    def __fix_baseline(self, coproject):
    357          """ Check for project in a different status, then check them out. """
    358          newprojs = []
    359          _logger.info("Looking for new projects in the check out.")
    360          status = coproject['status']
    361          for subproj in coproject.subprojects:
    362              if subproj['status'] == status:
    363                  continue
    364              _logger.info("New project detected in the checkout '%s'" % subproj.objectname)
    365              purpose = None
    366              if self._config.has_key('purpose'):
    367                  purpose = self._config['purpose']
    368                  _logger.info("Using purpose: '%s'" % purpose)
    369
    370              version = None
    371              if self._config.has_key('version'):
    372                  version = self._config['version']
    373                  _logger.info("Using version: '%s'" % version)
    374
    375              result = subproj.checkout(subproj.session.create(self._config['release']), version=version, purpose=purpose, subprojects=False)
    376              _logger.info('Checkout complete')
    377              if result.project != None and result.project.exists():
    378                  newcop = result.project
    379                  newprojs.append(newcop)
    380
    381                  _logger.info("Setting is_relative to true")
    382                  if "is_relative" in newcop.keys():
    383                      newcop["is_relative"] = "TRUE"
    384                  else:
    385                      newcop.create_attribute("is_relative", "boolean", "TRUE")
    386
    387                  if not self._config.get_boolean('use.reconfigure.template', False):
    388                      newcop.set_update_method('manual', False)
    389
    390                      _logger.info("Setting the baseline to '%s'" % subproj)
    391                      newcop.set_baseline(subproj, True)
    392
    393                      _logger.info("Cleaning up update properties")
    394                      self._clean_update_properties(newcop)
    395
    396                      _logger.info("Setting update properties.")
    397                      self._set_tasks_and_folders(newcop)
    398          return newprojs
    399
    400 -    def _check_conflicts(self, coproject):
    401          """ Private method. """
    402          if not self._config.get_boolean('show.conflicts', False):
    403              return
    404
    405          result = coproject.conflicts(True, True)
    406          ccm.log_result(result, ccm.CONFLICTS_LOG_RULES, _logger)
    407  #        for project in result.keys():
    408  #            for error in result[project]:
    409  #                if 'object' in error:
    410  #                    _logger.info("CONFLICTS: %s" % error['comment'])
    411  #                else:
    412  #                    _logger.info("CONFLICTS: %s" % error['comment'])
    413
    414      @staticmethod
    415 -    def _clean_update_properties(project):
    416          """ Private method. """
    417          for task in project.tasks:
    418              project.remove_task(task)
    419          for folder in project.folders:
    420              project.remove_folder(folder)
    421
    422      @staticmethod
    423 -    def __find_subproject(subprojects, project):
    424          """ Private method. """
    425          for subproj in subprojects:
    426              if subproj.is_same_family(project):
    427                  return subproj
    428          raise Exception("Error could not identify check out project for '%s'" % project)
    429
    430 -    def __set_subbaselines(self, project):
    431          """ Private method. """
    432          if len(self.__get_subbaselines()) > 0:
    433              subprojects = project.subprojects
    434              for subbaseline in self.__get_subbaselines():
    435                  subbaseline = project.session.create(subbaseline)
    436                  subproj = self.__find_subproject(subprojects, subbaseline)
    437                  _logger.info("Setting subproject '%s' baseline to '%s'" % (subproj, subbaseline))
    438                  subproj.set_baseline(subbaseline, True)
    439
    440 -    def __get_array(self, key):
    441          """ Private method. """
    442          result = []
    443          if (self._config.has_key(key)):
    444              if isinstance(self._config[key], type([])):
    445                  for value in self._config[key]:
    446                      value = value.strip()
    447                      if len(value) > 0:
    448                          result.append(value)
    449              else:
    450                  value = self._config[key].strip()
    451                  if len(value) > 0:
    452                      result.append(value)
    453          return result
    454
    455 -    def __get_subbaselines(self):
    456          """ Private method. """
    457          return self.__get_array('subbaselines')
    458
    459 -    def __get_tasks(self):
    460          """ Private method. """
    461          return self.__get_array('tasks')
    462
    463 -    def __get_folders(self):
    464          """ Private method. """
    465          return self.__get_array('folders')
    466
    467 -    def _set_tasks_and_folders(self, project):
    468          """ Private method. """
    469          for task in self.__get_tasks():
    470              _logger.info("Adding task %s" % task)
    471              project.add_task(project.session.create("Task %s" % task))
    472          for folder in self.__get_folders():
    473              _logger.info("Adding folder %s" % folder)
    474              project.add_folder(project.session.create("Folder %s" % folder))
    475
    476 -class PreparationUpdate(PreparationCheckout):
    477      """ Synergy project updater. """
    478
    479 -    def __init__(self, config, builder):
    480          """ Initialization. """
    481          PreparationCheckout.__init__(self, config, builder)
    482
    483 -    def check(self):
    484          """ Checks if all synergy resources are available. """
    485          PreparationAction.check(self)
    486
    487          session = self.get_session()
    488          ccm_object = session.create(self._config.name)
    489          role = session.role
    490          co_role = ccm.get_role_for_status(session, ccm_object['status'])
    491          _logger.info("Try to switch user to role: %s" % co_role)
    492          session.role = co_role
    493          session.role = role
    494
    495 -    def execute(self):
    496          """ Updating the mentioned project. """
    497
    498          session = self.get_session()
    499          ccmproject = session.create(self._config.name)
    500          role = session.role
    501
    502          status = ccmproject['status']
    503          co_role = ccm.get_role_for_status(session, status)
    504          session.role = co_role
    505
    506          if not self._config.get_boolean('use.reconfigure.template', False):
    507              _logger.info("Setting update properties to manual")
    508              ccmproject.set_update_method('manual', True)
    509              _logger.info("Cleaning up update properties")
    510              self._clean_update_properties(ccmproject)
    511              _logger.info("Setting update properties.")
    512              self._set_tasks_and_folders(ccmproject)
    513              _logger.info("Applying update properties.")
    514              ccmproject.apply_update_properties(baseline=False)
    515          replace_subprojects = True
    516          if not self._config.get_boolean('replace.subprojects', True):
    517              _logger.info("NOT replacing subprojects")
    518              replace_subprojects = False
    519          update_keepgoing = True
    520          if self._config.get_boolean('update.failonerror', False):
    521              _logger.info("The build will fail with update errors")
    522              update_keepgoing = False
    523
    524          _logger.info("Updating %s..." % ccmproject.objectname)
    525          result = ccmproject.update(True, replace_subprojects, update_keepgoing, result=ccm.UpdateResultSimple(ccmproject.session))
    526          ccm.log_result(result, ccm.UPDATE_LOG_RULES, _logger)
    527
    528          self._sync(ccmproject)
    529
    530          self._check_conflicts(ccmproject)
    531
    532          session.role = role
    533
    534 -class PreparationBuilder(object):
    535      """ Creates an updated work area from a configuration. """
    536 -    def __init__(self, configs, username = None, password = None, cache=None):
    537          """ Initialization. """
    538          self._configs = configs
    539          self._sessions = {}
    540          self._actions = []
    541          self.__username = username
    542          self.__password = password
    543          self.__provider = ccm.extra.CachedSessionProvider(cache=cache)
    544          for config in self._configs:
    545              if config.type == "snapshot":
    546                  self._actions.append(PreparationSnapshot(config, self))
    547              elif config.type == "checkout":
    548                  self._actions.append(PreparationCheckout(config, self))
    549              elif config.type == "update":
    550                  self._actions.append(PreparationUpdate(config, self))
    551
    552 -    def check(self):
    553          """ Check that all dependencies are there. """
    554          for action in self._actions:
    555              action.check()
    556
    557 -    def get_content(self):
    558          """ Run the each action. """
    559          for action in self._actions:
    560              action.execute()
    561
    562 -    def session(self, database, size=1, engine=None, dbpath=None):
    563          """ Handles pool rather that sessions. """
    564          assert size > 0, "The pool must contains at least one session!"
    565          if self.__provider is None:
    566              raise Exception("The builder has been closed.")
    567          if not self._sessions.has_key(database):
    568              _logger.info("Get a session for %s" % database)
    569              session = ccm.SessionPool(self.__username, self.__password, engine, dbpath, database, size, opener=self.__provider.get)
    570              self._sessions[database] = session
    571              # be developer by default
    572              session.role = "developer"
    573          session = self._sessions[database]
    574          if session.size < size:
    575              _logger.info("Resizing the pool for database %s to %d" % (database, size))
    576              session.size = size
    577              # be developer by default
    578              session.role = "developer"
    579          return session
    580
    581 -    def close(self):
    582          """ This is the preparation cleanup method.
    583              It closes all opened sessions.
    584          """
    585          _logger.debug("Closing sessions...")
    586          dbs = self._sessions.keys()
    587          while len(dbs) > 0:
    588              session = self._sessions.pop(dbs.pop())
    589              session.close()
    590          self.__provider.close()
    591          self.__provider = None
    592
    593
    594 -    def __del__(self):
    595          self.close()
    596