1 #============================================================================ 

     2 #Name        : extra.py 

     3 #Part of     : Helium 

     4 

     5 #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).

     6 #All rights reserved.

     7 #This component and the accompanying materials are made available

     8 #under the terms of the License "Eclipse Public License v1.0"

     9 #which accompanies this distribution, and is available

    10 #at the URL "http://www.eclipse.org/legal/epl-v10.html".

    11 #

    12 #Initial Contributors:

    13 #Nokia Corporation - initial contribution.

    14 #

    15 #Contributors:

    16 #

    17 #Description:

    18 #===============================================================================

    19 

    20 """ Library that contains custom Synergy functionnlities: e.g

    21         * Snapshotter that can snapshot unfrozen baselines

    22         * Threaded snapshotter.

    23 """

    24 import ccm

    25 import os

    26 import threading

    27 import threadpool

    28 import traceback

    29 import logging

    30 from xml.dom.minidom import getDOMImplementation, parse

    31 import StringIO     #pylint throws this up as unused but it is required by delete call in the code 

    32                     #so do not remove unless removeing the delete (which is required at some point).

    33 

    34 # Uncomment this line to enable logging in this module, or configure logging elsewhere

    35 #logging.basicConfig(level=logging.DEBUG)

    36 _logger = logging.getLogger('ccm.extra')

    37 

    38 class CCMExtraException(ccm.CCMException):

    39     """ Exception raised by the methods of this module. """

    40     def __init__(self, description, subexceptions):

    41         ccm.CCMException.__init__(self, description)

    42         self.subexceptions = subexceptions

    43     

    44     

    45 

    46 def Snapshot(project, targetdir, dir=None):

    47     """ This function can snapshot anything from Synergy, even prep/working projects """

    48     assert project != None, "a project object must be supplied"

    49     assert project.type == "project", "project must be of project type"

    50     if not dir:

    51         dir = project.root_dir()

    52     targetdir = os.path.join(targetdir, dir.name)

    53     os.makedirs(targetdir)

    54     for object in dir.children(project):

    55         if object.type == 'dir':

    56             Snapshot(project, targetdir, object)

    57         elif object.type == 'project':

    58             Snapshot(object, targetdir)

    59         else:

    60             object.to_file(os.path.join(targetdir, object.name))

    61 

    62 

    63 class _FastSnapshot:

    64     """ Snapshot Job executed by the thread pool. """

    65     def __init__(self, pool, project, targetdir, callback, exc_hld):

    66         """ Construtor, will store the parameter for the checkout. """

    67         self.pool = pool

    68         self.project = project

    69         self.targetdir = targetdir

    70         self.callback = callback

    71         self.exc_hld = exc_hld

    72 

    73     def __call__(self):

    74         """ Do the checkout, and then walkthrough the project hierarchy to find subproject to snapshot. """

    75         _logger.info("Snapshotting %s under %s" % (self.project, self.targetdir))

    76         self.project.snapshot(self.targetdir, False)

    77         def walk(dir, targetdir):

    78             """walkthrough the project hierarchy to find subproject to snapshot"""

    79             for object in dir.children(self.project):

    80                 if isinstance(object, ccm.Dir):

    81                     walk(object, os.path.join(targetdir, object.name))

    82                 elif isinstance(object, ccm.Project):

    83                     _logger.info("Adding project %s" % object.objectname)

    84                     self.pool.addWork(_FastSnapshot(self.pool, object, targetdir, self.callback, self.exc_hld))

    85                     

    86         if len(self.project.subprojects) > 0:

    87             rootdir = self.project.root_dir()

    88             walk(rootdir, os.path.join(self.targetdir, rootdir.name))

    89         return ""

    90 

    91 def FastSnapshot(project, targetdir, threads=4):

    92     """ Create snapshot running by running snapshots concurrently.

    93         Snapshot will be made recursively top-down, and each sub project will

    94         be snapshotted in parallel. 

    95     """

    96     assert threads > 0, "Number of threads must be > 0."

    97     assert project != None, "a project object must be supplied."

    98     assert project.type == "project", "project must be of project type."

    99     

   100     # error handling

   101     exceptions = []

   102     results = []

   103     def handle_exception(request, exc_info):

   104         """ append the exceptions"""

   105         _logger.error( "Exception occurred in request #%s: %s" % (request.requestID, exc_info[1]))

   106         exceptions.append(exc_info[1])

   107 

   108     def handle_result(result):

   109         """ append the result"""

   110         results.append(result)

   111    

   112     pool = threadpool.ThreadPool(threads)

   113     pool.addWork(_FastSnapshot(pool, project, targetdir, handle_result, handle_exception))

   114     pool.wait()

   115 

   116     if len(exceptions):

   117         raise CCMExtraException("Errors occurred during snapshot.", exceptions)

   118 

   119     return "\n".join(results)

   120 

   121 

   122 

   123 def FastMaintainWorkArea(project, path, pst=None, threads=4, wat=False):

   124     """ Maintain the workarea of a project in parallel. """

   125     assert threads > 0, "Number of threads must be > 0."

   126     assert isinstance(project, ccm.Project), "a valid project object must be supplied."

   127             

   128     # error handling

   129     exceptions = []

   130     results = []

   131     def handle_exception(request, exc_info):

   132         """append the exception"""

   133         _logger.error( "Exception occured in request #%s: %s\n%s" % (request.requestID, exc_info[1], traceback.format_exception(exc_info[0], exc_info[1], exc_info[2])))

   134         exceptions.append(exc_info[1])

   135     

   136     def handle_result(result):

   137         """append  the result"""

   138         results.append(result)

   139 

   140     class __MaintainProject:

   141         """_Maintain Project"""

   142         def __init__(self, subproject, toplevel, wat=False):

   143             self.subproject = subproject

   144             self.toplevel = toplevel

   145             self.wat = wat

   146         

   147         def __call__(self):

   148             output = ""

   149             _logger.info("Maintaining project %s" % self.subproject)

   150             for tuple in self.subproject.finduse():

   151                 if tuple['project'] == self.toplevel:

   152                     self.subproject['wa_path'] = os.path.join(self.toplevel['wa_path'], tuple['path'])

   153                     self.subproject["project_subdir_template"] = ""

   154                     _logger.info("Maintaining project %s under %s" % (self.subproject, self.subproject['wa_path']))

   155                     output = self.subproject.work_area(True, True, True, wat=self.wat)

   156             _logger.info("Project %s maintained" % self.subproject)

   157             return output

   158             

   159     pool = threadpool.ThreadPool(threads)

   160     project.work_area(True, False, True, path, pst, wat=wat)

   161     for subproject in project.get_members(type="project"):

   162         _logger.info("Adding project %s" % subproject)

   163         pool.addWork(__MaintainProject(subproject, project, wat), callback=handle_result, exc_callback=handle_exception)

   164     pool.wait()

   165     

   166     if len(exceptions) > 0:

   167         raise CCMExtraException("Errors occured during work area maintenance.", exceptions)

   168     

   169     return "\n".join(results)

   170 

   171 

   172 def get_toplevel_project(session, path):

   173     """get the top level project from CCM or return None"""

   174     try:

   175         wainfo = session.get_workarea_info(path)

   176         project = get_toplevel_project(session, os.path.dirname(wainfo['path']))

   177         if project == None:

   178             project = wainfo['project']

   179         return project

   180     except ccm.CCMException:

   181         return None

   182 

   183 

   184 class SessionProvider:

   185     """ A class which provides an open user session """

   186     def __init__(self, opener=None):

   187         """initialisation"""

   188         self._opener = opener

   189         if self._opener is None:

   190             self._opener = ccm.open_session

   191         

   192     def get(self, username=None, password=None, engine=None, dbpath=None, database=None, reuse=True):

   193         """return the paramaters required to open a synergy session"""

   194         _logger.debug("SessionProvider: Creating a new session.")

   195         return self._opener(username, password, engine, dbpath, database, reuse)

   196 

   197     def __del__(self):

   198         """delete the CCM session"""

   199         _logger.info("Deleting the session provider.")

   200         self.close()

   201 

   202     def close(self):

   203         """close the session which actually does nothing"""

   204         pass

   205         

   206         

   207 class CachedSessionProvider(SessionProvider):

   208     """

   209 <sessions>

   210     <session database="foobar" ccmaddr="xxxx"/>

   211     <session database="foobarx" ccmaddr="xxxx"/>

   212 </sessions>

   213     """

   214 

   215     def __init__(self, opener=None, cache=None):

   216         """ Creates CachedSessionProvider, with a specific 

   217             opener and cache file.

   218         """

   219         SessionProvider.__init__(self, opener=opener)

   220         _logger.info("Using CachedSessionProvider.")

   221         self.__closed = False

   222         self._lock = threading.Lock()

   223         self.cacheXml = cache

   224         self.cacheFree = {}

   225         self.cacheUsed = []

   226         self.load()

   227 

   228     

   229     def close(self):

   230         """ Closing the SessionProvider. """

   231         _logger.info("Closing the CachedSessionProvider.")

   232         self.save()

   233         if self.cacheXml == None:

   234             _logger.info("Cleaning up opened sessions.")

   235             self._lock.acquire()

   236             for dbname in self.cacheFree.keys():

   237                 while len(self.cacheFree[dbname]) > 0:

   238                     session = self.cacheFree[dbname].pop()

   239                     session.close_on_exit = True

   240                     session.close()

   241             while len(self.cacheUsed) > 0:

   242                 session = self.cacheUsed.pop()

   243                 session.close_on_exit = True

   244             self._lock.release()

   245         self.__closed = True

   246     

   247     def save(self):

   248         """ save the sessionProvider"""

   249         if self.cacheXml is not None and not self.__closed:

   250             _logger.info("Writing %s" % self.cacheXml)

   251             impl = getDOMImplementation()

   252             sessions = impl.createDocument(None, "sessions", None)

   253             top_element = sessions.documentElement

   254             self._lock.acquire()

   255             def add_session(dbname, session):

   256                 """add session"""

   257                 sessionNode = sessions.createElement("session")

   258                 sessionNode.setAttribute("database", dbname)

   259                 sessionNode.setAttribute("ccmaddr", session.addr())

   260                 top_element.appendChild(sessionNode)

   261             for dbname in self.cacheFree.keys():

   262                 for session in self.cacheFree[dbname]:

   263                     add_session(dbname, session)

   264             for session in self.cacheUsed:

   265                 add_session(session.database(), session)

   266             self._lock.release()

   267             open_f = open(self.cacheXml, "w+")

   268             open_f.write(sessions.toprettyxml())

   269             open_f.close()

   270             _logger.debug(sessions.toprettyxml())

   271             

   272     

   273     def load(self):

   274         """load the command"""

   275         if self.cacheXml is not None and os.path.exists(self.cacheXml):

   276             _logger.info("Loading %s" % self.cacheXml)

   277             doc = parse(open(self.cacheXml, 'r')) 

   278             sessions = doc.documentElement

   279             self._lock.acquire()

   280             try:

   281                 for child in sessions.childNodes:

   282                     if child.nodeType == child.ELEMENT_NODE and child.tagName == "session" and child.hasAttribute('database') and child.hasAttribute('ccmaddr'):

   283                         if child.getAttribute('database') not in self.cacheFree:

   284                             self.cacheFree[child.getAttribute('database')] = []

   285                         if ccm.session_exists(child.getAttribute('ccmaddr'), child.getAttribute('database')):

   286                             _logger.info(" + Session: database=%s, ccmaddr=%s" % (child.getAttribute('database'), child.getAttribute('ccmaddr')))

   287                             self.cacheFree[child.getAttribute('database')].append(ccm.Session(None, None, None, ccm_addr=child.getAttribute('ccmaddr'), close_on_exit=False))

   288                         else:

   289                             _logger.info(" - Session database=%s, ccmaddr=%s doesn't seem to be valid anymore." % (child.getAttribute('database'), child.getAttribute('ccmaddr')))

   290             finally:

   291                 self._lock.release()

   292 

   293 

   294     def get(self, username=None, password=None, engine=None, dbpath=None, database=None, reuse=True):

   295         """create a CCM session"""

   296         if self.__closed:

   297             raise Exception("Could not create further session the provider is closed.")

   298         _logger.debug("CachedSessionProvider: Getting a session.")

   299         if database is not None and database in self.cacheFree and len(self.cacheFree[database]) > 0:

   300             _logger.info("CachedSessionProvider: Reusing session.")

   301             self._lock.acquire()

   302             session_free = self.cacheFree[database].pop()

   303             self.cacheUsed.append(session_free)

   304             self._lock.release()

   305             return CachedProxySession(self, session_free) 

   306         else:

   307             _logger.debug("CachedSessionProvider: Creating new session.")

   308             session = SessionProvider.get(self, username, password, engine, dbpath, database, False)

   309             session.close_on_exit = False

   310             proxy_session = CachedProxySession(self, session)

   311             data_base = proxy_session.database()

   312             self._lock.acquire()

   313             if data_base not in self.cacheFree:

   314                 self.cacheFree[data_base] = []

   315             self.cacheUsed.append(session)

   316             self._lock.release()

   317             return proxy_session

   318 

   319     def free(self, session):

   320         """freeup a CCM session"""

   321         _logger.debug("CachedSessionProvider: Freeing session: %s" % session)

   322         data_base = session.database()

   323         if session in self.cacheUsed:

   324             _logger.debug("CachedSessionProvider: Removing session from used list.")

   325             self._lock.acquire()

   326             self.cacheUsed.remove(session)

   327             self.cacheFree[data_base].append(session)

   328             self._lock.release()

   329 

   330 class CachedProxySession:

   331     """ Proxy session which will cleanup the session and free it from the provider """

   332     

   333     def __init__(self, provider, session):

   334         """ Constructor. """

   335         self.__session = session 

   336         self.__provider = provider

   337     

   338     def __getattr__(self, attrib):

   339         """ Delegate attributes to the session object. """

   340         _logger.debug("CachedProxySession.__getattr__(%s)" % attrib)

   341         if attrib == "close":

   342             return self.__close

   343         return getattr(self.__session, attrib)

   344 

   345     def __close(self):

   346         """ Overriding the session closing. """

   347         _logger.debug("CachedProxySession.__close")

   348         self.__provider.free(self.__session)

   349         self.__session.close()

   350         

   351     def __del__(self):

   352         """ Free the session on destruction. """

   353         _logger.debug("CachedProxySession.__del__")

   354         self.__close()