Package ccm :: Module extra
[hide private]
[frames] | no frames]

Source Code for Module ccm.extra

  1  #============================================================================  
  2  #Name        : extra.py  
  3  #Part of     : Helium  
  4   
  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
  6  #All rights reserved. 
  7  #This component and the accompanying materials are made available 
  8  #under the terms of the License "Eclipse Public License v1.0" 
  9  #which accompanies this distribution, and is available 
 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
 11  # 
 12  #Initial Contributors: 
 13  #Nokia Corporation - initial contribution. 
 14  # 
 15  #Contributors: 
 16  # 
 17  #Description: 
 18  #=============================================================================== 
 19   
 20  """ Library that contains custom Synergy functionnlities: e.g 
 21          * Snapshotter that can snapshot unfrozen baselines 
 22          * Threaded snapshotter. 
 23  """ 
 24  import ccm 
 25  import os 
 26  import shutil 
 27  import threading 
 28  import threadpool 
 29  import traceback 
 30  import sys 
 31  import logging 
 32  import xml.dom.minidom 
 33  from xml.dom.minidom import getDOMImplementation, parse 
 34   
 35  # Uncomment this line to enable logging in this module, or configure logging elsewhere 
 36  #logging.basicConfig(level=logging.DEBUG) 
 37  _logger = logging.getLogger('ccm.extra') 
 38   
39 -class CCMExtraException(ccm.CCMException):
40 """ Exception raised by the methods of this module. """
41 - def __init__(self, description, subexceptions):
42 ccm.CCMException.__init__(self, description) 43 self.subexceptions = subexceptions
44 45 46
47 -def Snapshot(project, targetdir, dir=None):
48 """ This function can snapshot anything from Synergy, even prep/working projects """ 49 assert (project != None, "a project object must be supplied") 50 assert (project.type == "project", "project must be of project type") 51 if not dir: 52 dir = project.root_dir() 53 targetdir = os.path.join(targetdir, dir.name) 54 os.makedirs(targetdir) 55 for object in dir.children(project): 56 if object.type == 'dir': 57 Snapshot(project, targetdir, object) 58 elif object.type == 'project': 59 Snapshot(object, targetdir) 60 else: 61 object.to_file(os.path.join(targetdir, object.name))
62
63 -class __FileSnapshot:
64 - def __init__(self, object, targetdir):
65 self.object = object 66 self.targetdir = targetdir
67 - def __call__(self):
68 _logger.info("Getting %s (%s)" % (os.path.join(self.targetdir, self.object.name), self.object)) 69 self.object.to_file(os.path.join(self.targetdir, self.object.name)) 70 return "Getting %s (%s)" % (os.path.join(self.targetdir, self.object.name), self.object)
71
72 -class __ProjectSnapshot:
73 - def __init__(self, project, targetdir):
74 self.project = project 75 self.targetdir = targetdir
76 - def __call__(self):
77 _logger.info("Snapshotting '%s' under '%s'" % (self.project, self.targetdir)) 78 status = self.project['status'] 79 if status == 'released' or status == 'sqa' or status == 'test' or status == 'integrate': 80 _logger.info("Using Synergy wa_snapshot") 81 return self.project.snapshot(self.targetdir, True) 82 else: 83 _logger.info("Non static project, using custom snapshot.") 84 Snapshot(self.project, self.targetdir) 85 return ""
86
87 -def __FastSnapshot(pool, project, targetdir, callback, exc_hld, dir=None):
88 if not dir: 89 dir = project.root_dir() 90 targetdir = os.path.join(targetdir, dir.name) 91 os.makedirs(targetdir) 92 for object in dir.children(project): 93 if isinstance(object, ccm.Dir): 94 __FastSnapshot(pool, project, targetdir, callback, exc_hld, object) 95 elif isinstance(object, ccm.Project): 96 pool.addWork(__ProjectSnapshot(object, targetdir), callback=callback, exc_callback=exc_hld) 97 else: 98 _logger.info("Getting %s (%s)" % (os.path.join(targetdir, object.name), object)) 99 object.to_file(os.path.join(targetdir, object.name))
100 # Can't snapshot files in parallel. 101 #pool.addWork(__FileSnapshot(object, targetdir), callback=callback, exc_callback=exc_hld) 102
103 -def FastSnapshot(project, targetdir, threads=4):
104 """ Create snapshot running by running sbsnapshot concurently. """ 105 assert (threads > 0, "Number of threads must be > 0.") 106 assert (project != None, "a project object must be supplied.") 107 assert (project.type == "project", "project must be of project type.") 108 109 # error handling 110 exceptions = [] 111 results = [] 112 def handle_exception(request, exc_info): 113 _logger.error( "Exception occured in request #%s: %s" % (request.requestID, exc_info[1])) 114 exceptions.append(exc_info[1])
115 116 def handle_result(request, result): 117 results.append(result) 118 119 pool = threadpool.ThreadPool(threads) 120 __FastSnapshot(pool, project, targetdir, handle_result, handle_exception) 121 pool.wait() 122 123 if len(exceptions): 124 raise CCMExtraException("Errors occured during snapshot.", exceptions) 125 126 return "\n".join(results) 127 128 129
130 -def FastMaintainWorkArea(project, path, pst=None, threads=4, wat=False):
131 """ Maintain the workarea of a project in parallel. """ 132 assert (threads > 0, "Number of threads must be > 0.") 133 assert (isinstance(project, ccm.Project), "a valid project object must be supplied.") 134 135 # error handling 136 exceptions = [] 137 results = [] 138 def handle_exception(request, exc_info): 139 _logger.error( "Exception occured in request #%s: %s\n%s" % (request.requestID, exc_info[1], traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]))) 140 exceptions.append(exc_info[1])
141 142 def handle_result(request, result): 143 results.append(result) 144 145 class __MaintainProject: 146 def __init__(self, subproject, toplevel, wat=False): 147 self.subproject = subproject 148 self.toplevel = toplevel 149 self.wat = wat 150 151 def __call__(self): 152 output = "" 153 _logger.info("Maintaining project %s" % self.subproject) 154 for tuple in self.subproject.finduse(): 155 if tuple['project'] == self.toplevel: 156 self.subproject['wa_path'] = os.path.join(self.toplevel['wa_path'], tuple['path']) 157 self.subproject["project_subdir_template"] = "" 158 _logger.info("Maintaining project %s under %s" % (self.subproject, self.subproject['wa_path'])) 159 output = self.subproject.work_area(True, True, True, wat=self.wat) 160 _logger.info("Project %s maintained" % self.subproject) 161 return output 162 163 pool = threadpool.ThreadPool(threads) 164 project.work_area(True, False, True, path, pst, wat=wat) 165 for subproject in project.get_members(type="project"): 166 _logger.info("Adding project %s" % subproject) 167 pool.addWork(__MaintainProject(subproject, project, wat), callback=handle_result, exc_callback=handle_exception) 168 pool.wait() 169 170 if len(exceptions) > 0: 171 raise CCMExtraException("Errors occured during work area maintenance.", exceptions) 172 173 return "\n".join(results) 174 175 176
177 -def get_toplevel_project(session, path):
178 try: 179 wainfo = session.get_workarea_info(path) 180 project = get_toplevel_project(session, os.path.dirname(wainfo['path'])) 181 if project == None: 182 project = wainfo['project'] 183 return project 184 except ccm.CCMException, e: 185 return None
186 187
188 -class SessionProvider:
189 - def __init__(self, opener=None):
190 self._opener = opener 191 if self._opener is None: 192 self._opener = ccm.open_session
193
194 - def get(self, username=None, password=None, engine=None, dbpath=None, database=None, reuse=True):
195 _logger.debug("SessionProvider: Creating a new session.") 196 return self._opener(username, password, engine, dbpath, database, reuse)
197
198 - def __del__(self):
199 _logger.info("Deleting the session provider.") 200 self.close()
201
202 - def close(self):
203 pass
204 205
206 -class CachedSessionProvider(SessionProvider):
207 """ 208 <sessions> 209 <session database="foobar" ccmaddr="xxxx"/> 210 <session database="foobarx" ccmaddr="xxxx"/> 211 </sessions> 212 """ 213
214 - def __init__(self, opener=None, cache=None):
215 """ Creates CachedSessionProvider, with a specific 216 opener and cache file. 217 """ 218 SessionProvider.__init__(self, opener=opener) 219 _logger.info("Using CachedSessionProvider.") 220 self.__closed = False 221 self._lock = threading.Lock() 222 self.cacheXml = cache 223 self.cacheFree = {} 224 self.cacheUsed = [] 225 self.load()
226 227
228 - def close(self):
229 """ Closing the SessionProvider. """ 230 _logger.info("Closing the CachedSessionProvider.") 231 self.save() 232 if self.cacheXml == None: 233 _logger.info("Cleaning up opened sessions.") 234 self._lock.acquire() 235 for dbname in self.cacheFree.keys(): 236 while len(self.cacheFree[dbname]) > 0: 237 session = self.cacheFree[dbname].pop() 238 session.close_on_exit = True 239 session.close() 240 while len(self.cacheUsed) > 0: 241 session = self.cacheUsed.pop() 242 session.close_on_exit = True 243 self._lock.release() 244 self.__closed = True
245
246 - def save(self):
247 if self.cacheXml is not None and not self.__closed: 248 _logger.info("Writing %s" % self.cacheXml) 249 impl = getDOMImplementation() 250 sessions = impl.createDocument(None, "sessions", None) 251 top_element = sessions.documentElement 252 self._lock.acquire() 253 def add_session(dbname, session): 254 sessionNode = sessions.createElement("session") 255 sessionNode.setAttribute("database", dbname) 256 sessionNode.setAttribute("ccmaddr", session.addr()) 257 top_element.appendChild(sessionNode)
258 for dbname in self.cacheFree.keys(): 259 for session in self.cacheFree[dbname]: 260 add_session(dbname, session) 261 for session in self.cacheUsed: 262 add_session(session.database(), session) 263 self._lock.release() 264 o = open(self.cacheXml, "w+") 265 o.write(sessions.toprettyxml()) 266 o.close() 267 _logger.debug(sessions.toprettyxml())
268 269
270 - def load(self):
271 if self.cacheXml is not None and os.path.exists(self.cacheXml): 272 _logger.info("Loading %s" % self.cacheXml) 273 doc = parse(open(self.cacheXml, 'r')) 274 sessions = doc.documentElement 275 self._lock.acquire() 276 try: 277 for child in sessions.childNodes: 278 if child.nodeType == child.ELEMENT_NODE and child.tagName == "session" and child.hasAttribute('database') and child.hasAttribute('ccmaddr'): 279 if child.getAttribute('database') not in self.cacheFree: 280 self.cacheFree[child.getAttribute('database')] = [] 281 if ccm.session_exists(child.getAttribute('ccmaddr'), child.getAttribute('database')): 282 _logger.info(" + Session: database=%s, ccmaddr=%s" % (child.getAttribute('database'), child.getAttribute('ccmaddr'))) 283 self.cacheFree[child.getAttribute('database')].append(ccm.Session(None, None, None, ccm_addr=child.getAttribute('ccmaddr'), close_on_exit=False)) 284 else: 285 _logger.info(" - Session database=%s, ccmaddr=%s doesn't seem to be valid anymore." % (child.getAttribute('database'), child.getAttribute('ccmaddr'))) 286 finally: 287 self._lock.release()
288 289
290 - def get(self, username=None, password=None, engine=None, dbpath=None, database=None, reuse=True):
291 if self.__closed: 292 raise Exception("Could not create further session the provider is closed.") 293 _logger.debug("CachedSessionProvider: Getting a session.") 294 if database is not None and database in self.cacheFree and len(self.cacheFree[database]) > 0: 295 _logger.info("CachedSessionProvider: Reusing session.") 296 self._lock.acquire() 297 s = self.cacheFree[database].pop() 298 self.cacheUsed.append(s) 299 self._lock.release() 300 return CachedProxySession(self, s) 301 else: 302 _logger.debug("CachedSessionProvider: Creating new session.") 303 session = SessionProvider.get(self, username, password, engine, dbpath, database, False) 304 session.close_on_exit = False 305 s = CachedProxySession(self, session) 306 db = s.database() 307 self._lock.acquire() 308 if db not in self.cacheFree: 309 self.cacheFree[db] = [] 310 self.cacheUsed.append(session) 311 self._lock.release() 312 return s
313
314 - def free(self, session):
315 _logger.debug("CachedSessionProvider: Freeing session: %s" % session) 316 db = session.database() 317 if session in self.cacheUsed: 318 _logger.debug("CachedSessionProvider: Removing session from used list.") 319 self._lock.acquire() 320 self.cacheUsed.remove(session) 321 self.cacheFree[db].append(session) 322 self._lock.release()
323
324 -class CachedProxySession:
325 """ Proxy session which will cleanup the session and free it from the provider """ 326
327 - def __init__(self, provider, session):
328 """ Constructor. """ 329 self.__session = session 330 self.__provider = provider
331
332 - def __getattr__(self, attrib):
333 """ Delegate attributes to the session object. """ 334 _logger.debug("CachedProxySession.__getattr__(%s)" % attrib) 335 if attrib == "close": 336 return self.__close 337 return getattr(self.__session, attrib)
338
339 - def __close(self):
340 """ Overriding the session closing. """ 341 _logger.debug("CachedProxySession.__close") 342 self.__provider.free(self.__session) 343 self.__session.close()
344
345 - def __del__(self):
346 """ Free the session on destruction. """ 347 _logger.debug("CachedProxySession.__del__") 348 self.__close()
349