symrec

+ 1 #============================================================================ + 2 #Name : symrec.py + 3 #Part of : Helium + 4 + 5 #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). + 6 #All rights reserved. + 7 #This component and the accompanying materials are made available + 8 #under the terms of the License "Eclipse Public License v1.0" + 9 #which accompanies this distribution, and is available + 10 #at the URL "http://www.eclipse.org/legal/epl-v10.html". + 11 # + 12 #Initial Contributors: + 13 #Nokia Corporation - initial contribution. + 14 # + 15 #Contributors: + 16 # + 17 #Description: + 18 #=============================================================================== + 19 + 20 """ SYMREC metadata file generation. """ + 21 import xml.dom.minidom + 22 import amara + 23 import codecs + 24 import os + 25 import re + 26 import logging + 27 import fileutils + 28 import csv + 29 + 30 LOGGER = logging.getLogger("symrec") + 31 +

32 -def _cleanup_list(input): +

33 result = [] + 34 for chars in input: + 35 if chars is not None and chars.strip() != "": + 36 result.append(chars) + 37 return result +

38 +

39 -def xml_setattr(node, attr, value): +

40 """ Create the attribute if needed. """ + 41 if hasattr(node, attr): + 42 setattr(node, unicode(attr), unicode(value)) + 43 else: + 44 node.xml_set_attribute(unicode(attr), unicode(value)) +

45 +

46 -class ServicePack(object): +

47 +

48 - def __init__(self, node): +

49 self.__xml = node +

50 + 51 @property +

52 - def name(self): +

53 return self.__xml.name +

54 + 55 @property +

56 - def files(self): +

57 result = [] + 58 if hasattr(self.__xml, 'file'): + 59 for filen in self.__xml.file: + 60 result.append(filen.name) + 61 return result +

62 + 63 @property +

64 - def instructions(self): +

65 result = [] + 66 if hasattr(self.__xml, 'instructions'): + 67 for instr in self.__xml.instructions: + 68 result.append(instr.name) + 69 return result +

70 +

71 -class ReleaseMetadata(object): +

72 """ Create or read Metadata XML from SYMREC/SYMDEC. """ + 73 +

74 - def __init__(self, filename, service=None, product=None, release=None): +

75 self._filename = filename + 76 if os.path.exists(filename): + 77 self._xml = amara.parse(open(filename, "r")) + 78 if service != None: + 79 self.service = service + 80 if product != None: + 81 self.product = product + 82 if release != None: + 83 self.release = release + 84 elif service!=None and product!=None and release!=None: + 85 self._xml = amara.create_document(u"releaseInformation") + 86 + 87 # releaseDetails + 88 self._xml.releaseInformation.xml_append(self._xml.xml_create_element(u'releaseDetails')) + 89 + 90 # releaseID + 91 self._xml.releaseInformation.releaseDetails.xml_append(self._xml.xml_create_element(u'releaseID')) + 92 + 93 # service + 94 serv = self._xml.xml_create_element(u'service') + 95 xml_setattr(serv, 'name', unicode(service)) + 96 self._xml.releaseInformation.releaseDetails.releaseID.xml_append(serv) + 97 # product + 98 prod = self._xml.xml_create_element(u'product') + 99 xml_setattr(prod, 'name', unicode(product)) +100 self._xml.releaseInformation.releaseDetails.releaseID.xml_append(prod) +101 # release +102 rel = self._xml.xml_create_element(u'release') +103 xml_setattr(rel, 'name', unicode(release)) +104 self._xml.releaseInformation.releaseDetails.releaseID.xml_append(rel) +105 +106 # releaseFiles +107 self._xml.releaseInformation.xml_append(self._xml.xml_create_element(u'releaseFiles')) +108 +109 # releaseFiles +110 self._xml.releaseInformation.xml_append(self._xml.xml_create_element(u'externalFiles')) +111 else: +112 raise Exception("Error metadata file doesn't exists.") +

113 +114 +

115 - def get_dependsof(self): +

116 """ Return a ReleaseMetada object pointing to the dependency release. """ +117 if self.dependsof_service != None and self.dependsof_product != None and self.dependsof_release != None: +118 filename = os.path.join(os.path.dirname(self._filename), "../../..", +119 self.dependsof_service, +120 self.dependsof_product, +121 self.dependsof_release) +122 return ReleaseMetadata(find_latest_metadata(filename)) +123 else: +124 return None +

125 +126 +

127 - def set_dependsof(self, filename): +

128 """ Setting the dependency release. """ +129 metadata = ReleaseMetadata(filename) +130 self.dependsof_service = metadata.service +131 self.dependsof_product = metadata.product +132 self.dependsof_release = metadata.release +

133 +

134 - def add_package(self, name, type=None, default=True, filters=None, extract="single", md5checksum=None, size=None): +

135 """ Adding a package to the metadata file. """ +136 # check if update mode +137 package = None +138 if hasattr(self._xml.releaseInformation.releaseFiles, 'package'): +139 for pkg in self._xml.releaseInformation.releaseFiles.package: +140 if (pkg.name.lower() == os.path.basename(name).lower()): +141 package = pkg +142 break +143 +144 # if not found create new package. +145 if package is None: +146 package = self._xml.xml_create_element(u'package') +147 self._xml.releaseInformation.releaseFiles.xml_append(package) +148 +149 xml_setattr(package, 'name', os.path.basename(name)) +150 if type != None: +151 xml_setattr(package, 'type', type) +152 else: +153 xml_setattr(package, 'type', os.path.splitext(name)[1].lstrip('.')) +154 xml_setattr(package, 'default', str(default).lower()) +155 xml_setattr(package, 'extract', extract) +156 if filters and len(filters)>0: +157 xml_setattr(package, 'filters', ','.join(filters)) +158 xml_setattr(package, 's60filter', ','.join(filters)) +159 else: +160 xml_setattr(package, 'filters', '') +161 xml_setattr(package, 's60filter', '') +162 if md5checksum != None: +163 xml_setattr(package, unicode("md5checksum"), unicode(md5checksum)) +164 if size != None: +165 xml_setattr(package, unicode("size"), unicode(size)) +

166 +167 +

168 - def keys(self): +

169 keys = [] +170 if hasattr(self._xml, 'releaseInformation') and hasattr(self._xml.releaseInformation, 'releaseFiles') \ +171 and hasattr(self._xml.releaseInformation.releaseFiles, 'package'): +172 for pkg in self._xml.releaseInformation.releaseFiles.package: +173 keys.append(pkg.name) +174 return keys +

175 +

176 - def __getitem__(self, key): +

177 if not (hasattr(self._xml, 'releaseInformation') and hasattr(self._xml.releaseInformation, 'releaseFiles') \ +178 and hasattr(self._xml.releaseInformation.releaseFiles, 'package')): +179 raise Exception("Key '%s' not found." % key) +180 for pkg in self._xml.releaseInformation.releaseFiles.package: +181 if pkg.name.lower() == key.lower(): +182 filters = [] +183 s60filters = [] +184 md5checksum = None +185 size = None +186 if u'filters' in pkg.xml_attributes: +187 filters = _cleanup_list(pkg.filters.split(',')) +188 if u's60filter' in pkg.xml_attributes: +189 s60filters = _cleanup_list(pkg.s60filter.split(',')) +190 if u'md5checksum' in pkg.xml_attributes: +191 md5checksum = pkg.md5checksum +192 if u'size' in pkg.xml_attributes: +193 size = pkg.size +194 return {'type': pkg.type, 'extract': pkg.extract, 'default': (pkg.default=="true"), \ +195 'filters': filters, 's60filter': s60filters, 'md5checksum': md5checksum, 'size': size} +196 raise Exception("Key '%s' not found." % key) +

197 +

198 - def __setitem__(self, key, value): +

199 self.add_package(key, value['type'], value['default'], value['filters'], value['extract'], value['md5checksum'], value['size']) +

200 +

201 - def set_releasedetails_info(self, name, value, details="releaseID"): +

202 """ Generic function to set releaseid info. """ +203 detailsnode = None +204 if not hasattr(self._xml.releaseInformation.releaseDetails, details): +205 detailsnode = self._xml.xml_create_element(details) +206 self._xml.releaseInformation.releaseDetails.xml_append(detailsnode) +207 else: +208 detailsnode = self._xml.releaseInformation.releaseDetails.xml_child_elements[details] +209 namenode = None +210 if not hasattr(detailsnode, name): +211 namenode = self._xml.xml_create_element(name, attributes={u'name': unicode(value)}) +212 detailsnode.xml_append(namenode) +213 else: +214 namenode = detailsnode.xml_child_elements[name] +215 namenode.name = value +

216 +217 +

218 - def get_releasedetails_info(self, name, details="releaseID"): +

219 """ Generic function to extract releaseid info. """ +220 if hasattr(self._xml.releaseInformation.releaseDetails, details): +221 group = getattr(self._xml.releaseInformation.releaseDetails, details) +222 if hasattr(group, name) != None and getattr(getattr(group, name), 'name'): +223 return getattr(group, name).name +224 return None +

225 +

226 - def getVariantPackage(self, variant_name): +

227 for variant in self._xml.releaseInformation.localeVariants.variant: +228 if(variant != None): +229 if variant.name.lower() == variant_name.lower(): +230 return variant.file.name +

231 +232 +

233 - def xml(self): +

234 """ Returning the XML as a string. """ +235 return self._xml.xml() +

236 +

237 - def save(self, filename = None): +

238 """ Saving the XML into the provided filename. """ +239 if filename == None: +240 filename = self._filename +241 file_object = codecs.open(os.path.join(filename), 'w', "utf_8") +242 file_object.write(self._xml.xml(indent=u"yes")) +243 file_object.close() +

244 +245 @property +

246 - def servicepacks(self): +

247 """ Getting the service pack names. """ +248 result = [] +249 if hasattr(self._xml, 'releaseInformation') and hasattr(self._xml.releaseInformation, 'servicePacks'): +250 for sp in self._xml.releaseInformation.servicePacks.servicePack: +251 result.append(ServicePack(sp)) +252 return result +

253 +254 filename = property(lambda self:self._filename) +255 service = property(lambda self:self.get_releasedetails_info('service'), lambda self, value:self.set_releasedetails_info('service', value)) +256 product = property(lambda self:self.get_releasedetails_info('product'), lambda self, value:self.set_releasedetails_info('product', value)) +257 release = property(lambda self:self.get_releasedetails_info('release'), lambda self, value:self.set_releasedetails_info('release', value)) +258 dependsof_service = property(lambda self:self.get_releasedetails_info('service', 'dependsOf'), lambda self, value:self.set_releasedetails_info('service', value, 'dependsOf')) +259 dependsof_product = property(lambda self:self.get_releasedetails_info('product', 'dependsOf'), lambda self, value:self.set_releasedetails_info('product', value, 'dependsOf')) +260 dependsof_release = property(lambda self:self.get_releasedetails_info('release', 'dependsOf'), lambda self, value:self.set_releasedetails_info('release', value, 'dependsOf')) +261 baseline_service = property(lambda self:self.get_releasedetails_info('service', 'previousBaseline'), lambda self, value:self.set_releasedetails_info('service', value, 'previousBaseline')) +262 baseline_product = property(lambda self:self.get_releasedetails_info('product', 'previousBaseline'), lambda self, value:self.set_releasedetails_info('product', value, 'previousBaseline')) +263 baseline_release = property(lambda self:self.get_releasedetails_info('release', 'previousBaseline'), lambda self, value:self.set_releasedetails_info('release', value, 'previousBaseline')) +

264 +265 +

266 -class MD5Updater(ReleaseMetadata): +

267 """ Update Metadata XML already created from SYMREC/SYMDEC. """ +

268 - def __init__(self, filename): +

269 ReleaseMetadata.__init__(self, filename) +270 self._filepath = os.path.dirname(filename) +

271 +

272 - def update(self): +

273 """ Update each existing package md5checksum and size attribute.""" +274 for name in self.keys(): +275 fullname = os.path.join(self._filepath, name) +276 if os.path.exists(fullname): +277 result = self[name] +278 result['md5checksum'] = unicode(fileutils.getmd5(fullname)) +279 result['size'] = unicode(os.path.getsize(fullname)) +280 self[name] = result +

281 +282 +

283 -class ValidateReleaseMetadata(ReleaseMetadata): +

284 """ This class validate if a metadata file is stored in the correct location and +285 if all deps exists. +286 """ +

287 - def __init__(self, filename): +

288 ReleaseMetadata.__init__(self, filename) +289 self.location = os.path.dirname(filename) +

290 +

291 - def is_valid(self, checkmd5=True): +

292 """ Run the validation mechanism. """ +293 if os.path.basename(self.location) != self.release: +294 LOGGER.error("Release doesn't match.") +295 return False +296 if os.path.basename(os.path.dirname(self.location)) != self.product: +297 LOGGER.error("Product doesn't match.") +298 return False +299 if os.path.basename(os.path.dirname(os.path.dirname(self.location))) != self.service: +300 LOGGER.error("Service doesn't match.") +301 return False +302 +303 for name in self.keys(): +304 path = os.path.join(self.location, name) +305 if not os.path.exists(path): +306 LOGGER.error("%s doesn't exist." % path) +307 return False +308 try: +309 LOGGER.debug("Trying to open %s" % path) +310 content_file = open(path) +311 content_file.read(1) +312 except IOError: +313 LOGGER.error("%s is not available yet" % path) +314 return False +315 +316 if checkmd5 and self[name].has_key('md5checksum'): +317 if self[name]['md5checksum'] != None: +318 if fileutils.getmd5(path).lower() != self[name]['md5checksum']: +319 LOGGER.error("%s md5checksum missmatch." % path) +320 return False +321 +322 for sp in self.servicepacks: +323 for name in sp.files: +324 path = os.path.join(self.location, name) +325 if not os.path.exists(path): +326 LOGGER.error("%s doesn't exist." % path) +327 return False +328 for name in sp.instructions: +329 path = os.path.join(self.location, name) +330 if not os.path.exists(path): +331 LOGGER.error("%s doesn't exist." % path) +332 return False +333 +334 dependency = self.get_dependsof() +335 if dependency != None: +336 return ValidateReleaseMetadata(dependency.filename).is_valid(checkmd5) +337 return True +

338 +

339 -class MetadataMerger(object): +

340 """ Merge packages definition to the root metadata. """ +341 +

342 - def __init__(self, metadata): +

343 """ Construct a metadata merger providing root metadata filename. """ +344 self._metadata = ReleaseMetadata(metadata) +

345 +

346 - def merge(self, filename): +

347 """ Merge the content of filename into the root metadata. """ +348 metadata = ReleaseMetadata(filename) +349 for name in metadata.keys(): +350 if name in self._metadata.keys(): +351 LOGGER.warning('Package %s already declared, overriding previous definition!' % name) +352 self._metadata[name] = metadata[name] +

353 +

354 - def xml(self): +

355 """ Returning the XML as a string. """ +356 return self._metadata.xml() +

357 +

358 - def save(self, filename = None): +

359 """ Saving the XML into the provided filename. """ +360 return self._metadata.save(filename) +

361 +

362 -class Metadata2TDD(ReleaseMetadata): +

363 +

364 - def __init__(self, filename, includes=None, excludes=None): +

365 ReleaseMetadata.__init__(self, filename) +366 if includes is None: +367 includes = [] +368 if excludes is None: +369 excludes = [] +370 self.location = os.path.dirname(filename) +371 self.includes = includes +372 self.excludes = excludes +

373 +

374 - def archives_to_tdd(self, metadata): +

375 tdd = "\t[\n" +376 for name in metadata.keys(): +377 path_ = os.path.join(os.path.dirname(metadata.filename), name) +378 if (((len(self.includes) == 0) and metadata[name]['extract']) or (self.includes in metadata[name]['s60filter'])) and self.excludes not in metadata[name]['s60filter']: +379 tdd += "\t\t{\n" +380 tdd += "\t\t\t\"command\": \"unzip_%s\",\n" % metadata[name]['extract'] +381 tdd += "\t\t\t\"src\": \"%s\",\n" % os.path.normpath(path_).replace('\\', '/') +382 tdd += "\t\t},\n" +383 tdd += "\t],\n" +384 return tdd +

385 +

386 - def to_tdd(self): +

387 """ Generating a TDD file that contains a list of list of filenames. """ +388 tdd = "[\n" +389 # generates unarchiving steps for dependency +390 dependency = self.get_dependsof() +391 if dependency != None: +392 tdd += self.archives_to_tdd(dependency) +393 # generates unarchiving steps +394 tdd += self.archives_to_tdd(self) +395 tdd += "]\n" +396 return tdd +

397 +398 +399 +

400 -def find_latest_metadata(releasedir): +

401 """ Finding the release latest release metadata file. """ +402 metadatas = [] +403 for filename in os.listdir(releasedir): +404 if re.match(r'^release_metadata(_\d+)?\.xml$', filename, re.I) is not None: +405 LOGGER.debug("Found %s" % filename) +406 metadatas.append(filename) +407 # reverse the order... +408 metadatas.sort(reverse=True) +409 if len(metadatas) > 0: +410 return os.path.normpath(os.path.join(releasedir, metadatas[0])) +411 return None +

412 +413 +414 +

415 -class ValidateReleaseMetadataCached(ValidateReleaseMetadata): +

416 """ Cached version of the metadata validation. """ +

417 - def __init__(self, filename, cachefile=None): +

418 ValidateReleaseMetadata.__init__(self, filename) +419 self.__cachefile = cachefile +

420 +

421 - def is_valid(self, checkmd5=True): +

422 """ Check if file is in the local cache. +423 Add valid release to the cache. +424 """ +425 metadatas = self.load_cache() +426 if self.in_cache(metadatas, os.path.normpath(self._filename)): +427 LOGGER.debug("Release found in cache.") +428 return self.value_from_cache(metadatas, os.path.normpath(self._filename)) +429 else: +430 result = ValidateReleaseMetadata.is_valid(self, checkmd5) +431 LOGGER.debug("Updating the cache.") +432 metadatas.append([os.path.normpath(self._filename), result]) +433 self.update_cache(metadatas) +434 return result +

435 +

436 - def in_cache(self, metadatas, key): +

437 for metadata in metadatas: +438 if metadata[0] == key: +439 return True +440 return False +

441 +

442 - def value_from_cache(self, metadatas, key): +

443 for metadata in metadatas: +444 if metadata[0] == key: +445 return metadata[1] +446 return None +

447 +

448 - def load_cache(self): +

449 metadatas = [] +450 if self.__cachefile is not None and os.path.exists(self.__cachefile): +451 for row in csv.reader(open(self.__cachefile, "rb")): +452 if len(row) == 2: +453 metadatas.append([os.path.normpath(row[0]), row[1].lower() == "true"]) +454 elif len(row) == 1: +455 # backward compatibility with old cache. +456 metadatas.append([os.path.normpath(row[0]), True]) +457 return metadatas +

458 +

459 - def update_cache(self, metadatas): +

460 if self.__cachefile is not None and os.path.exists(os.path.dirname(self.__cachefile)): +461 writer = csv.writer(open(self.__cachefile, "wb")) +462 writer.writerows(metadatas) +

463 +

464 -class ValidateTicklerReleaseMetadata(ValidateReleaseMetadataCached): +

465 """ This class validate if a metadata file is stored in the correct location and +466 if all deps exists. +467 """ +

468 - def __init__(self, filename): +

469 ReleaseMetadata.__init__(self, filename) +470 self.location = os.path.dirname(filename) +

471 +

472 - def is_valid(self, checkmd5=True): +

473 """ Run the validation mechanism. """ +474 tickler_path = os.path.join(self.location,"TICKLER") +475 if not os.path.exists(tickler_path): +476 LOGGER.error("Release not available yet") +477 return False +478 else: +479 return ValidateReleaseMetadataCached.is_valid(self, checkmd5) +

480 +

Source Code for Module symrec