sbsv2/raptor/bin/sbsv2cache.py
changeset 0 044383f39525
child 3 e1eecf4d390d
equal deleted inserted replaced
-1:000000000000 0:044383f39525
       
     1 #
       
     2 # Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
       
     3 # All rights reserved.
       
     4 # This component and the accompanying materials are made available
       
     5 # under the terms of the License "Eclipse Public License v1.0"
       
     6 # which accompanies this distribution, and is available
       
     7 # at the URL "http://www.eclipse.org/legal/epl-v10.html".
       
     8 #
       
     9 # Initial Contributors:
       
    10 # Nokia Corporation - initial contribution.
       
    11 #
       
    12 # Contributors:
       
    13 #
       
    14 # Description: 
       
    15 # Creates CBR tool compatible cache files from SBSv2 .whatlog variant output
       
    16 #
       
    17 
       
    18 
       
    19 import sys
       
    20 import os
       
    21 from optparse import OptionParser
       
    22 import xml.parsers.expat
       
    23 import re
       
    24 
       
    25 
       
    26 # Global dictionary of ComponentReleasable objects, keyed on bld.inf file
       
    27 BuildReleasables = {}
       
    28 
       
    29 # Provide a means to form  "traditional" ABLD-like build platforms and variants from SBSv2 configurations
       
    30 ConfigMatch = re.compile(r'^(?P<PLATFORM>\w+)_(?P<VARIANT>\w+)(\.((?P<PLATFORMADD>smp)|\w+))*')
       
    31 
       
    32 WinscwTreeMatch = re.compile(r'[\\|\/]epoc32[\\|\/]release[\\|\/]winscw[\\|\/](?P<VARIANT>(urel|udeb))[\\|\/]', re.IGNORECASE)
       
    33 WinDriveMatch = re.compile(r'[A-Za-z]:')
       
    34 
       
    35 # $self->{abldcache}->{'<bld.inf location> export -what'} =
       
    36 # $self->{abldcache}->{'<bld.inf location> <phase> <platform> <variant> -what'} =
       
    37 # $self->{abldcache}->{'plats'} =
       
    38 CacheGroupPrefix = "$self->{abldcache}->{\'"
       
    39 CacheGroupSuffix = "\'} =\n"
       
    40 CacheExportGroup = CacheGroupPrefix+"%s export -what"+CacheGroupSuffix
       
    41 CacheBuildOutputGroup = CacheGroupPrefix+"%s %s %s %s -what"+CacheGroupSuffix
       
    42 CachePlatsGroup = CacheGroupPrefix+"plats"+CacheGroupSuffix
       
    43 CacheListOpen = "\t[\n"
       
    44 CacheListItem = "\t\'%s\'"
       
    45 CacheListItemPair = "\t[\'%s\', \'%s\']"
       
    46 CacheListClose = "\t];\n\n"
       
    47 
       
    48 
       
    49 class ComponentReleasable(object):
       
    50 	"""Wraps up a bld.inf file in terms of its packagable releasable output."""
       
    51 	
       
    52 	# If EPOCROOT is set, provide a means to confirm that potentially publishable releasables live under EPOCROOT/epoc32
       
    53 	ReleaseTreeMatch = None
       
    54 	if os.environ.has_key("EPOCROOT"):
       
    55 		ReleaseTreeMatch = re.compile(r'\"*'+os.path.abspath(os.path.join(os.environ["EPOCROOT"],"epoc32")).replace('\\',r'\/').replace('\/',r'[\\|\/]+')+r'[\\|\/]+', re.IGNORECASE)
       
    56 		
       
    57 	def __init__(self, aBldInfFile, aVerbose=False):
       
    58 		self.__BldInfFile = aBldInfFile
       
    59 		self.__Verbose = aVerbose
       
    60 		self.__Exports = {}
       
    61 		self.__BuildOutput = {}
       
    62 		self.__Platforms = {}
       
    63 		
       
    64 	def __IsReleasableItem(self, aBuildItem):
       
    65 		if self.ReleaseTreeMatch and self.ReleaseTreeMatch.match(aBuildItem):
       
    66 			return True
       
    67 		
       
    68 		if self.__Verbose:
       
    69 			print "Discarding: \'%s\' from \'%s\' as not in the release tree." % (aBuildItem, self.__BldInfFile)
       
    70 		return False
       
    71 
       
    72 	def __StoreBuildItem(self, aPlatform, aVariant, aBuildItem):
       
    73 		if not self.__BuildOutput.has_key(aPlatform):
       
    74 			self.__BuildOutput[aPlatform] = {}
       
    75 			if aPlatform != "ALL":
       
    76 				self.__Platforms[aPlatform.upper()] = 1
       
    77 		if not self.__BuildOutput[aPlatform].has_key(aVariant):
       
    78 			self.__BuildOutput[aPlatform][aVariant] = {}
       
    79 		
       
    80 		if aBuildItem:
       
    81 			self.__BuildOutput[aPlatform][aVariant][aBuildItem] = 1
       
    82 		
       
    83 	def AddExport(self, aDestination, aSource):
       
    84 		if not self.__IsReleasableItem(aDestination):
       
    85 			return
       
    86 		self.__Exports[aDestination] = aSource
       
    87 
       
    88 	def AddBuildOutput(self, aBuildItem, aPlatform="ALL", aVariant="ALL"):
       
    89 		if not self.__IsReleasableItem(aBuildItem):
       
    90 			return
       
    91 		if aPlatform != "ALL" and aVariant == "ALL":
       
    92 			self.__StoreBuildItem(aPlatform, "urel", aBuildItem)
       
    93 			self.__StoreBuildItem(aPlatform, "udeb", aBuildItem)
       
    94 		else:
       
    95 			self.__StoreBuildItem(aPlatform, aVariant, aBuildItem)
       
    96 		
       
    97 	def Finalise(self):
       
    98 		# Re-visit the stored build items and, in the context of all build platforms having been processed for the
       
    99 		# component, copy platform-generic "ALL" output to the concrete build platform outputs
       
   100 		if self.__BuildOutput.has_key("ALL"):
       
   101 			allItems = self.__BuildOutput["ALL"]["ALL"].keys()		
       
   102 			for platform in self.__BuildOutput.keys():
       
   103 				for variant in self.__BuildOutput[platform].keys():
       
   104 					for allItem in allItems:
       
   105 						self.__StoreBuildItem(platform, variant, allItem)			
       
   106 			del self.__BuildOutput["ALL"]
       
   107 	
       
   108 	def GetBldInf(self):
       
   109 		return self.__BldInfFile
       
   110 
       
   111 	def GetExports(self):
       
   112 		return self.__Exports
       
   113 
       
   114 	def GetBuildOutput(self):
       
   115 		return self.__BuildOutput
       
   116 
       
   117 	def GetPlatforms(self):
       
   118 		return self.__Platforms
       
   119 
       
   120 	def HasReleasables(self):
       
   121 		return (self.__BuildOutput or self.__Exports)
       
   122 							
       
   123 
       
   124 def error(aMessage):
       
   125 	sys.stderr.write("ERROR: sbsv2cache.py : %s\n" % aMessage)
       
   126 	sys.exit(1)
       
   127 	
       
   128 def processReleasableElement(aContext, aName, aValue, aVerbose):
       
   129 	bldinf = aContext["bldinf"]
       
   130 	mmp = aContext["mmp"]
       
   131 	config = aContext["config"]
       
   132 
       
   133 	platform = ""
       
   134 	variant = ""
       
   135 	configMatchResults = ConfigMatch.match(config)
       
   136 	if configMatchResults:
       
   137 		platform = configMatchResults.group('PLATFORM')
       
   138 		variant = configMatchResults.group('VARIANT')	
       
   139 		if configMatchResults.group('PLATFORMADD'):
       
   140 			platform += configMatchResults.group('PLATFORMADD')
       
   141 	
       
   142 	if not BuildReleasables.has_key(bldinf):
       
   143 		BuildReleasables[bldinf] = ComponentReleasable(bldinf, aVerbose)
       
   144 	
       
   145 	componentReleasable = BuildReleasables[bldinf]
       
   146 	
       
   147 	if aName == "export" :
       
   148 		componentReleasable.AddExport(aValue["destination"], aValue["source"])
       
   149 	elif aName == "member":
       
   150 		componentReleasable.AddExport(aValue.keys()[0], aContext["zipfile"])
       
   151 	elif aName == "build":
       
   152 		componentReleasable.AddBuildOutput(aValue.keys()[0], platform, variant)
       
   153 	elif aName == "resource" or aName == "bitmap":
       
   154 		item = aValue.keys()[0]
       
   155 		# Identify winscw urel/udeb specific resources, and store accordingly
       
   156 		winscwTreeMatchResult = WinscwTreeMatch.search(item)
       
   157 		if platform == "winscw" and winscwTreeMatchResult:
       
   158 			componentReleasable.AddBuildOutput(item, platform, winscwTreeMatchResult.group("VARIANT").lower())
       
   159 		else:
       
   160 			componentReleasable.AddBuildOutput(item, platform)
       
   161 	elif aName == "stringtable":
       
   162 		componentReleasable.AddBuildOutput(aValue.keys()[0])			
       
   163 
       
   164 def parseLog(aLog, aVerbose):
       
   165 	if not os.path.exists(aLog):
       
   166 		error("Log file %s does not exist." % aLog)
       
   167 		
       
   168 	parser = xml.parsers.expat.ParserCreate()
       
   169 	parser.buffer_text = True
       
   170 	
       
   171 	elementContext = {}
       
   172 	currentElement = []
       
   173 		
       
   174 	def start_element(name, attributes):
       
   175 		if name == "whatlog" or name == "archive":
       
   176 			elementContext.update(attributes)
       
   177 		elif elementContext.has_key("bldinf"):
       
   178 			if name == "export":
       
   179 				# Exports are all attributes, so deal with them directly
       
   180 				processReleasableElement(elementContext, name, attributes, aVerbose)
       
   181 			else:
       
   182 				# Other elements wrap values, get these later
       
   183 				currentElement.append(name)
       
   184 						
       
   185 	def end_element(name):
       
   186 		if name == "whatlog":
       
   187 			elementContext.clear()
       
   188 		elif name == "archive":
       
   189 			del elementContext["zipfile"]
       
   190 	
       
   191 	def char_data(data):
       
   192 		if elementContext.has_key("bldinf") and currentElement:
       
   193 			processReleasableElement(elementContext, currentElement.pop(), {str(data):1}, aVerbose)
       
   194 	
       
   195 	parser.StartElementHandler = start_element
       
   196 	parser.EndElementHandler = end_element
       
   197 	parser.CharacterDataHandler = char_data
       
   198 
       
   199 	try:
       
   200 		if aVerbose:
       
   201 			print "Parsing: " + aLog
       
   202 			
       
   203 		parser.ParseFile(open(aLog, "r"))
       
   204 	except xml.parsers.expat.ExpatError, e:	
       
   205 		error("Failure parsing log file \'%s\' (line %s)" % (aLog, e.lineno))
       
   206 
       
   207 def normFileForCache(aFile):
       
   208 	normedFile = WinDriveMatch.sub("",aFile)
       
   209 	normedFile = normedFile.replace("/", "\\")
       
   210 	normedFile = normedFile.replace("\\", "\\\\")
       
   211 	normedFile = normedFile.replace("\\\\\\\\", "\\\\")
       
   212 	normedFile = normedFile.replace("\"", "")
       
   213 	return normedFile
       
   214 	
       
   215 def dumpCacheFileList(aCacheFileObject, aItems, aPairs=False):	
       
   216 	numItems = len(aItems)
       
   217 	suffix = ",\n"
       
   218 	
       
   219 	aCacheFileObject.write(CacheListOpen)
       
   220 	for item in aItems:
       
   221 		if aItems.index(item) == numItems-1:
       
   222 			suffix = "\n"			
       
   223 		if aPairs:
       
   224 			aCacheFileObject.write((CacheListItemPair % (normFileForCache(item[0]), normFileForCache(item[1]))) + suffix)
       
   225 		else:
       
   226 			aCacheFileObject.write((CacheListItem % normFileForCache(item)) + suffix)
       
   227 	aCacheFileObject.write(CacheListClose)
       
   228 	
       
   229 def createCacheFile(aComponentReleasable, aOutputPath, aSourceExports, aVerbose):	
       
   230 	if not aComponentReleasable.HasReleasables():
       
   231 		return
       
   232 	
       
   233 	cacheFileDir = os.path.normpath(\
       
   234 				os.path.join(aOutputPath, \
       
   235 	            WinDriveMatch.sub("",os.path.dirname(aComponentReleasable.GetBldInf())).lstrip(r'/').lstrip(r'\\')))
       
   236 	cacheFile = os.path.join(cacheFileDir, "cache")
       
   237 	
       
   238 	bldInfLoc = WinDriveMatch.sub("",os.path.dirname(aComponentReleasable.GetBldInf())).replace("/", "\\")
       
   239 
       
   240 	if aVerbose:
       
   241 		print "Creating: " + cacheFile
       
   242 	
       
   243 	if not os.path.exists(cacheFileDir):
       
   244 		os.makedirs(cacheFileDir)
       
   245 	
       
   246 	try:
       
   247 		cacheFileObject = open(cacheFile, 'w')
       
   248 	
       
   249 		exports = aComponentReleasable.GetExports()
       
   250 		if exports:
       
   251 			cacheFileObject.write(CacheExportGroup % bldInfLoc)
       
   252 			if aSourceExports:
       
   253 				dumpCacheFileList(cacheFileObject, exports.items(), True)
       
   254 			else:
       
   255 				dumpCacheFileList(cacheFileObject, exports.keys())
       
   256 	
       
   257 		buildOutput = aComponentReleasable.GetBuildOutput()		
       
   258 		if buildOutput:
       
   259 			for plat in buildOutput.keys():
       
   260 				# Most cache output is represented as if performed for the "abld target" phase, but tools platforms
       
   261 				# are presented as if performed by "abld build", and so must additionally replicate any exports
       
   262 				# performed for the component in their variant output
       
   263 				phase = "target"
       
   264 				additionalOutput = []
       
   265 				if plat == "tools" or plat == "tools2":
       
   266 					phase = "build"
       
   267 					if exports:
       
   268 						additionalOutput = exports.keys()
       
   269 				
       
   270 				for variant in buildOutput[plat].keys():
       
   271 					cacheFileObject.write(CacheBuildOutputGroup % (bldInfLoc, phase, plat, variant))
       
   272 					dumpCacheFileList(cacheFileObject, buildOutput[plat][variant].keys() + additionalOutput)
       
   273 	
       
   274 		cacheFileObject.write(CachePlatsGroup)
       
   275 		dumpCacheFileList(cacheFileObject, aComponentReleasable.GetPlatforms().keys())
       
   276 		
       
   277 		cacheFileObject.close()
       
   278 	except IOError:
       
   279 		error("Failure creating cache file %s." % cacheFile)
       
   280 
       
   281 
       
   282 def main():
       
   283 	parser = OptionParser(prog="sbsv2cache.py")
       
   284 	parser.add_option("-l", "--log", action="append", dest="logs", help="log file to parse for <whatlog/> wrapped content.")
       
   285 	parser.add_option("-o", "--outputpath", action="store", dest="outputpath", help="root location to generate cache files.")
       
   286 	parser.add_option("-s", "--sourceexports", action="store_true", default=False, dest="sourceexports", help="generate cache files where each element in the export array is a ['destination', 'source'] array rather than just a 'destination' element.")
       
   287 	parser.add_option("-v", "--verbose", action="store_true", default=False, dest="verbose", help="provide more information as things happen.")
       
   288 	
       
   289 	(options, leftover_args) = parser.parse_args(sys.argv[1:])
       
   290 
       
   291 	if leftover_args or not options.logs or not options.outputpath:
       
   292 		parser.print_help()
       
   293 		sys.exit(1)
       
   294 		
       
   295 	print "sbsv2cache: started"
       
   296 	
       
   297 	# Parse build logs to populate the BuildReleasables dictionary
       
   298 	for log in options.logs:
       
   299 		parseLog(os.path.abspath(log), options.verbose)
       
   300 	
       
   301 	# Finalise components in BuildReleasables and create cache files as we go
       
   302 	for component in BuildReleasables.keys():
       
   303 		BuildReleasables[component].Finalise()
       
   304 		createCacheFile(BuildReleasables[component], os.path.abspath(options.outputpath), options.sourceexports, options.verbose)
       
   305 		
       
   306 	print "sbsv2cache: finished"
       
   307 	
       
   308 if __name__ == "__main__":
       
   309 	main()
       
   310 	
       
   311