--- a/sbsv2/raptor/RELEASE-NOTES.txt Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/RELEASE-NOTES.txt Mon Mar 22 08:04:37 2010 +0000
@@ -1,9 +1,16 @@
Release Notes for Symbian Build System v2
next version
+- Fix: Workaround for emake engine log corruption when clockskew errors occur (annofile2log).
+ Allow Raptor to obtain log from emake annotation file where it is uncorrupted. A new
+ Make engine option "copyannofile2log" enables/disables this mode for emake. If this option is disabled
+ or if no annotation file is specified for the build then Raptor reads logs directly as normal.
- SF Bug 2125 - [Raptor] - tracecompiler what output incorrect if mmp basename contains '.' e.g. fred.prd.mmp
+- SF Bug 2191 - [Raptor] - When forcesuccess is enabled, exit status for a failed recipe is "retry" but should be "failed"
- Fix: extend tracecompiler tests to Linux
- Fix: Amendment to SF Bug 1511 fix - removal of blanked DEFFILE keyword from e32abiv2ani.flm
+- Fix: improve robustness to bad -c options
+
version 2.12.4
--- a/sbsv2/raptor/lib/config/locations.xml Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/lib/config/locations.xml Mon Mar 22 08:04:37 2010 +0000
@@ -30,7 +30,7 @@
<env name='SBS_GNUTOUCH' default='/bin/touch' type='tool'/>
<env name='SBS_GNUFIND' default='/usr/bin/find' type='tool'/>
<env name='SBS_GNUGREP' default='/bin/grep' type='tool'/>
- <env name='SBS_GNUSORT' default='/bin/sort' type='tool'/>
+ <env name='SBS_GNUSORT' default='sort' type='tool'/>
<env name='SBS_SHELL' default="$(SBS_HOME)/$(HOSTPLATFORM_DIR)/bin/sh" type='tool'/>
<env name='SBS_ZIP' default="$(SBS_HOME)/$(HOSTPLATFORM_DIR)/bin/zip" type='tool'/>
<env name='SBS_UNZIP' default="$(SBS_HOME)/$(HOSTPLATFORM_DIR)/bin/unzip" type='tool'/>
--- a/sbsv2/raptor/lib/config/make.xml Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/lib/config/make.xml Mon Mar 22 08:04:37 2010 +0000
@@ -47,6 +47,9 @@
<!-- is the text output with -j buffered or scrambled? -->
<set name="scrambled" value="true"/>
+
+ <!-- workaround for damaged log output from emake -->
+ <set name="copylogfromannofile" value="false"/>
</var>
<alias name="make" meaning="make_engine"/>
@@ -67,6 +70,9 @@
<set name="build" value="$(EMAKE) HAVE_ORDERONLY= -r"/>
<set name="scrambled" value="false"/>
<set name='TALON_DESCRAMBLE' value=''/>
+
+ <!-- workaround for damaged log output from emake -->
+ <set name="copylogfromannofile" value="true"/>
</var>
<alias name="emake" meaning="emake_engine"/>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/python/plugins/filter_bz2log.py Mon Mar 22 08:04:37 2010 +0000
@@ -0,0 +1,88 @@
+# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Compress the full Raptor log file using the BZip2 algorithm, maximum compression.
+#
+#
+
+import os
+import sys
+import raptor
+import filter_interface
+import bz2
+
+class StringListCompressor(object):
+ def __init__(self, complevel=5, filename="file.log.bz2"):
+ self.compressor = bz2.BZ2Compressor(complevel)
+ self.stringlist = []
+ self.outputopenedok = False
+ self.filename = filename
+ try:
+ self.fh = open(self.filename, "wb")
+ self.outputopenedok = True
+ except:
+ self.outputopenedok = False
+
+ def write(self, data):
+ if self.outputopenedok:
+ compresseddata = self.compressor.compress(data)
+ self.fh.write(compresseddata)
+
+ def __del__(self):
+ if self.outputopenedok:
+ compresseddata = self.compressor.flush()
+ self.fh.write(compresseddata)
+ self.fh.close()
+
+class Bz2log(filter_interface.Filter):
+ def __init__(self):
+ self.__inRecipe = False
+ self.compressor = None
+
+ def open(self, raptor_instance):
+ """Open a log file for the various I/O methods to write to."""
+
+ if raptor_instance.logFileName == None:
+ self.out = sys.stdout # Default to stdout if no log file is given
+ else:
+ logname = str(raptor_instance.logFileName.path.replace("%TIME", raptor_instance.timestring))
+
+ # Ensure that filename has the right extension; append ".bz2" if required
+ if not logname.lower().endswith(".bz2"):
+ logname += ".bz2"
+
+ try:
+ dirname = str(raptor_instance.logFileName.Dir())
+ if dirname and not os.path.isdir(dirname):
+ os.makedirs(dirname)
+ except:
+ self.formatError("cannot create directory %s", dirname)
+ return False
+
+ # Use highest compression level 9 which corresponds to a 900KB dictionary
+ self.compressor = StringListCompressor(9, logname)
+ if not self.compressor.outputopenedok:
+ self.out = None
+ self.formatError("failed to initialise compression routines." )
+ return False
+ return True
+
+ def write(self, data):
+ """Write data compressed log"""
+ if self.compressor:
+ self.compressor.write(data)
+ return True
+
+ def close(self):
+ """Close the log file"""
+ return True
--- a/sbsv2/raptor/python/plugins/filter_terminal.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/python/plugins/filter_terminal.py Mon Mar 22 08:04:37 2010 +0000
@@ -161,7 +161,7 @@
# list of strings to catch recipe warnings (must be lowercase)
self.recipe_warning_expr = ["warning:"]
-
+
def isMakeWarning(self, text):
"""A simple test for warnings.
Can be extended do to more comprehensive checking."""
@@ -201,6 +201,9 @@
if self.raptor.quiet:
self.quiet = True
+ # the build configurations which were reported
+ self.built_configs = []
+
# keep count of errors and warnings
self.err_count = 0
self.warn_count = 0
@@ -393,7 +396,9 @@
self.recipeBody.append(text)
else:
self.recipelineExceeded += 1
-
+ elif text.startswith("<info>Buildable configuration '"):
+ # <info>Buildable configuration 'name'</info>
+ self.built_configs.append(text[30:-8])
def logit(self):
""" log a message """
@@ -435,8 +440,11 @@
sys.stdout.write("%s : errors: %s\n" % (raptor.name,
self.err_count))
else:
- sys.stdout.write("\nno warnings or errors\n")
+ sys.stdout.write("\nno warnings or errors\n\n")
+ for bc in self.built_configs:
+ sys.stdout.write("built " + bc + "\n")
+
sys.stdout.write("\nRun time %d seconds\n" % self.raptor.runtime);
sys.stdout.write("\n")
return True
--- a/sbsv2/raptor/python/raptor.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/python/raptor.py Mon Mar 22 08:04:37 2010 +0000
@@ -42,7 +42,6 @@
import sys
import types
import time
-import re
import traceback
import pluginbox
from xml.sax.saxutils import escape
@@ -232,8 +231,9 @@
def makefile(self, makefilename_base, engine, named = False):
"""Makefiles for individual mmps not feasible at the moment"""
- pass # Cannot, currently, "unfurl an mmp" directly but do want
- # to be able to simulate the overall recursive unfurling of a build.
+ pass
+ # Cannot, currently, "unfurl an mmp" directly but do want
+ # to be able to simulate the overall recursive unfurling of a build.
class Component(ModelNode):
"""A group of projects or, in symbian-speak, a bld.inf.
@@ -321,7 +321,6 @@
# insert the start time into the Makefile name?
- buildconfig = build.GetConfig("build").GenerateBuildUnits(build.cache)
self.configs = build.buildUnitsToBuild
# Pass certain CLI flags through to the makefile-generating sbs calls
@@ -385,8 +384,7 @@
makefile_path = str(build.topMakefile) + "_" + str(loop_number)
try:
os.unlink(makefile_path) # until we have dependencies working properly
- except Exception,e:
- # print "couldn't unlink %s: %s" %(componentMakefileName, str(e))
+ except Exception:
pass
# add some basic data in a component-wide variant
@@ -859,32 +857,6 @@
self.cache.Load(self.systemFLM)
- def GetConfig(self, configname):
- names = configname.split(".")
-
- cache = self.cache
-
- base = names[0]
- mods = names[1:]
-
- if base in cache.groups:
- x = cache.FindNamedGroup(base)
- elif base in cache.aliases:
- x = cache.FindNamedAlias(base)
- elif base in cache.variants:
- x = cache.FindNamedVariant(base)
- else:
- raise Exception("Unknown build configuration '%s'" % configname)
-
- x.ClearModifiers()
-
-
- try:
- for m in mods: x.AddModifier( cache.FindNamedVariant(m) )
- except KeyError:
- raise Exception("Unknown build configuration '%s'" % configname)
- return x
-
def GetBuildUnitsToBuild(self, configNames):
"""Return a list of the configuration objects that correspond to the
list of configuration names in the configNames parameter.
@@ -900,17 +872,7 @@
else:
configNames.append(self.defaultConfig)
- buildUnitsToBuild = set()
-
-
- for c in set(configNames):
- self.Debug("BuildUnit: %s", c)
- try:
- x = self.GetConfig(c)
- gb = x.GenerateBuildUnits(self.cache)
- buildUnitsToBuild.update( gb )
- except Exception, e:
- self.FatalError(str(e))
+ buildUnitsToBuild = raptor_data.GetBuildUnits(configNames, self.cache, self)
for b in buildUnitsToBuild:
self.Info("Buildable configuration '%s'", b.name)
@@ -1003,7 +965,6 @@
dir = generic_path.Path(aDir)
bldInf = dir.Append(self.buildInformation)
- componentgroup = []
if bldInf.isFile():
return bldInf
@@ -1150,7 +1111,7 @@
self.out.write(raptor_timing.Timing.discovery_string(object_type = object_type,
count = count))
except Exception, exception:
- Error(exception.Text, function = "InfoDiscoveryTime")
+ self.Error(exception.Text, function = "InfoDiscoveryTime")
def InfoStartTime(self, object_type, task, key):
if self.timing:
@@ -1158,7 +1119,7 @@
self.out.write(raptor_timing.Timing.start_string(object_type = object_type,
task = task, key = key))
except Exception, exception:
- Error(exception.Text, function = "InfoStartTime")
+ self.Error(exception.Text, function = "InfoStartTime")
def InfoEndTime(self, object_type, task, key):
if self.timing:
@@ -1166,7 +1127,7 @@
self.out.write(raptor_timing.Timing.end_string(object_type = object_type,
task = task, key = key))
except Exception, exception:
- Error(exception.Text, function = "InfoEndTime")
+ self.Error(exception.Text, function = "InfoEndTime")
def Debug(self, format, *extras, **attributes):
"Send a debugging message to the configured channel"
@@ -1280,6 +1241,9 @@
self.AssertBuildOK()
buildUnitsToBuild = self.GetBuildUnitsToBuild(self.configNames)
+ if len(buildUnitsToBuild) == 0:
+ raise BuildCannotProgressException("No configurations to build.")
+
self.buildUnitsToBuild = buildUnitsToBuild
# find out what components to build, and in what way
--- a/sbsv2/raptor/python/raptor_data.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/python/raptor_data.py Mon Mar 22 08:04:37 2010 +0000
@@ -65,7 +65,7 @@
# Make sure not to start up on an unsupported platform
if not HostPlatform.IsKnown(HostPlatform.hostplatform):
- raise Exception("raptor_data module loaded on an unrecognised platform '%s'. Expected one of %s" % (hostplatform, str(hostplatforms)))
+ raise Exception("raptor_data module loaded on an unrecognised platform '%s'. Expected one of %s" % (HostPlatform.hostplatform, str(HostPlatform.hostplatforms)))
# raptor_data module classes
@@ -139,7 +139,13 @@
raise BadReferenceError()
def GetModifiers(self, cache):
- return [ cache.FindNamedVariant(m) for m in self.modifiers ]
+ mods = []
+ for m in self.modifiers:
+ try:
+ mods.append(cache.FindNamedVariant(m))
+ except KeyError:
+ raise BadReferenceError(m)
+ return mods
def Valid(self):
return self.ref
@@ -710,7 +716,7 @@
def __str__(self):
attributes = "name='" + self.name + "' type='" + self.type + "'"
- if default != None:
+ if self.default != None:
attributes += " default='" + self.default + "'"
if type == "tool":
@@ -906,7 +912,6 @@
s += "</var>"
return s
-import traceback
class VariantRef(Reference):
def __init__(self, ref=None):
@@ -918,7 +923,7 @@
def Resolve(self, cache):
try:
return cache.FindNamedVariant(self.ref)
- except KeyError, e:
+ except KeyError:
raise BadReferenceError(self.ref)
class MissingVariantException(Exception):
@@ -961,7 +966,7 @@
missing_variants.append(r.ref)
if len(missing_variants) > 0:
- raise MissingVariantException("Missing variants '%s'", " ".join(missing_variants))
+ raise MissingVariantException("Missing variants '%s'" % " ".join(missing_variants))
def GenerateBuildUnits(self, cache):
self.Resolve(cache)
@@ -1026,25 +1031,27 @@
def GenerateBuildUnits(self, cache):
units = []
-
+
missing_variants = []
for r in self.childRefs:
- refMods = r.GetModifiers(cache)
-
try:
obj = r.Resolve(cache=cache)
except BadReferenceError:
missing_variants.append(r.ref)
else:
obj.ClearModifiers()
+ try:
+ refMods = r.GetModifiers(cache)
+ except BadReferenceError,e:
+ missing_variants.append(str(e))
+ else:
+ for m in refMods + self.modifiers:
+ obj.AddModifier(m)
- for m in refMods + self.modifiers:
- obj.AddModifier(m)
-
- units.extend( obj.GenerateBuildUnits(cache) )
+ units.extend( obj.GenerateBuildUnits(cache) )
if len(missing_variants) > 0:
- raise MissingVariantException("Missing variants '%s'", " ".join(missing_variants))
+ raise MissingVariantException("Missing variants '%s'" % " ".join(missing_variants))
return units
@@ -1055,7 +1062,7 @@
Reference.__init__(self, ref)
def __str__(self):
- return "<%s /><groupRef ref='%s' mod='%s'/>" % (prefix, self.ref, ".".join(self.modifiers))
+ return "<groupRef ref='%s' mod='%s'/>" % (self.ref, ".".join(self.modifiers))
def Resolve(self, cache):
try:
@@ -1063,6 +1070,81 @@
except KeyError:
raise BadReferenceError(self.ref)
+def GetBuildUnits(configNames, cache, logger):
+ """expand a list of config strings like "arm.v5.urel" into a list
+ of BuildUnit objects that can be queried for settings.
+
+ The expansion tries to be tolerant of errors in the XML so that a
+ typo in one part of a group does not invalidate the whole group.
+ """
+
+ # turn dot-separated name strings into Model objects (Group, Alias, Variant)
+ models = []
+
+ for c in set(configNames):
+ ok = True
+ names = c.split(".")
+
+ base = names[0]
+ mods = names[1:]
+
+ if base in cache.groups:
+ x = cache.FindNamedGroup(base)
+ elif base in cache.aliases:
+ x = cache.FindNamedAlias(base)
+ elif base in cache.variants:
+ x = cache.FindNamedVariant(base)
+ else:
+ logger.Error("Unknown build configuration '%s'" % base)
+ continue
+
+ x.ClearModifiers()
+
+ for m in mods:
+ if m in cache.variants:
+ x.AddModifier( cache.FindNamedVariant(m) )
+ else:
+ logger.Error("Unknown build variant '%s'" % m)
+ ok = False
+
+ if ok:
+ models.append(copy.copy(x))
+
+ # turn Model objects into BuildUnit objects
+ #
+ # all objects have a GenerateBuildUnits method but don't use
+ # that for Groups because it is not tolerant of errors (the
+ # first error raises an exception and the rest of the group is
+ # abandoned)
+ units = []
+
+ while len(models) > 0:
+ x = models.pop()
+ try:
+ if isinstance(x, (Alias, Variant)):
+ # these we just turn straight into BuildUnits
+ units.extend(x.GenerateBuildUnits(cache))
+ elif isinstance(x, Group):
+ # deal with each part of the group separately (later)
+ for child in x.childRefs:
+ modChild = copy.copy(child)
+ modChild.modifiers = child.modifiers + [m.name for m in x.modifiers]
+ models.append(modChild)
+ elif isinstance(x, Reference):
+ # resolve references and their modifiers
+ try:
+ obj = x.Resolve(cache)
+ modObj = copy.copy(obj)
+ modObj.modifiers = x.GetModifiers(cache)
+ except BadReferenceError,e:
+ logger.Error("Unknown reference '%s'" % str(e))
+ else:
+ models.append(modObj)
+ except Exception, e:
+ logger.Error(str(e))
+
+ return units
+
class ToolErrorException(Exception):
def __init__(self, s):
Exception.__init__(self,s)
@@ -1364,6 +1446,9 @@
class UninitialisedVariableException(Exception):
pass
+class RecursionException(Exception):
+ pass
+
class Evaluator(object):
"""Determine the values of variables under different Configurations.
Either of specification and buildUnit may be None."""
@@ -1436,7 +1521,6 @@
for k, v in self.dict.items():
if v.find('$(' + k + ')') != -1:
raise RecursionException("Recursion Detected in variable '%s' in configuration '%s' " % (k,configName))
- expanded = "RECURSIVE_INVALID_STRING"
else:
expanded = self.ExpandAll(v, specName, configName)
--- a/sbsv2/raptor/python/raptor_make.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/python/raptor_make.py Mon Mar 22 08:04:37 2010 +0000
@@ -31,11 +31,58 @@
import traceback
import sys
from xml.sax.saxutils import escape
+from xml.sax.saxutils import unescape
class BadMakeEngineException(Exception):
pass
+def XMLEscapeLog(stream):
+ inRecipe = False
+
+ for line in stream:
+ if line.startswith("<recipe"):
+ inRecipe = True
+ elif line.startswith("</recipe"):
+ inRecipe = False
+
+ # unless we are inside a "recipe", any line not starting
+ # with "<" is free text that must be escaped.
+ if inRecipe or line.startswith("<"):
+ yield line
+ else:
+ yield escape(line)
+
+def AnnoFileParseOutput(annofile):
+ af = open(annofile, "r")
+
+ inOutput = False
+ inParseJob = False
+ for line in af:
+ line = line.rstrip("\n\r")
+
+ if not inOutput:
+ if line.startswith("<output>"):
+ inOutput = True
+ yield unescape(line[8:])+'\n'
+ # This is make output so don't unescape it.
+ elif line.startswith('<output src="prog">'):
+ line = line[19:]
+ inOutput = True
+ yield unescape(line)+'\n'
+ else:
+ end_output = line.find("</output>")
+
+ if end_output != -1:
+ line = line[:end_output]
+ inOutput = False
+
+ yield unescape(line)+'\n'
+
+ af.close()
+
+
+
# raptor_make module classes
class MakeEngine(object):
@@ -82,6 +129,25 @@
self.jobsOption = evaluator.Get("jobs")
self.defaultMakeOptions = evaluator.Get("defaultoptions")
+ # Logging
+ # copylogfromannofile means, for emake, that we should ignore
+ # emake's console output and instead extract output from its annotation
+ # file. This is a workaround for a problem where some emake
+ # console output is lost. The annotation file has a copy of this
+ # output in the "parse" job and it turns out to be uncorrupted.
+ self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true")
+ self.annoFileName = None
+
+ if self.copyLogFromAnnoFile:
+ for o in self.raptor.makeOptions:
+ if o.startswith("--emake-annofile="):
+ self.annoFileName = o[17:]
+ self.raptor.Info("annofile: " + o)
+
+ if not self.annoFileName:
+ self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>")
+ self.copyLogFromAnnoFile = False
+
# buffering
self.scrambled = (evaluator.Get("scrambled") == "true")
@@ -479,8 +545,16 @@
# clock skew messages from some build engines scatter their
# output across our xml.
stderrfilename = makefile+'.stderr'
+ stdoutfilename = makefile+'.stdout'
command += " 2>'%s' " % stderrfilename
+ # Keep a copy of the stdout too in the case of using the
+ # annofile - so that we can trap the problem that
+ # makes the copy-log-from-annofile workaround necessary
+ # and perhaps determine when we can remove it.
+ if self.copyLogFromAnnoFile:
+ command += " >'%s' " % stdoutfilename
+
# Substitute the makefile name for any occurrence of #MAKEFILE#
command = command.replace("#MAKEFILE#", str(makefile))
@@ -518,28 +592,23 @@
stream = p.stdout
inRecipe = False
- line = " "
- while line:
- line = stream.readline()
-
- if line.startswith("<recipe"):
- inRecipe = True
- elif line.startswith("</recipe"):
- inRecipe = False
-
- # unless we are inside a "recipe", any line not starting
- # with "<" is free text that must be escaped.
- if inRecipe or line.startswith("<"):
- self.raptor.out.write(line)
- else:
- self.raptor.out.write(escape(line))
+
+ if not self.copyLogFromAnnoFile:
+ for l in XMLEscapeLog(stream):
+ self.raptor.out.write(l)
+
+ returncode = p.wait()
+ else:
+ returncode = p.wait()
- # should be done now
- returncode = p.wait()
+ annofilename = self.annoFileName.replace("#MAKEFILE#", makefile)
+ self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename)
+ try:
+ for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)):
+ self.raptor.out.write(l)
+ except Exception,e:
+ self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e))
- # Report end-time of the build
- self.raptor.InfoEndTime(object_type = "makefile",
- task = "build", key = str(makefile))
# Take all the stderr output that went into the .stderr file
# and put it back into the log, but safely so it can't mess up
@@ -551,6 +620,9 @@
e.close()
except Exception,e:
self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e))
+ # Report end-time of the build
+ self.raptor.InfoEndTime(object_type = "makefile",
+ task = "build", key = str(makefile))
if returncode != 0 and not self.raptor.keepGoing:
self.Tidy()
@@ -560,7 +632,7 @@
self.raptor.Error("Exception '%s' during '%s'", str(e), command)
self.Tidy()
# Still report end-time of the build
- self.raptor.InfoEnd(object_type = "Building", task = "Makefile",
+ self.raptor.InfoEndTime(object_type = "Building", task = "Makefile",
key = str(makefile))
return False
@@ -667,6 +739,8 @@
return False
return True
+
+
# raptor_make module functions
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/annofile2log.py Mon Mar 22 08:04:37 2010 +0000
@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+
+from raptor_tests import SmokeTest
+
+def run():
+ t = SmokeTest()
+ t.id = "43563"
+ t.name = "annofile2log_canned"
+ t.description = "test workaround for log corruption from a make engine whose name begins with 'e'"
+
+ t.usebash = True
+ t.errors = 0
+ t.returncode = 1
+ t.exceptions = 0
+ t.command = "cd smoke_suite/test_resources/annofile2log && ( diff -wB <(python testanno2log.py <(bzip2 -dc scrubbed_ncp_dfs_resource.anno.bz2)) <(bzip2 -dc scrubbed_ncp_dfs_resource.stdout.bz2))"
+
+ t.mustmatch_multiline = [
+ ".*1a2.*" +
+ "Starting build: 488235.{1,3}" +
+ "14009c12884.{1,4}" +
+ "---.{1,4}" +
+ "Finished build: 488235 Duration: 1:15 \(m:s\) Cluster availability: 100%.*"
+ ]
+
+
+ t.run()
+
+ t.print_result()
+ return t
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/keepgoing.py Mon Mar 22 08:04:37 2010 +0000
@@ -0,0 +1,106 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+
+from raptor_tests import SmokeTest
+
+def run():
+ t = SmokeTest()
+ t.description = """Raptor should keep going and build as much as possible with the -k option specified."""
+
+ command = "sbs -b smoke_suite/test_resources/simple/bld.inf -k"
+ config = " --configpath=test/smoke_suite/test_resources/keepgoing"
+ targets = [
+ "$(EPOCROOT)/epoc32/release/armv5/udeb/test.exe",
+ "$(EPOCROOT)/epoc32/release/armv5/udeb/test.exe.map",
+ "$(EPOCROOT)/epoc32/release/armv5/urel/test.exe",
+ "$(EPOCROOT)/epoc32/release/armv5/urel/test.exe.map",
+ "$(EPOCROOT)/epoc32/release/armv5/udeb/test.exe.sym",
+ "$(EPOCROOT)/epoc32/release/armv5/urel/test.exe.sym"
+ ]
+ buildtargets = [
+ "test_/armv5/udeb/test.o",
+ "test_/armv5/urel/test.o",
+ "test_/armv5/udeb/test.o.d",
+ "test_/armv5/udeb/test3.o.d",
+ "test_/armv5/udeb/test4.o.d",
+ "test_/armv5/udeb/test5.o.d",
+ "test_/armv5/udeb/test1.o.d",
+ "test_/armv5/udeb/test6.o.d",
+ "test_/armv5/udeb/test2.o.d",
+ "test_/armv5/udeb/test3.o",
+ "test_/armv5/udeb/test4.o",
+ "test_/armv5/udeb/test5.o",
+ "test_/armv5/udeb/test1.o",
+ "test_/armv5/udeb/test6.o",
+ "test_/armv5/udeb/test2.o",
+ "test_/armv5/urel/test.o.d",
+ "test_/armv5/urel/test3.o.d",
+ "test_/armv5/urel/test4.o.d",
+ "test_/armv5/urel/test5.o.d",
+ "test_/armv5/urel/test1.o.d",
+ "test_/armv5/urel/test6.o.d",
+ "test_/armv5/urel/test2.o.d",
+ "test_/armv5/urel/test3.o",
+ "test_/armv5/urel/test4.o",
+ "test_/armv5/urel/test5.o",
+ "test_/armv5/urel/test1.o",
+ "test_/armv5/urel/test6.o",
+ "test_/armv5/urel/test2.o",
+ "test_/armv5/udeb/test_udeb_objects.via",
+ "test_/armv5/urel/test_urel_objects.via"
+ ]
+
+ # using a non-existent config with -c should build any independent configs
+ t.id = "115a"
+ t.name = "keepgoing_bad_config"
+ t.command = command + " -c armv5 -c armv5.bogus"
+ t.targets = targets
+ t.addbuildtargets("smoke_suite/test_resources/simple/bld.inf", buildtargets)
+ t.mustmatch = ["sbs: error: Unknown build variant 'bogus'"]
+ t.warnings = 0
+ t.errors = 1
+ t.returncode = 1
+ t.run()
+
+ # using groups with bad sub-groups should build any independent groups
+ t.id = "115b"
+ t.name = "keepgoing_bad_subgroup"
+ t.command = command + config + " -c lots_of_products"
+ t.mustmatch = ["Unknown reference 'qwertyuio'",
+ "Unknown reference 'asdfghjkl'",
+ "Unknown reference 'zxcvbnm_p'"]
+ t.warnings = 0
+ t.errors = 3
+ t.returncode = 1
+ t.run()
+
+ # using groups with bad sub-sub-groups should build any independent groups
+ t.id = "115c"
+ t.name = "keepgoing_bad_subsubgroup"
+ t.command = command + config + " -c lots_of_products_2"
+ t.mustmatch = ["Unknown reference 'qwertyuio'",
+ "Unknown reference 'asdfghjkl'",
+ "Unknown reference 'zxcvbnm_p'"]
+ t.warnings = 0
+ t.errors = 3
+ t.returncode = 1
+ t.run()
+
+ # summarise
+ t.id = "115"
+ t.name = "keepgoing"
+ t.print_result()
+ return t
--- a/sbsv2/raptor/test/smoke_suite/parallel_parsing.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/test/smoke_suite/parallel_parsing.py Mon Mar 22 08:04:37 2010 +0000
@@ -54,7 +54,7 @@
warnings = 0
t.id = "104"
- t.name = "parallelparsing"
+ t.name = "parallel_parsing"
t.description = description
t.command = command
t.targets = targets
--- a/sbsv2/raptor/test/smoke_suite/terminal_filter_tests.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/test/smoke_suite/terminal_filter_tests.py Mon Mar 22 08:04:37 2010 +0000
@@ -19,10 +19,10 @@
def run():
t = SmokeTest()
- t.id = "87"
- t.name = "terminal_filter_tests"
- t.description = "terminal_filter_tests: Tests the terminal filter against" \
- + " log files to ensure it 'does the right thing'"
+ t.description = "Tests against log files to ensure it 'does the right thing'"
+
+ t.id = "87a"
+ t.name = "terminal_filter_tests_log"
t.command = "$(SBS_HOME)/test/smoke_suite/test_resources/refilter/testfilterterminal"
t.countmatch = [
# One of each type of error occurs early in the 'sbs' call where there
@@ -36,4 +36,19 @@
]
t.errors = 4
t.run()
+
+ t.id = "87b"
+ t.name = "terminal_filter_tests_configs"
+ t.command = "sbs -b smoke_suite/test_resources/simple/bld.inf"
+ t.countmatch = []
+ t.errors = 0
+ t.mustmatch_singleline = ["built 'armv5_urel'",
+ "built 'armv5_udeb'",
+ "built 'winscw_urel'",
+ "built 'winscw_udeb'" ]
+ t.run()
+
+ t.id = "87"
+ t.name = "terminal_filter_tests"
+ t.print_result()
return t
Binary file sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.anno.bz2 has changed
Binary file sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.stdout.bz2 has changed
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/testanno2log.py Mon Mar 22 08:04:37 2010 +0000
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Component description file
+#
+
+
+import sys
+import os
+sys.path.append(os.path.join(os.environ['SBS_HOME'],"python"))
+
+from raptor_make import XMLEscapeLog
+from raptor_make import AnnoFileParseOutput
+
+
+retcode=0
+
+
+annofile = sys.argv[1]
+
+sys.stdout.write("<build>\n")
+try:
+ for l in XMLEscapeLog(AnnoFileParseOutput(annofile)):
+ sys.stdout.write(l)
+
+except Exception,e:
+ sys.stderr.write("error: " + str(e) + "\n")
+ retcode = 1
+sys.stdout.write("</build>\n")
+
+sys.exit(retcode)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/keepgoing/groups.xml Mon Mar 22 08:04:37 2010 +0000
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<build xmlns="http://symbian.com/xml/build"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://symbian.com/xml/build build/2_0.xsd">
+
+ <!-- This group deliberately contains some non-existent products
+ -->
+ <group name="lots_of_products">
+ <groupRef ref='armv5'/>
+ <groupRef ref='armv5' mod="qwertyuio"/>
+ <groupRef ref='armv5' mod="asdfghjkl"/>
+ <groupRef ref='armv5' mod="zxcvbnm_p"/>
+ </group>
+
+ <!-- This group just contains the one above to check multiple levels
+ -->
+ <group name="lots_of_products_2">
+ <groupRef ref="lots_of_products"/>
+ </group>
+
+</build>
--- a/sbsv2/raptor/test/unit_suite/raptor_data_unit.py Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/test/unit_suite/raptor_data_unit.py Mon Mar 22 08:04:37 2010 +0000
@@ -430,8 +430,105 @@
self.failUnless(self.checkForParam(p, "D", None))
f = extended.GetFLMIncludePath(cache)
self.assertEqual(f.File(), "base.flm")
+
+ def testGetBuildUnits(self):
+ r = raptor.Raptor()
+
+ # <group name="g1">
+ g1 = raptor_data.Group("g1")
+ r.cache.AddGroup(g1)
+
+ # <groupRef ref="g2" mod="A.B"/>
+ g2a = raptor_data.GroupRef()
+ g2a.SetProperty("ref", "g2")
+ g2a.SetProperty("mod", "A.B")
+ g1.AddChild(g2a)
+
+ # <groupRef ref="g2" mod="C.D"/>
+ g2b = raptor_data.GroupRef()
+ g2b.SetProperty("ref", "g2")
+ g2b.SetProperty("mod", "C.D")
+ g1.AddChild(g2b)
+
+ # <group name="g2">
+ g2 = raptor_data.Group("g2")
+ r.cache.AddGroup(g2)
+
+ # <varRef ref="V" mod="E.F"/>
+ v2 = raptor_data.VariantRef()
+ v2.SetProperty("ref", "V")
+ v2.SetProperty("mod", "E.F")
+ g2.AddChild(v2)
+
+ # <varRef ref="V" mod="G.H"/>
+ v3 = raptor_data.VariantRef()
+ v3.SetProperty("ref", "V")
+ v3.SetProperty("mod", "G.H")
+ g2.AddChild(v3)
+
+ # <aliasRef ref="X" mod="I.J"/>
+ v4 = raptor_data.AliasRef()
+ v4.SetProperty("ref", "X")
+ v4.SetProperty("mod", "I.J")
+ g2.AddChild(v4)
+
+ # <aliasRef ref="X" mod="K.L"/>
+ v5 = raptor_data.AliasRef()
+ v5.SetProperty("ref", "X")
+ v5.SetProperty("mod", "K.L")
+ g2.AddChild(v5)
+
+ r.cache.AddVariant(raptor_data.Variant("A"))
+ r.cache.AddVariant(raptor_data.Variant("B"))
+ r.cache.AddVariant(raptor_data.Variant("C"))
+ r.cache.AddVariant(raptor_data.Variant("D"))
+ r.cache.AddVariant(raptor_data.Variant("E"))
+ r.cache.AddVariant(raptor_data.Variant("F"))
+ r.cache.AddVariant(raptor_data.Variant("G"))
+ r.cache.AddVariant(raptor_data.Variant("H"))
+ r.cache.AddVariant(raptor_data.Variant("I"))
+ r.cache.AddVariant(raptor_data.Variant("J"))
+ r.cache.AddVariant(raptor_data.Variant("K"))
+ r.cache.AddVariant(raptor_data.Variant("L"))
+
+ r.cache.AddVariant(raptor_data.Variant("V"))
+
+ # <alias name="X" meaning="A.B.C.D.E.F.G.H/>
+ alias = raptor_data.Alias("X")
+ alias.SetProperty("meaning", "A.B.C.D.E.F.G.H")
+ r.cache.AddAlias(alias)
+
+ r.cache.AddVariant(raptor_data.Variant("Y"))
+ r.cache.AddVariant(raptor_data.Variant("Z"))
-
+ units = raptor_data.GetBuildUnits(["g1.Y", "g1.Z"], r.cache, r)
+
+ # <group name="g1">
+ # <groupRef ref="g2" mod="A.B"/> g2.A.B
+ # <groupRef ref="g2" mod="C.D"/> g2.C.D
+ # </group>
+ # <group name="g2">
+ # <varRef ref="V" mod="E.F"/> V.E.F
+ # <varRef ref="V" mod="G.H"/> V.G.H
+ # <aliasRef ref="X" mod="I.J"/> X.I.J
+ # <aliasRef ref="X" mod="K.L"/> X.K.L
+ # </group>
+ # <alias name="X" meaning="A.B.C.D.E.F.G.H/>
+ #
+ expected = [ "VEFABY", "VGHABY", "ABCDEFGHIJABY", "ABCDEFGHKLABY",
+ "VEFCDY", "VGHCDY", "ABCDEFGHIJCDY", "ABCDEFGHKLCDY",
+ "VEFABZ", "VGHABZ", "ABCDEFGHIJABZ", "ABCDEFGHKLABZ",
+ "VEFCDZ", "VGHCDZ", "ABCDEFGHIJCDZ", "ABCDEFGHKLCDZ" ]
+
+ self.failUnlessEqual(len(units), len(expected))
+
+ for u in units:
+ vars = "".join([v.name for v in u.variants])
+ self.failUnless(vars in expected, vars + " was not expected")
+ expected.remove(vars)
+
+ self.failUnless(len(expected) == 0, str(expected) + " not found")
+
# run all the tests
from raptor_tests import SmokeTest
--- a/sbsv2/raptor/util/talon/talon.c Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/util/talon/talon.c Mon Mar 22 08:04:37 2010 +0000
@@ -586,7 +586,7 @@
if (p->returncode != 0)
{
- char *exitstr = retries > 0 ? "retry" : "failed";
+ char *exitstr = force_success ? "failed" : retries > 0 ? "retry" : "failed";
snprintf(status, STATUS_STRMAX - 1, "\n<status exit='%s' code='%d' attempt='%d'%s%s />", exitstr, p->returncode, attempt, flagsstr, reasonstr );
} else {
snprintf(status, STATUS_STRMAX - 1, "\n<status exit='ok' attempt='%d'%s%s />", attempt, flagsstr, reasonstr );
--- a/sbsv2/raptor/util/talon/tests/config.sh Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/util/talon/tests/config.sh Mon Mar 22 08:04:37 2010 +0000
@@ -16,9 +16,19 @@
#
# set up the environment for some talon tests.
+eval `$SBS_HOME/bin/gethost.sh -e`
+
+if [[ "$HOSTPLATFORM" =~ "win" ]]; then
+TEST_SHELL=$(cygpath -w $SBS_HOME/win32/bin/talon.exe)
+TEST_TALON_SHELL=$(cygpath -w $SBS_CYGWIN/bin/bash.exe)
+else
+TEST_SHELL=$SBS_HOME/$HOSTPLATFORM_DIR/bin/talon
+TEST_TALON_SHELL=/bin/bash
+fi
+
cat >settings.mk <<-endofsettings
- SHELL:=$(cygpath -w $SBS_HOME/win32/bin/talon.exe)
- TALON_SHELL:=$(cygpath -w $SBS_CYGWIN/bin/bash.exe)
+ SHELL:=$TEST_SHELL
+ TALON_SHELL:=$TEST_TALON_SHELL
TALON_BUILDID:=100
TALON_DEBUG:=""
export TALON_SHELL TALON_BUILDID TALON_DEBUG
--- a/sbsv2/raptor/util/talon/tests/t4.mk Wed Mar 17 18:11:44 2010 +0000
+++ b/sbsv2/raptor/util/talon/tests/t4.mk Mon Mar 22 08:04:37 2010 +0000
@@ -23,6 +23,11 @@
$(info testing timeouts)
$(info )
-all:
+.PHONY: all passed
+
+all: passed
+ @echo "t4-PASSED"
+
+passed:
@|PLATFORM=armv5;MMP=barney.mmp;BLDINF=somebld.inf;|echo "Started a slow command under talon (attempt $$TALON_ATTEMPT):";echo "SHELL=$$SHELL";if (( $$TALON_ATTEMPT <4 )); then echo sleeping 6; sleep 6; echo "hi"; else echo "Not sleeping this time"; fi; echo "this should not appear in the recipe tags unless you try 4 times."
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/util/talon/tests/t6.mk Mon Mar 22 08:04:37 2010 +0000
@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+include settings.mk
+
+# Making sure that forcesuccess works.
+
+TALON_RECIPEATTRIBUTES:=flags='$$TALON_FLAGS'
+TALON_RETRIES:=1
+
+export TALON_RECIPEATTRIBUTES TALON_RETRIES
+
+.PHONY: all fred
+
+all: fred
+ @echo "t6-PASSED"
+
+fred:
+ |TALON_FLAGS=FORCESUCCESS;|echo "Forcesuccess'd command"; exit 1
+
+
Binary file sbsv2/raptor/win32/bin/talon.exe has changed