--- a/sbsv2/raptor/RELEASE-NOTES.txt Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/RELEASE-NOTES.txt Thu Mar 18 14:00:53 2010 +0000
@@ -1,7 +1,12 @@
Release Notes for Symbian Build System v2
next version
+- Fix: Workaround for emake engine log corruption when clockskew errors occur (annofile2log).
+ Allow Raptor to obtain log from emake annotation file where it is uncorrupted. A new
+ Make engine option "copyannofile2log" enables/disables this mode for emake. If this option is disabled
+ or if no annotation file is specified for the build then Raptor reads logs directly as normal.
- SF Bug 2125 - [Raptor] - tracecompiler what output incorrect if mmp basename contains '.' e.g. fred.prd.mmp
+- SF Bug 2191 - [Raptor] - When forcesuccess is enabled, exit status for a failed recipe is "retry" but should be "failed"
- Fix: extend tracecompiler tests to Linux
- Fix: Amendment to SF Bug 1511 fix - removal of blanked DEFFILE keyword from e32abiv2ani.flm
- Fix: improve robustness to bad -c options
--- a/sbsv2/raptor/lib/config/make.xml Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/lib/config/make.xml Thu Mar 18 14:00:53 2010 +0000
@@ -47,6 +47,9 @@
<!-- is the text output with -j buffered or scrambled? -->
<set name="scrambled" value="true"/>
+
+ <!-- workaround for damaged log output from emake -->
+ <set name="copylogfromannofile" value="false"/>
</var>
<alias name="make" meaning="make_engine"/>
@@ -67,6 +70,9 @@
<set name="build" value="$(EMAKE) HAVE_ORDERONLY= -r"/>
<set name="scrambled" value="false"/>
<set name='TALON_DESCRAMBLE' value=''/>
+
+ <!-- workaround for damaged log output from emake -->
+ <set name="copylogfromannofile" value="true"/>
</var>
<alias name="emake" meaning="emake_engine"/>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/python/plugins/filter_bz2log.py Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,88 @@
+# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Compress the full Raptor log file using the BZip2 algorithm, maximum compression.
+#
+#
+
+import os
+import sys
+import raptor
+import filter_interface
+import bz2
+
+class StringListCompressor(object):
+ def __init__(self, complevel=5, filename="file.log.bz2"):
+ self.compressor = bz2.BZ2Compressor(complevel)
+ self.stringlist = []
+ self.outputopenedok = False
+ self.filename = filename
+ try:
+ self.fh = open(self.filename, "wb")
+ self.outputopenedok = True
+ except:
+ self.outputopenedok = False
+
+ def write(self, data):
+ if self.outputopenedok:
+ compresseddata = self.compressor.compress(data)
+ self.fh.write(compresseddata)
+
+ def __del__(self):
+ if self.outputopenedok:
+ compresseddata = self.compressor.flush()
+ self.fh.write(compresseddata)
+ self.fh.close()
+
+class Bz2log(filter_interface.Filter):
+ def __init__(self):
+ self.__inRecipe = False
+ self.compressor = None
+
+ def open(self, raptor_instance):
+ """Open a log file for the various I/O methods to write to."""
+
+ if raptor_instance.logFileName == None:
+ self.out = sys.stdout # Default to stdout if no log file is given
+ else:
+ logname = str(raptor_instance.logFileName.path.replace("%TIME", raptor_instance.timestring))
+
+ # Ensure that filename has the right extension; append ".bz2" if required
+ if not logname.lower().endswith(".bz2"):
+ logname += ".bz2"
+
+ try:
+ dirname = str(raptor_instance.logFileName.Dir())
+ if dirname and not os.path.isdir(dirname):
+ os.makedirs(dirname)
+ except:
+ self.formatError("cannot create directory %s", dirname)
+ return False
+
+ # Use highest compression level 9 which corresponds to a 900KB dictionary
+ self.compressor = StringListCompressor(9, logname)
+ if not self.compressor.outputopenedok:
+ self.out = None
+ self.formatError("failed to initialise compression routines." )
+ return False
+ return True
+
+ def write(self, data):
+ """Write data compressed log"""
+ if self.compressor:
+ self.compressor.write(data)
+ return True
+
+ def close(self):
+ """Close the log file"""
+ return True
--- a/sbsv2/raptor/python/raptor_make.py Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/python/raptor_make.py Thu Mar 18 14:00:53 2010 +0000
@@ -31,11 +31,58 @@
import traceback
import sys
from xml.sax.saxutils import escape
+from xml.sax.saxutils import unescape
class BadMakeEngineException(Exception):
pass
+def XMLEscapeLog(stream):
+ inRecipe = False
+
+ for line in stream:
+ if line.startswith("<recipe"):
+ inRecipe = True
+ elif line.startswith("</recipe"):
+ inRecipe = False
+
+ # unless we are inside a "recipe", any line not starting
+ # with "<" is free text that must be escaped.
+ if inRecipe or line.startswith("<"):
+ yield line
+ else:
+ yield escape(line)
+
+def AnnoFileParseOutput(annofile):
+ af = open(annofile, "r")
+
+ inOutput = False
+ inParseJob = False
+ for line in af:
+ line = line.rstrip("\n\r")
+
+ if not inOutput:
+ if line.startswith("<output>"):
+ inOutput = True
+ yield unescape(line[8:])+'\n'
+ # This is make output so don't unescape it.
+ elif line.startswith('<output src="prog">'):
+ line = line[19:]
+ inOutput = True
+ yield unescape(line)+'\n'
+ else:
+ end_output = line.find("</output>")
+
+ if end_output != -1:
+ line = line[:end_output]
+ inOutput = False
+
+ yield unescape(line)+'\n'
+
+ af.close()
+
+
+
# raptor_make module classes
class MakeEngine(object):
@@ -82,6 +129,25 @@
self.jobsOption = evaluator.Get("jobs")
self.defaultMakeOptions = evaluator.Get("defaultoptions")
+ # Logging
+ # copylogfromannofile means, for emake, that we should ignore
+ # emake's console output and instead extract output from its annotation
+ # file. This is a workaround for a problem where some emake
+ # console output is lost. The annotation file has a copy of this
+ # output in the "parse" job and it turns out to be uncorrupted.
+ self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true")
+ self.annoFileName = None
+
+ if self.copyLogFromAnnoFile:
+ for o in self.raptor.makeOptions:
+ if o.startswith("--emake-annofile="):
+ self.annoFileName = o[17:]
+ self.raptor.Info("annofile: " + o)
+
+ if not self.annoFileName:
+ self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>")
+ self.copyLogFromAnnoFile = False
+
# buffering
self.scrambled = (evaluator.Get("scrambled") == "true")
@@ -479,8 +545,16 @@
# clock skew messages from some build engines scatter their
# output across our xml.
stderrfilename = makefile+'.stderr'
+ stdoutfilename = makefile+'.stdout'
command += " 2>'%s' " % stderrfilename
+ # Keep a copy of the stdout too in the case of using the
+ # annofile - so that we can trap the problem that
+ # makes the copy-log-from-annofile workaround necessary
+ # and perhaps determine when we can remove it.
+ if self.copyLogFromAnnoFile:
+ command += " >'%s' " % stdoutfilename
+
# Substitute the makefile name for any occurrence of #MAKEFILE#
command = command.replace("#MAKEFILE#", str(makefile))
@@ -518,28 +592,23 @@
stream = p.stdout
inRecipe = False
- line = " "
- while line:
- line = stream.readline()
-
- if line.startswith("<recipe"):
- inRecipe = True
- elif line.startswith("</recipe"):
- inRecipe = False
-
- # unless we are inside a "recipe", any line not starting
- # with "<" is free text that must be escaped.
- if inRecipe or line.startswith("<"):
- self.raptor.out.write(line)
- else:
- self.raptor.out.write(escape(line))
+
+ if not self.copyLogFromAnnoFile:
+ for l in XMLEscapeLog(stream):
+ self.raptor.out.write(l)
+
+ returncode = p.wait()
+ else:
+ returncode = p.wait()
- # should be done now
- returncode = p.wait()
+ annofilename = self.annoFileName.replace("#MAKEFILE#", makefile)
+ self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename)
+ try:
+ for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)):
+ self.raptor.out.write(l)
+ except Exception,e:
+ self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e))
- # Report end-time of the build
- self.raptor.InfoEndTime(object_type = "makefile",
- task = "build", key = str(makefile))
# Take all the stderr output that went into the .stderr file
# and put it back into the log, but safely so it can't mess up
@@ -551,6 +620,9 @@
e.close()
except Exception,e:
self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e))
+ # Report end-time of the build
+ self.raptor.InfoEndTime(object_type = "makefile",
+ task = "build", key = str(makefile))
if returncode != 0 and not self.raptor.keepGoing:
self.Tidy()
@@ -560,7 +632,7 @@
self.raptor.Error("Exception '%s' during '%s'", str(e), command)
self.Tidy()
# Still report end-time of the build
- self.raptor.InfoEnd(object_type = "Building", task = "Makefile",
+ self.raptor.InfoEndTime(object_type = "Building", task = "Makefile",
key = str(makefile))
return False
@@ -667,6 +739,8 @@
return False
return True
+
+
# raptor_make module functions
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/annofile2log.py Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+
+from raptor_tests import SmokeTest
+
+def run():
+ t = SmokeTest()
+ t.id = "43563"
+ t.name = "annofile2log_canned"
+ t.description = "test workaround for log corruption from a make engine whose name begins with 'e'"
+
+ t.usebash = True
+ t.errors = 0
+ t.returncode = 1
+ t.exceptions = 0
+ t.command = "cd smoke_suite/test_resources/annofile2log && ( diff -wB <(python testanno2log.py <(bzip2 -dc scrubbed_ncp_dfs_resource.anno.bz2)) <(bzip2 -dc scrubbed_ncp_dfs_resource.stdout.bz2))"
+
+ t.mustmatch_multiline = [
+ ".*1a2.*" +
+ "Starting build: 488235.{1,3}" +
+ "14009c12884.{1,4}" +
+ "---.{1,4}" +
+ "Finished build: 488235 Duration: 1:15 \(m:s\) Cluster availability: 100%.*"
+ ]
+
+
+ t.run()
+
+ t.print_result()
+ return t
Binary file sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.anno.bz2 has changed
Binary file sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/scrubbed_ncp_dfs_resource.stdout.bz2 has changed
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/smoke_suite/test_resources/annofile2log/testanno2log.py Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Component description file
+#
+
+
+import sys
+import os
+sys.path.append(os.path.join(os.environ['SBS_HOME'],"python"))
+
+from raptor_make import XMLEscapeLog
+from raptor_make import AnnoFileParseOutput
+
+
+retcode=0
+
+
+annofile = sys.argv[1]
+
+sys.stdout.write("<build>\n")
+try:
+ for l in XMLEscapeLog(AnnoFileParseOutput(annofile)):
+ sys.stdout.write(l)
+
+except Exception,e:
+ sys.stderr.write("error: " + str(e) + "\n")
+ retcode = 1
+sys.stdout.write("</build>\n")
+
+sys.exit(retcode)
--- a/sbsv2/raptor/util/talon/talon.c Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/util/talon/talon.c Thu Mar 18 14:00:53 2010 +0000
@@ -586,7 +586,7 @@
if (p->returncode != 0)
{
- char *exitstr = retries > 0 ? "retry" : "failed";
+ char *exitstr = force_success ? "failed" : retries > 0 ? "retry" : "failed";
snprintf(status, STATUS_STRMAX - 1, "\n<status exit='%s' code='%d' attempt='%d'%s%s />", exitstr, p->returncode, attempt, flagsstr, reasonstr );
} else {
snprintf(status, STATUS_STRMAX - 1, "\n<status exit='ok' attempt='%d'%s%s />", attempt, flagsstr, reasonstr );
--- a/sbsv2/raptor/util/talon/tests/config.sh Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/util/talon/tests/config.sh Thu Mar 18 14:00:53 2010 +0000
@@ -16,9 +16,19 @@
#
# set up the environment for some talon tests.
+eval `$SBS_HOME/bin/gethost.sh -e`
+
+if [[ "$HOSTPLATFORM" =~ "win" ]]; then
+TEST_SHELL=$(cygpath -w $SBS_HOME/win32/bin/talon.exe)
+TEST_TALON_SHELL=$(cygpath -w $SBS_CYGWIN/bin/bash.exe)
+else
+TEST_SHELL=$SBS_HOME/$HOSTPLATFORM_DIR/bin/talon
+TEST_TALON_SHELL=/bin/bash
+fi
+
cat >settings.mk <<-endofsettings
- SHELL:=$(cygpath -w $SBS_HOME/win32/bin/talon.exe)
- TALON_SHELL:=$(cygpath -w $SBS_CYGWIN/bin/bash.exe)
+ SHELL:=$TEST_SHELL
+ TALON_SHELL:=$TEST_TALON_SHELL
TALON_BUILDID:=100
TALON_DEBUG:=""
export TALON_SHELL TALON_BUILDID TALON_DEBUG
--- a/sbsv2/raptor/util/talon/tests/t4.mk Tue Mar 16 17:04:21 2010 +0000
+++ b/sbsv2/raptor/util/talon/tests/t4.mk Thu Mar 18 14:00:53 2010 +0000
@@ -23,6 +23,11 @@
$(info testing timeouts)
$(info )
-all:
+.PHONY: all passed
+
+all: passed
+ @echo "t4-PASSED"
+
+passed:
@|PLATFORM=armv5;MMP=barney.mmp;BLDINF=somebld.inf;|echo "Started a slow command under talon (attempt $$TALON_ATTEMPT):";echo "SHELL=$$SHELL";if (( $$TALON_ATTEMPT <4 )); then echo sleeping 6; sleep 6; echo "hi"; else echo "Not sleeping this time"; fi; echo "this should not appear in the recipe tags unless you try 4 times."
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/util/talon/tests/t6.mk Thu Mar 18 14:00:53 2010 +0000
@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+include settings.mk
+
+# Making sure that forcesuccess works.
+
+TALON_RECIPEATTRIBUTES:=flags='$$TALON_FLAGS'
+TALON_RETRIES:=1
+
+export TALON_RECIPEATTRIBUTES TALON_RETRIES
+
+.PHONY: all fred
+
+all: fred
+ @echo "t6-PASSED"
+
+fred:
+ |TALON_FLAGS=FORCESUCCESS;|echo "Forcesuccess'd command"; exit 1
+
+
Binary file sbsv2/raptor/win32/bin/talon.exe has changed