New script which parses raptor logs directly to produce a BRAG summary.
authorSimon Howkins <simonh@symbian.org>
Wed, 11 Nov 2009 13:22:02 +0000
changeset 753 5069de517698
parent 752 1f07674ec99f
child 754 79bd241158d3
New script which parses raptor logs directly to produce a BRAG summary. New version of XML structure accommodates causes of failure as well as the effect of the failure, and deals with excessive error messages. New structure also avoids a potential problem (bug?, questionable feature?) that both IE and firefox exhibit. Some simplifications of brag utilities module.
common/tools/brag/brag.xsl
common/tools/brag/logToBRAG.pl
common/tools/brag/raptorToBRAG.pl
common/tools/brag/sbsToBrag.pl
common/tools/brag/toBrag.pm
common/tools/brag/yarpToBRAG.pl
--- a/common/tools/brag/brag.xsl	Tue Nov 10 16:52:27 2009 +0000
+++ b/common/tools/brag/brag.xsl	Wed Nov 11 13:22:02 2009 +0000
@@ -138,7 +138,13 @@
 				<ul>
 				<xsl:for-each select="/buildStatus/phase/step/failures[@level = $severity]/failure[@package = $package]">
 					<xsl:sort select="@package"/>
-					<li><xsl:value-of select="@package"/>: <xsl:value-of select="."/></li>
+					<li><xsl:value-of select="effect"/></li>
+					<xsl:if test="@unreported_causes != '0'">
+						<br/>(Too much text to show everything; <xsl:value-of select="@unreported_causes"/> lines not shown.)
+					</xsl:if>
+					<xsl:for-each select="causes">
+						<pre><xsl:value-of select="."/></pre>
+					</xsl:for-each>
 				</xsl:for-each>
 				</ul>
 				</dd></dl>
--- a/common/tools/brag/logToBRAG.pl	Tue Nov 10 16:52:27 2009 +0000
+++ b/common/tools/brag/logToBRAG.pl	Wed Nov 11 13:22:02 2009 +0000
@@ -100,7 +100,7 @@
 		{
 			last if $rule->{severity} eq "ignore";
 			# We found a match
-			my $failure = bless{ Kids => [ bless { Text => $line }, "Characters" ] }, "failure";
+			my $failure = bless{ Kids => [ bless { Kids => [ bless { Text => $line }, "Characters" ]}, "effect"] }, "failure";
 			# Ensure we have a <failures> tag for this severity
 			if (!exists $severityShortcut->{$rule->{severity}})
 			{
--- a/common/tools/brag/raptorToBRAG.pl	Tue Nov 10 16:52:27 2009 +0000
+++ b/common/tools/brag/raptorToBRAG.pl	Wed Nov 11 13:22:02 2009 +0000
@@ -97,8 +97,7 @@
 
 	# Now create the failure itself, and add it to this failure set
 	my $failureItem = bless {
-#		href => "",
-		Kids => [ bless { Text => $failure->{subcategory} }, "Characters" ]
+		Kids => [ bless { Kids => [ bless { Text => $failure->{subcategory} }, "Characters" ]}, "effect" ],
 	}, "failure";
 	if ($failure->{component})
 	{
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/brag/sbsToBrag.pl	Wed Nov 11 13:22:02 2009 +0000
@@ -0,0 +1,153 @@
+#!perl -w
+#
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Generate the BRAG-compatible XML summary of the Raptor log from the CSV output of the raptor parser
+
+use strict;
+
+use FindBin;
+use lib "$FindBin::Bin";
+
+use XML::Parser;
+#use Getopt::Long;
+
+use ToBrag;
+
+my $raptorLogGlob = shift or die "First argument must be raptor XML log to read\n";
+shift and die "Only one argument please\n";
+
+@ARGV = glob $raptorLogGlob;
+
+# Start to build structure to be output as XML (same format as XML::Parser would create for us)
+my ($doc, $buildStatus) = ToBrag::createDocumentAndRoot("buildStatus");
+# Obtain a phase object
+my $buildPhase = ToBrag::ensureChild($buildStatus, "phase", "name", "Build");
+
+# Parse the Raptor logs
+# (Use XML::Parser in "Stream" mode so we don't have to hold all the data in memory at the same time)
+my $xml = XML::Parser->new(Style => "Stream", Pkg => "main");
+
+foreach my $log (@ARGV)
+{
+	$xml->parsefile($log);
+}
+
+print "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+print "<?xml-stylesheet type='text/xsl' href='brag.xsl'?>\n";
+ToBrag::printTree($doc->[0]);
+print "\n";
+
+exit (0);
+
+my $context;
+sub StartTag
+{
+	my $expat = shift;
+	my $tagName = shift;
+	my %attrib = %_;
+
+	if ($tagName eq "recipe")
+	{
+		$context = {%attrib};
+	}
+	elsif ($tagName eq "status")
+	{
+		$context->{"exit"} = $attrib{"exit"};
+	}
+}
+sub EndTag
+{
+	my $expat = shift;
+	my $tagName = shift;
+
+	if ($tagName eq "recipe")
+	{
+		die unless $context;
+
+		if ($context->{"exit"} ne "ok")
+		{
+			# Create a more readable error message
+			my %errorIdToDetail = (
+				tem => {message => "Failed to execute '$context->{source}' invoked via $context->{bldinf}", severity => "major"},
+				msvctoolscompile => {message => "Failed to compile $context->{source}", severity => "minor"},
+				compile => {message => "Failed to compile $context->{source}", severity => "minor"},
+				compile2object => {message => "Failed to compile $context->{source}", severity => "minor"},
+				win32compile2object => {message => "Failed to compile $context->{source}", severity => "minor"},
+				tools2lib => {message => "Failed to build library $context->{target}", severity => "minor"},
+				ar => {message => "Failed to build library $context->{target}", severity => "minor"},
+				win32archive => {message => "Failed to build library $context->{target}", severity => "minor"},
+				"link" => {message => "Failed to create symbols for $context->{target}", severity => "minor"},
+				postlink => {message => "Failed to link $context->{target}", severity => "minor"},
+				win32stageonelink => {message => "Failed to link $context->{target} (stage 1)", severity => "minor"},
+				win32stagetwolink => {message => "Failed to link $context->{target}", severity => "minor"},
+				win32simplelink => {message => "Failed to link $context->{target}", severity => "minor"},
+				win32processexports => {message => "Failed to export $context->{source} to $context->{target}", severity => "minor"},
+				tracecompile => {message => "Trace compile failure for $context->{target}", severity => "unknown"},
+				extension_makefile => {message => "Failed within an extension makefile connected to $context->{bldinf}", severity => "major"},
+			);
+#			die $context->{name} unless exists $errorIdToDetail{$context->{name}};
+
+			my $message = $errorIdToDetail{$context->{name}}->{message} || "Unknown failure tag '$context->{name}' ($context->{source} -> $context->{target})";
+			$context->{severity} = $errorIdToDetail{$context->{name}}->{severity} || "unknown";
+
+			# Obtain a step object
+			my $step = ToBrag::ensureStep($buildPhase, $context->{config});
+			# Also create empty <failures> tags with severities in a sensible order
+			ToBrag::ensureFailureSet($step, "critical");
+			ToBrag::ensureFailureSet($step, "major");
+			ToBrag::ensureFailureSet($step, "minor");
+			# Obtain a failures object
+			my $failureSet = ToBrag::ensureFailureSet($step, $context->{severity});
+
+			# Now create the failure itself, and add it to this failure set
+			my $failureItem = bless {
+				Kids => [ bless { Kids => [ bless { Text => $message }, "Characters" ]}, "effect" ],
+			}, "failure";
+			if ($context->{component})
+			{
+				$context->{bldinf} =~ s{^\w:(/sf/.*?/.*?)/.*$}{$1};
+				$failureItem->{package} = $context->{bldinf};
+			}
+			my @causes = grep { $_ && ! m/^\+ / } split("\n", $context->{Chars});
+			@causes = map { "  $_" } @causes;
+			if (@causes)
+			{
+				my @reportedCauses = @causes[0 .. min($#causes, 49)];
+				my $causesItem = bless {
+					Kids => [ bless { Text => join "\n", @reportedCauses }, "Characters" ]
+				}, "causes";
+				push @{$failureItem->{Kids}}, $causesItem;
+				my $unreportedCauses = scalar @causes - scalar @reportedCauses;
+				$failureItem->{unreported_causes} = $unreportedCauses;
+			}
+			push @{$failureSet->{Kids}}, $failureItem, $ToBrag::xmlNewline;
+		}
+		
+		$context = undef;
+	}
+}
+sub Text
+{
+	s/^\n*//;
+	if ($context)
+	{
+		$context->{Chars} .= $_;
+	}
+}
+
+sub min
+{
+	return ($_[0] < $_[1]) ? $_[0] : $_[1] ;
+}
+
--- a/common/tools/brag/toBrag.pm	Tue Nov 10 16:52:27 2009 +0000
+++ b/common/tools/brag/toBrag.pm	Wed Nov 11 13:22:02 2009 +0000
@@ -21,6 +21,19 @@
 # A useful constant
 our $xmlNewline = bless { Text => "\n" }, "Characters";
 
+sub createDocumentAndRoot
+{
+	my $rootTag = shift;
+
+	my $root = bless
+	{
+		Kids =>
+		[ $ToBrag::xmlNewline ]
+	}, $rootTag;
+
+	return [$root], $root;
+}
+
 sub createBuildStatus
 {
 	return [
@@ -55,17 +68,7 @@
 	my $phase = shift;
 	my $stepName = shift;
 
-	my ($step) = grep { ref $_ eq "step" && $_->{name} eq $stepName } @{$phase->{Kids}};
-	unless ($step)
-	{
-		$step = bless
-		{
-			name => $stepName,
-			Kids => [ $ToBrag::xmlNewline ]
-		}, "step";
-		push @{$phase->{Kids}}, $step, $ToBrag::xmlNewline;
-	}
-	return $step;
+	return ensureChild($phase, "step", "name", $stepName);
 }
 
 sub ensureFailureSet
@@ -73,17 +76,27 @@
 	my $step = shift;
 	my $level = shift;
 
-	my ($failureSet) = grep { ref $_ eq "failures" && $_->{level} eq $level } @{$step->{Kids}};
-	unless ($failureSet)
+	return ensureChild($step, "failures", "level", $level);
+}
+
+sub ensureChild
+{
+	my $parent = shift;
+	my $childName = shift;
+	my $childAttr = shift;
+	my $childAttrValue = shift;
+
+	my ($child) = grep { ref $_ eq $childName && $_->{$childAttr} eq $childAttrValue } @{$parent->{Kids}};
+	unless ($child)
 	{
-		$failureSet = bless
+		$child = bless
 		{
-			level => $level,
+			$childAttr => $childAttrValue,
 			Kids => [ $ToBrag::xmlNewline ]
-		}, "failures";
-		push @{$step->{Kids}}, $failureSet, $ToBrag::xmlNewline;
+		}, $childName;
+		push @{$parent->{Kids}}, $child, $ToBrag::xmlNewline;
 	}
-	return $failureSet;
+	return $child;
 }
 
 # Prints out the XML tree to STDOUT
@@ -96,7 +109,14 @@
 	$tagName =~ s{^main::}{};
 	if ($tagName eq "Characters")
 	{
-		print $tree->{Text};
+		if ($tree->{Text} =~ m{[<>&]})
+		{
+			print "<![CDATA[$tree->{Text}]]>";
+		}
+		else
+		{
+			print $tree->{Text};
+		}
 		return;
 	}
 	
@@ -104,7 +124,7 @@
 
 	foreach my $attr (
 		sort {
-			my $order = "name level start stop href";
+			my $order = "name level start stop href package effect";
 			my $ixA = index $order, $a;
 			my $ixB = index $order, $b;
 			die "$a $b" if $ixA + $ixB == -2;
--- a/common/tools/brag/yarpToBRAG.pl	Tue Nov 10 16:52:27 2009 +0000
+++ b/common/tools/brag/yarpToBRAG.pl	Wed Nov 11 13:22:02 2009 +0000
@@ -121,7 +121,7 @@
 		my $failureItem = bless {
 #			href => "",
 			"package" => $failure->{package},
-			Kids => [ bless { Text => $message }, "Characters" ],
+			Kids => [ bless { Kids => [ bless { Text => $message }, "Characters" ]}, "effect"],
 		}, "failure";
 		push @{$failureSet->{Kids}}, $failureItem, $ToBrag::xmlNewline;
 	}