Merge Tags
authorMattD <mattd@symbian.org>
Mon, 16 Nov 2009 19:27:25 +0000
changeset 770 27a55a8e0baf
parent 769 81fe60d54485 (diff)
parent 761 3901909be1ab (current diff)
child 771 6d4f56b5730d
Merge Tags
--- a/common/build.postbuild.xml	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/build.postbuild.xml	Mon Nov 16 19:27:25 2009 +0000
@@ -323,8 +323,7 @@
         </exec>
     </target>
 
-    <target name="sf-run-analysis-raptor">
-      <!-- Process raptor logs to remove non-well-formedness  -->
+    <target name="sf-run-analysis-raptor">            
       <echo message="Preprocessing *_compile.log files"/>
       <delete file="${build.log.dir}/analysis/${build.id}_preprocess.log"/>
       <for param="logfile">
@@ -344,55 +343,23 @@
         </sequential>
       </for>
       
-      <!-- Cook the processed raptor logs to produce something in the right format for the BRAG system -->
-      <mkdir dir="${build.log.dir}/summary/"/>
-      <exec executable="perl" output="${build.log.dir}/summary/sbs_BRAG.xml" logError="yes" failonerror="false">
-        <arg value="${sf.common.config.dir}/tools/brag/sbsToBRAG.pl"/>
-        <arg value="${build.log.dir}/analysis/*_compile_preprocessed.log"/>
-      </exec>
-      
       <echo message="Extracting whatlog information from *whatlog*_compile_preprocessed.log files"/>
-      <for param="whatlogfile">
-        <path>
-          <fileset dir="${build.log.dir}/analysis">
-            <include name="*whatlog*_compile_preprocessed.log"/>
-          </fileset>
-        </path>
-        <sequential>
-          <echo message="Extracting whatlog info from @{whatlogfile}..."/>
-          <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" append="true">
-            <arg value="${sf.common.config.dir}/tools/raptor/parse.pl"/>
-            <arg value="--releaseable"/>
-            <arg value="--log=@{whatlogfile}"/>
-            <arg value="--basedir=${build.log.dir}"/>
-            <arg value="--append"/>
-          </exec>
-        </sequential>
-      </for>
+      <apply executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" parallel="true">
+        <arg value="releaseables.pl"/>
+        <arg value="--basedir=${build.log.dir}"/>
+        <fileset dir="${build.log.dir}/analysis">
+          <include name="*whatlog*_compile_preprocessed.log"/>
+        </fileset>
+      </apply>
       
       <echo message="Extracting error information from *_compile_preprocessed.log files"/>
-      <for param="raptorlogfile">
-        <path>
-          <fileset dir="${build.log.dir}/analysis">
-            <include name="*_compile_preprocessed.log"/>
-          </fileset>
-        </path>
-        <sequential>
-          <echo message="Extracting error info from @{raptorlogfile}..."/>
-          <propertyregex override="yes" property="raptorconfig" input="@{raptorlogfile}" regexp=".*[\\/].*__(.*)_(single|multiple)_thread.*" replace="\1" casesensitive="false" defaultValue="noconfig"/>
-          <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_raptorparse.log" append="true">
-            <arg value="${sf.common.config.dir}/tools/raptor/parse.pl"/>
-            <arg value="--error"/>
-            <arg value="--warning"/>
-            <arg value="--unreciped"/>
-            <arg value="--recipe"/>
-            <arg value="--config=${raptorconfig}"/>
-            <arg value="--log=@{raptorlogfile}"/>
-            <arg value="--basedir=${build.log.dir}/raptorbits"/>
-            <arg value="--append"/>
-          </exec>
-        </sequential>
-      </for>
+      <apply executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_raptorparse.log" parallel="true">
+        <arg value="uh.pl"/>
+        <arg value="--basedir=${build.log.dir}"/>
+        <fileset dir="${build.log.dir}/analysis">
+          <include name="*_compile_preprocessed.log"/>
+        </fileset>
+      </apply>
       
       <echo message="Summarizing Raptor data"/>
       <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_raptorparse.log" append="true">
@@ -527,7 +494,7 @@
       <fmpp sourceFile="${sf.common.config.dir}/diamonds/sf-run-analysis-diamonds.xml.ftl" outputFile="${build.drive}/output/logs/sf-run-analysis-diamonds.xml">
         <data expandProperties="yes">
           ant: antProperties()
-          raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,config,component,phase,recipe,file,line]})
+          raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,config,component,mmp,phase,recipe,file,line]})
         </data>
       </fmpp>
     </target>
@@ -682,6 +649,10 @@
         <copy todir="${publish.dir}" preservelastmodified="true" failonerror="false">
            <fileset dir="${build.output.dir}/zips/release/"><include name="build_*.zip"/></fileset>
         </copy>
+        <mkdir dir="${publish.dir}/html"/>
+        <copy todir="${publish.dir}/html" preservelastmodified="true" failonerror="false">
+           <fileset dir="${build.log.dir}/html"><include name="*.html"/></fileset>
+        </copy>
     </target>
 
     <!-- TODO: Make this work for package builds once we have sysmodel story worked out -->
--- a/common/common_props.ant.xml	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/common_props.ant.xml	Mon Nov 16 19:27:25 2009 +0000
@@ -102,11 +102,11 @@
     <property name="sf.spec.bccheck.enable" value="false"/> <!-- This disables it being called during a normal build, although the target can be called seperately -->
     <property name="sf.spec.bccheck.enable.la" value="true"/>
     <property name="sf.spec.bccheck.enable.ha" value="true"/>
-    <property name="sf.spec.bccheck.enable.dynamic" value="true"/>
+    <property name="sf.spec.bccheck.enable.dynamic" value="false"/>
     <property name="sf.spec.bccheck.baseline.name" value="symbian1"/>
     <property name="sf.spec.bccheck.baseline.skd.dir" value="\\v800008\builds01\releases\installed\s60_5th_sdk_installed"/>
-    <property name="sf.spec.bccheck.baseline.s60.version" value="5.1"/>
-    <property name="sf.spec.bccheck.current.s60.version" value="5.1"/>
+    <property name="sf.spec.bccheck.baseline.s60.version" value="5.0"/>
+    <property name="sf.spec.bccheck.current.s60.version" value="SF2"/>
     <property name="sf.spec.bccheck.reportid" value="report"/>
     <property name="sf.spec.bccheck.package.name" value="${env.COMPUTERNAME}-bctest-${sf.spec.job.name}_${sf.spec.job.codeline}.${sf.spec.job.number}.zip"/>
 </project>
--- a/common/diamonds/linksForDiamonds.xml.ftl	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/diamonds/linksForDiamonds.xml.ftl	Mon Nov 16 19:27:25 2009 +0000
@@ -9,6 +9,11 @@
     </file>
     <file>
       <type>log</type>
+      <name>Raptor Build Summary</name>
+      <url>file:///${ant['sf.spec.publish.networkdrive']}\${ant['sf.spec.job.name']}\builds\${ant['sf.spec.job.codeline']}\${ant['build.id']}\html\index.html</url>
+    </file>
+    <file>
+      <type>log</type>
       <name>Build BOM</name>
       <url>file:///${ant['sf.spec.publish.networkdrive']}\${ant['sf.spec.job.name']}\builds\${ant['sf.spec.job.codeline']}\${ant['build.id']}\build_BOM.zip</url>
     </file>
--- a/common/templates/zip.cfg.xml.ftl.template	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/templates/zip.cfg.xml.ftl.template	Mon Nov 16 19:27:25 2009 +0000
@@ -44,56 +44,7 @@
 			<set name="archives.dir" value="${build.output.dir}/zips/release" />
 			<set name="grace.filters" value="src" />
 			<set name="grace.extract" value="double" />
-			<!-- Zips of zips for SFL code -->
-			<config name="sfl-adaptation">
-				<set name="name" value="src_sfl_adaptation"/>
-				<set name="include" value="src_sfl_adaptation_*.zip"/>
-			</config>
-			<config name="sfl-app">
-				<set name="name" value="src_sfl_app"/>
-				<set name="include" value="src_sfl_app_*.zip"/>
-			</config>
-			<config name="sfl-mw">
-				<set name="name" value="src_sfl_mw"/>
-				<set name="include" value="src_sfl_mw_*.zip"/>
-			</config>
-			<config name="sfl-os">
-				<set name="name" value="src_sfl_os"/>
-				<set name="include" value="src_sfl_os_*.zip"/>
-			</config>
-			<config name="sfl-ostools">
-				<set name="name" value="src_sfl_ostools"/>
-				<set name="include" value="src_sfl_ostools_*.zip"/>
-			</config>
-			<config name="sfl-tools">
-				<set name="name" value="src_sfl_tools"/>
-				<set name="include" value="src_sfl_tools_*.zip"/>
-			</config>
-			<!-- Zips of zips for OSS code -->
-			<config name="oss-adaptation">
-				<set name="name" value="src_oss_adaptation"/>
-				<set name="include" value="src_oss_adaptation_*.zip"/>
-			</config>
-			<config name="oss-app">
-				<set name="name" value="src_oss_app"/>
-				<set name="include" value="src_oss_app_*.zip"/>
-			</config>
-			<config name="oss-mw">
-				<set name="name" value="src_oss_mw"/>
-				<set name="include" value="src_oss_mw_*.zip"/>
-			</config>
-			<config name="oss-os">
-				<set name="name" value="src_oss_os"/>
-				<set name="include" value="src_oss_os_*.zip"/>
-			</config>
-			<config name="oss-ostools">
-				<set name="name" value="src_oss_ostools"/>
-				<set name="include" value="src_oss_ostools_*.zip"/>
-			</config>
-			<config name="oss-tools">
-				<set name="name" value="src_oss_tools"/>
-				<set name="include" value="src_oss_tools_*.zip"/>
-			</config>
+			<!-- Zips of zips for SFL code and for OSS code will be inserted here -->
 		</config>
 		<!-- Specify this config to package all rnd inputs to the build that should be released -->
 		<config name="rnd" abstract="true">
--- a/common/tools/brag/sbsToBrag.pl	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/brag/sbsToBrag.pl	Mon Nov 16 19:27:25 2009 +0000
@@ -87,6 +87,7 @@
 				tools2lib => {message => "Failed to build library $context->{target}", severity => "minor"},
 				ar => {message => "Failed to build library $context->{target}", severity => "minor"},
 				win32archive => {message => "Failed to build library $context->{target}", severity => "minor"},
+				win32def2lib => {message => "Failed to build DLL entry point library $context->{target}", severity => "minor"},
 				"link" => {message => "Failed to create symbols for $context->{target}", severity => "minor"},
 				postlink => {message => "Failed to link $context->{target}", severity => "minor"},
 				win32stageonelink => {message => "Failed to link $context->{target} (stage 1)", severity => "minor"},
--- a/common/tools/populateziptemplate.pl	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/populateziptemplate.pl	Mon Nov 16 19:27:25 2009 +0000
@@ -118,6 +118,20 @@
   				},
   			]
   		};
+		$zipConfig->{config}->{config}->{"src-by-layer"}->{config}->{$license."-$layer"} =
+		{
+			set => 
+			[
+				{
+					name => "name",
+					value => join "_", "src", $license, $layer,
+				},
+				{
+					name => "include",
+					value => join "_", "src", $license, $layer, "*.zip",
+				},
+			]
+		};
 	}
 	elsif ($package->{source} =~ m{/rnd/([^/]+)/([^/]+)})
 	{
--- a/common/tools/raptor/RaptorCommon.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/RaptorCommon.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -20,7 +20,7 @@
 
 sub init
 {
-	my $filename = "$::basedir/summary.csv";
+	my $filename = "$::raptorbitsdir/summary.csv";
 	if (!-f$filename)
 	{
 		print "Writing summary file $filename\n";
@@ -31,10 +31,10 @@
 
 sub dump_fault
 {
-	my ($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line) = @_;
+	my ($category, $subcategory, $severity, $location, $component, $mmp, $phase, $recipe, $file, $line) = @_;
 	
-	open(SUMMARY, ">>$::basedir/summary.csv");
-	print SUMMARY "$category,$subcategory,$severity,$::raptor_config,$component,$phase,$recipe,$file,$line\n";
+	open(SUMMARY, ">>$::raptorbitsdir/summary.csv");
+	print SUMMARY "$category,$subcategory,$severity,$location,$component,$mmp,$phase,$recipe,$file,$line\n";
 	close(SUMMARY);
 }
 
--- a/common/tools/raptor/RaptorError.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/RaptorError.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -51,7 +51,7 @@
 
 sub process
 {
-	my ($text, $component, $phase, $recipe, $file, $line) = @_;
+	my ($text, $logfile, $component, $mmp, $phase, $recipe, $file, $line) = @_;
 	
 	my $category = $CATEGORY_RAPTORERROR;
 	my $severity = '';
@@ -61,47 +61,47 @@
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
 		$subcategory = $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,No bld\.inf found at,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		$subcategory = $CATEGORY_RAPTORERROR_NOBLDINFFOUND;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,Can't find mmp file,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MINOR;
 		$subcategory = $CATEGORY_RAPTORERROR_CANTFINDMMPFILE;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,The make-engine exited with errors,)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
 		$subcategory = $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,tool .* from config .* did not return version .* as required,)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
 		$subcategory = $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,Unknown build configuration '.*',)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
 		$subcategory = $CATEGORY_RAPTORERROR_UNKNOWNBUILDCONFIG;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,No build configurations given,)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
 		$subcategory = $CATEGORY_RAPTORERROR_NOBUILDCONFIGSGIVEN;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	else # log everything by default
 	{
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 }
 
@@ -109,7 +109,7 @@
 {
 	RaptorCommon::init();
 	
-	$filename = "$::basedir/raptor_error.txt";
+	$filename = "$::raptorbitsdir/raptor_error.txt";
 	if (!-f$filename)
 	{
 		print "Writing errors file $filename\n";
@@ -160,7 +160,7 @@
 		print FILE "$characters\n\n";
 		close(FILE);
 		
-		process($characters, '', '', '', "raptor_error.txt", $failure_item);
+		process($characters, $::current_log_file, '', '', '', '', "raptor_error.txt", $failure_item);
 	}
 	
 	$characters = '';
--- a/common/tools/raptor/RaptorInfo.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/RaptorInfo.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -55,7 +55,7 @@
 
 sub on_start_buildlog_info
 {
-	my $filename = "$::basedir/info.txt";
+	my $filename = "$::raptorbitsdir/info.txt";
 	print "Writing info file $filename\n" if (!-f$filename);
 	open(FILE, ">>$filename");
 }
--- a/common/tools/raptor/RaptorRecipe.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/RaptorRecipe.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -59,67 +59,79 @@
 my $CATEGORY_RECIPEFAILURE_ARMCC_GENERICWARNINGSERRORS = 'armcc_generic_warnings_errors';
 my $CATEGORY_RECIPEFAILURE_ELF2E32_SYMBOLMISSINGFROMELFFILE = 'elf2e32_symbol_missing_from_elf_file';
 
+my $mmp_with_issues = {};
+
 
 sub process
 {
-	my ($text, $component, $phase, $recipe, $file, $line) = @_;
+	my ($text, $config, $component, $mmp, $phase, $recipe, $file, $line) = @_;
 	
 	my $category = $CATEGORY_RECIPEFAILURE;
 	my $severity = '';
 	my $subcategory = '';
 	
-	if ($text =~ m,Error:  #5: cannot open source input file .*: No such file or directory,)
+	# if mmp is defined assign severity=MAJOR for the first failure
+	# then severity=MINOR to all other (for each logfile)
+	if ($mmp and defined $mmp_with_issues->{$::current_log_file}->{$mmp})
+	{
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+	}
+	elsif ($mmp)
+	{
+		$mmp_with_issues->{$::current_log_file} = {} if (!defined $mmp_with_issues->{$::current_log_file});
+		$mmp_with_issues->{$::current_log_file}->{$mmp} = 1;
+		$severity = $RaptorCommon::SEVERITY_MAJOR;
+	}
+	else
 	{
 		$severity = $RaptorCommon::SEVERITY_MAJOR;
+	}
+	
+	
+	if ($text =~ m,Error:  #5: cannot open source input file .*: No such file or directory,)
+	{
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,Fatal error: L6002U: Could not open file .*: No such file or directory,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,elf2e32 : Error: E1001: Could not open file : .*.,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,elf2e32 : Error: E1036: Symbol .* Missing from ELF File,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_SYMBOLMISSINGFROMELFFILE;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,Error: L6833E: File '.*' does not exist,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMAR_FILEDOESNOTEXIST;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,: Warning:  #236-D: controlling expression is constant,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CONTROLLINGEXPRESSIONISCONSTANT;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,/armcc.exe , and $text =~ m,Internal fault: ,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_INTERNALFAULT;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	# the following captures generic armcc error/warnings, not captured by regexps above
 	elsif ($text =~ m,/armcc.exe , and $text =~ m,: \d+ warnings\, \d+ errors$,)
 	{
-		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_GENERICWARNINGSERRORS;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	else # log everything by default
 	{
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 }
 
@@ -199,12 +211,20 @@
 			print "WARNING: can't understand bldinf attribute of recipe: $recipe_info->{bldinf}. Won't dump to failed recipes file.\n";
 		}
 		
+		# also normalize mmp path if this exists
+		if ($recipe_info->{mmp})
+		{
+			$recipe_info->{mmp} = lc($recipe_info->{mmp});
+			$recipe_info->{mmp} =~ s,^[A-Za-z]:,,;
+			$recipe_info->{mmp} =~ s,[\\],/,g;
+		}
+		
 		$characters =~ s,^[\r\n]*,,;
 		$characters =~ s,[\r\n]*$,,;
 		
 		if ($package)
 		{
-			$filename = "$::basedir/$package.txt";
+			$filename = "$::raptorbitsdir/$package.txt";
 			if (!-f$filename)
 			{
 				print "Writing recipe file $filename\n";
@@ -231,7 +251,7 @@
 			close(FILE);
 		}
 		
-		process($characters, $recipe_info->{bldinf}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt", $failure_item);
+		process($characters, $recipe_info->{config}, $recipe_info->{bldinf}, $recipe_info->{mmp}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt", $failure_item);
 	}
 
 	$characters = '';
--- a/common/tools/raptor/RaptorReleaseable.pm	Mon Nov 16 14:47:25 2009 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,293 +0,0 @@
-# Copyright (c) 2009 Symbian Foundation Ltd
-# This component and the accompanying materials are made available
-# under the terms of the License "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Symbian Foundation Ltd - initial contribution.
-#
-# Contributors:
-#
-# Description:
-# Raptor parser module.
-# Extract releaseable (whatlog) information
-
-package RaptorReleaseable;
-
-use strict;
-use RaptorCommon;
-
-our $reset_status = {};
-my $buildlog_status = {};
-my $whatlog_status = {};
-my $bitmap_status = {};
-my $resource_status = {};
-my $build_status = {};
-my $export_status = {};
-my $stringtable_status = {};
-my $archive_status = {};
-my $archive_member_status = {};
-my $whatlog_default_status = {};
-
-$reset_status->{name} = 'reset_status';
-$reset_status->{next_status} = {buildlog=>$buildlog_status};
-
-$buildlog_status->{name} = 'buildlog_status';
-$buildlog_status->{next_status} = {whatlog=>$whatlog_status};
-$buildlog_status->{on_start} = 'RaptorReleaseable::on_start_buildlog';
-
-$whatlog_status->{name} = 'whatlog_status';
-$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status};
-$whatlog_status->{on_start} = 'RaptorReleaseable::on_start_whatlog';
-$whatlog_status->{on_end} = 'RaptorReleaseable::on_end_whatlog';
-
-$bitmap_status->{name} = 'bitmap_status';
-$bitmap_status->{next_status} = {};
-$bitmap_status->{on_start} = 'RaptorReleaseable::on_start_bitmap';
-$bitmap_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag';
-$bitmap_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag';
-
-$resource_status->{name} = 'resource_status';
-$resource_status->{next_status} = {};
-$resource_status->{on_start} = 'RaptorReleaseable::on_start_resource';
-$resource_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag';
-$resource_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag';
-
-$build_status->{name} = 'build_status';
-$build_status->{next_status} = {};
-$build_status->{on_start} = 'RaptorReleaseable::on_start_build';
-$build_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag';
-$build_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag';
-
-$stringtable_status->{name} = 'stringtable_status';
-$stringtable_status->{next_status} = {};
-$stringtable_status->{on_start} = 'RaptorReleaseable::on_start_stringtable';
-$stringtable_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag';
-$stringtable_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag';
-
-$archive_status->{name} = 'archive_status';
-$archive_status->{next_status} = {member=>$archive_member_status};
-
-$archive_member_status->{name} = 'archive_member_status';
-$archive_member_status->{next_status} = {};
-$archive_member_status->{on_start} = 'RaptorReleaseable::on_start_archive_member';
-$archive_member_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag';
-$archive_member_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag';
-
-$export_status->{name} = 'export_status';
-$export_status->{next_status} = {};
-$export_status->{on_start} = 'RaptorReleaseable::on_start_export';
-
-$whatlog_default_status->{name} = 'whatlog_default_status';
-$whatlog_default_status->{next_status} = {};
-$whatlog_default_status->{on_start} = 'RaptorReleaseable::on_start_whatlog_default';
-
-my $whatlog_info = {};
-my $curbldinf = 'unknown';
-my $curconfig = 'unknown';
-my $curfiletype = 'unknown';
-my $characters = '';
-
-sub on_start_buildlog
-{
-	mkdir("$::basedir/releaseables");
-}
-
-sub on_start_whatlog
-{
-	my ($el) = @_;
-	
-	$whatlog_info = {};
-	
-	my $bldinf = '';
-	my $config = '';
-	my $attributes = $el->{Attributes};
-	for (keys %{$attributes})
-	{
-		#print "reading attribute $_\n";
-		if ($attributes->{$_}->{'LocalName'} eq 'bldinf')
-		{
-			$bldinf = $attributes->{$_}->{'Value'};
-			#print "bldinf=$bldinf\n";
-		}
-		elsif ($attributes->{$_}->{'LocalName'} eq 'config')
-		{
-			$config = $attributes->{$_}->{'Value'};
-			$config =~ s,\.whatlog$,,;
-		}
-	}
-	
-	if ($bldinf eq '')
-	{
-		print "WARNING: whatlog tag with no bldinf attribute. Skipping\n";
-		return;
-	}
-	
-	$curbldinf = $bldinf;
-	$curconfig = $config;
-	$whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf});
-	$whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig});
-}
-
-sub on_start_whatlog_subtag
-{
-	my ($ft) = @_;
-	
-	$curfiletype = $ft;
-	$characters = '';
-	$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype});
-}
-
-sub on_chars_whatlog_subtag
-{
-	my ($ch) = @_;
-	
-	$characters .= $ch->{Data};
-	
-	#print "characters is now -->$characters<--\n";
-}
-
-sub on_end_whatlog_subtag
-{
-	$characters = normalize_filepath($characters);
-	
-	push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters);
-	
-	$curfiletype = 'unknown';
-	$characters = '';
-}
-
-sub on_start_bitmap
-{
-	on_start_whatlog_subtag('bitmap');
-}
-
-sub on_start_resource
-{
-	on_start_whatlog_subtag('resource');
-}
-
-sub on_start_build
-{
-	on_start_whatlog_subtag('build');
-}
-
-sub on_start_stringtable
-{
-	on_start_whatlog_subtag('stringtable');
-}
-
-sub on_start_archive_member
-{
-	on_start_whatlog_subtag('export');
-}
-
-sub on_start_export
-{
-	my ($el) = @_;
-	
-	$whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export});
-	
-	my $destination = '';
-	my $attributes = $el->{Attributes};
-	for (keys %{$attributes})
-	{
-		#print "reading attribute $_\n";
-		if ($attributes->{$_}->{'LocalName'} eq 'destination')
-		{
-			$destination = $attributes->{$_}->{'Value'};
-			#print "destination=$destination\n";
-			last;
-		}
-	}
-	
-	if ($destination eq '')
-	{
-		print "WARNING: export tag with no destination attribute. Skipping\n";
-		return;
-	}
-	
-	$destination = normalize_filepath($destination);
-	
-	push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination);
-}
-
-sub on_end_whatlog
-{
-	my $unknown_counter = 0;
-	
-	for my $bldinf (keys %{$whatlog_info})
-	{
-		for my $config (keys %{$whatlog_info->{$bldinf}})
-		{
-			my $normalized = lc($bldinf);
-			$normalized =~ s,^[A-Za-z]:,,;
-			$normalized =~ s,[\\],/,g;
-			
-			$normalized =~ m,^/sf/([^/]+)/([^/]+)/,;
-			my $layer = $1;
-			my $package = $2;
-			
-			mkdir("$::basedir/releaseables/$layer");
-			mkdir("$::basedir/releaseables/$layer/$package");
-			
-			my $filename = "$::basedir/releaseables/$layer/$package/info.tsv";
-			
-			print "Writing info file $filename\n" if (!-f$filename);
-			open(FILE, ">>$filename");
-			
-			for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}})
-			{
-				for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}}))
-				{
-					print FILE "$_\t$filetype\t$config\n";
-				}
-			}
-			
-			close(FILE);
-		}
-	}
-}
-
-sub normalize_filepath
-{
-	my ($filepath) = @_;
-	
-	if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,)
-	{
-		print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n";
-	}
-	
-	# strip all CR's and NL's
-	$filepath =~ s,[\r\n],,g;
-	
-	# strip all whitespaces at string start/end
-	$filepath =~ s,^\s+,,g;
-	$filepath =~ s,\s+$,,g;
-	
-	# remove drive letter and colon from the beginning of the string
-	$filepath =~ s,^[A-Za-z]:,,;
-	
-	# normalize slashes
-	$filepath =~ s,\\,/,g;
-	$filepath =~ s,//,/,g;
-	
-	if ($filepath !~ m,^/epoc32/,i)
-	{
-		print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n";
-	}
-	
-	return $filepath;
-}
-
-sub on_start_whatlog_default
-{
-	my ($el) = @_;
-	
-	my $tagname = $el->{LocalName};
-	
-	print "WARNING: unsupported tag '$tagname' in <whatlog> context\n";
-}
-
-1;
\ No newline at end of file
--- a/common/tools/raptor/RaptorUnreciped.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/RaptorUnreciped.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -51,7 +51,7 @@
 
 sub process
 {
-	my ($text, $component, $phase, $recipe, $file, $line) = @_;
+	my ($text, $logfile, $component, $mmp, $phase, $recipe, $file, $line) = @_;
 
 	my $category = $CATEGORY_RAPTORUNRECIPED;	
 	my $severity = '';
@@ -61,31 +61,31 @@
 	{
 		$severity = $RaptorCommon::SEVERITY_MAJOR;
 		my $subcategory = $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,make\.exe: Target .* not remade because of errors,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MINOR;
 		my $subcategory = $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,: warning: ignoring old commands for target,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MINOR;
 		my $subcategory = $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,: warning: overriding commands for target,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MINOR;
 		my $subcategory = $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,make\.exe: Nothing to be done for .*,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MINOR;
 		my $subcategory = $CATEGORY_RAPTORUNRECIPED_NOTHINGTOBEDONEFOR;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,^(true|false)$,)
 	{
@@ -93,7 +93,7 @@
 	}
 	else # log everything by default
 	{
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 }
 
@@ -101,7 +101,7 @@
 {
 	RaptorCommon::init();
 	
-	$filename = "$::basedir/raptor_unreciped.txt";
+	$filename = "$::raptorbitsdir/raptor_unreciped.txt";
 	if (!-f$filename)
 	{
 		print "Writing unreciped file $filename\n";
@@ -163,7 +163,7 @@
 			print FILE "$line\n\n";
 			close(FILE);
 			
-			process($line, '', '', '', "raptor_unreciped.txt", $failure_item);
+			process($line, $::current_log_file, '', '', '', '', "raptor_unreciped.txt", $failure_item);
 		}
 	}
 	
--- a/common/tools/raptor/RaptorWarning.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/RaptorWarning.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -45,7 +45,7 @@
 
 sub process
 {
-	my ($text, $component, $phase, $recipe, $file, $line) = @_;
+	my ($text, $logfile, $component, $mmp, $phase, $recipe, $file, $line) = @_;
 	
 	my $category = $CATEGORY_RAPTORWARNING;
 	my $severity = '';
@@ -55,11 +55,11 @@
 	{
 		$severity = $RaptorCommon::SEVERITY_MINOR;
 		my $subcategory = $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2;
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 	else # log everything by default
 	{
-		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line);
 	}
 }
 
@@ -67,7 +67,7 @@
 {
 	RaptorCommon::init();
 	
-	$filename = "$::basedir/raptor_warning.txt";
+	$filename = "$::raptorbitsdir/raptor_warning.txt";
 	if (!-f$filename)
 	{
 		print "Writing warnings file $filename\n";
@@ -118,7 +118,7 @@
 		print FILE "$characters\n\n";
 		close(FILE);
 		
-		process($characters, '', '', '', "raptor_warning.txt", $failure_item);
+		process($characters, $::current_log_file, '', '', '', '', "raptor_warning.txt", $failure_item);
 	}
 	
 	$characters = '';
--- a/common/tools/raptor/XML/SAX/PurePerl.pm	Mon Nov 16 14:47:25 2009 +0000
+++ b/common/tools/raptor/XML/SAX/PurePerl.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -319,6 +319,12 @@
             $self->characters({Data => $chars});
             last;
         }
+        elsif ($data =~ /(.*?)\]+$/s) {
+            my $chars = $1;
+            $reader->move_along(length($chars));
+            $self->characters({Data => $chars});
+            $data = $reader->data(3);
+        }
         else {
             $self->characters({Data => $data});
             $reader->move_along(length($data));
--- a/common/tools/raptor/parse.pl	Mon Nov 16 14:47:25 2009 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,134 +0,0 @@
-# Copyright (c) 2009 Symbian Foundation Ltd
-# This component and the accompanying materials are made available
-# under the terms of the License "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Symbian Foundation Ltd - initial contribution.
-#
-# Contributors:
-#
-# Description:
-# Run the raptor parsers
-
-use strict;
-use FindBin;
-use lib $FindBin::Bin;
-use RaptorReleaseable;
-use RaptorError;
-use RaptorWarning;
-use RaptorInfo;
-use RaptorUnreciped;
-use RaptorRecipe;
-
-use XML::SAX;
-use RaptorSAXHandler;
-use Getopt::Long;
-
-my @logfiles;
-my $releaseable_module = 0;
-my $error_module = 0;
-my $warning_module = 0;
-my $info_module = 0;
-my $unreciped_module = 0;
-my $recipe_module = 0;
-our $basedir = '';
-our $raptor_config = '';
-my $append = 0;
-my $help = 0;
-GetOptions((
-	'log:s' => \@logfiles,
-	'releaseable!' => \$releaseable_module,
-	'error!' => \$error_module,
-	'warning!' => \$warning_module,
-	'info!' => \$info_module,
-	'unreciped!' => \$unreciped_module,
-	'recipe!' => \$recipe_module,
-	'basedir=s' => \$basedir,
-	'config=s' => \$raptor_config,
-	'append!' => \$append,
-	'help!' => \$help
-));
-
-$help = 1 if (!@logfiles);
-
-if ($help)
-{
-	print "Run the raptor parsers\n";
-	print "Usage: perl parse.pl [MODULES] --log=FILE1 --log=FILE2 ... [OPTIONS]\n";
-	print "where MODULES are:\n";
-	print "\t--releaseable Extract releaseable (whatlog) information\n";
-	print "\t--error Extracts raptor errors, i.e. content of <error> tags\n";
-	print "\t--warning Extracts raptor warnings, i.e. content of <warning> tags\n";
-	print "\t--info Extracts raptor info text i.e. content of <info> tags from a raptor log file\n";
-	print "\t--unreciped Extracts output text in <buildlog> context which doesn't belong to any <recipe> tags\n";
-	print "\t--recipe Extract, analyzes and dumps raptor recipes i.e. content of <recipe> tags from a raptor log file\n";
-	print "where OPTIONS are:\n";
-	print "\t--basedir=DIR Generate output file under DIR\n";
-	print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n";
-	exit(0);
-}
-
-if (!$basedir)
-{
-	$basedir = time;
-	
-	print "Using $basedir as basedir.\n";
-}
-if (-d $basedir)
-{
-	if ($append)
-	{
-		print "Directory $basedir exists. Appending new info to it.\n";
-	}
-	else
-	{
-		print "Directory $basedir exists. Quitting.\n";
-		exit(1);
-	}
-}
-mkdir($basedir);
-#print "Created dir $basedir.\n";
-
-# create empty summary file anyway
-open(SUMMARY, ">$basedir/summary.csv");
-close(SUMMARY);
-
-my $saxhandler = RaptorSAXHandler->new();
-if ($releaseable_module)
-{
-	print "Adding RaptorReleaseable module to the observers\n";
-	$saxhandler->add_observer('RaptorReleaseable', $RaptorReleaseable::reset_status);
-}
-if ($error_module)
-{
-	print "Adding RaptorError module to the observers\n";
-	$saxhandler->add_observer('RaptorError', $RaptorError::reset_status);
-}
-if ($warning_module)
-{
-	print "Adding RaptorWarning module to the observers\n";
-	$saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status);
-}
-if ($info_module)
-{
-	print "Adding RaptorInfo module to the observers\n";
-	$saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status);
-}
-if ($unreciped_module)
-{
-	print "Adding RaptorUnreciped module to the observers\n";
-	$saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status);
-}
-if ($recipe_module)
-{
-	print "Adding RaptorRecipe module to the observers\n";
-	$saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status);
-}
-my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
-for (@logfiles)
-{
-	$parser->parse_uri($_);
-}
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/releaseables.pl	Mon Nov 16 19:27:25 2009 +0000
@@ -0,0 +1,50 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extract releaseable (whatlog) information from Raptor log files
+
+use strict;
+use releaseables;
+use FindBin;
+use lib $FindBin::Bin;
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+our $basedir = '.';
+my $help = 0;
+GetOptions((
+	'basedir=s' => \$basedir,
+	'help!' => \$help
+));
+my @logfiles = @ARGV;
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+	print "Extract releaseable (whatlog) information from Raptor log files\n";
+	print "Usage: perl releaseables.pl [OPTIONS] FILE1 FILE2 ...\n";
+	print "where OPTIONS are:\n";
+	print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n";
+	exit(0);
+}
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->add_observer('releaseables', $releaseables::reset_status);
+
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+	$parser->parse_uri($_);
+}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/releaseables.pm	Mon Nov 16 19:27:25 2009 +0000
@@ -0,0 +1,293 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Raptor parser module.
+# Extract releaseable (whatlog) information
+
+package releaseables;
+
+use strict;
+
+our $reset_status = {};
+my $buildlog_status = {};
+my $whatlog_status = {};
+my $bitmap_status = {};
+my $resource_status = {};
+my $build_status = {};
+my $export_status = {};
+my $stringtable_status = {};
+my $archive_status = {};
+my $archive_member_status = {};
+my $whatlog_default_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {whatlog=>$whatlog_status};
+$buildlog_status->{on_start} = 'releaseables::on_start_buildlog';
+
+$whatlog_status->{name} = 'whatlog_status';
+$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status};
+$whatlog_status->{on_start} = 'releaseables::on_start_whatlog';
+$whatlog_status->{on_end} = 'releaseables::on_end_whatlog';
+
+$bitmap_status->{name} = 'bitmap_status';
+$bitmap_status->{next_status} = {};
+$bitmap_status->{on_start} = 'releaseables::on_start_bitmap';
+$bitmap_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$bitmap_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$resource_status->{name} = 'resource_status';
+$resource_status->{next_status} = {};
+$resource_status->{on_start} = 'releaseables::on_start_resource';
+$resource_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$resource_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$build_status->{name} = 'build_status';
+$build_status->{next_status} = {};
+$build_status->{on_start} = 'releaseables::on_start_build';
+$build_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$build_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$stringtable_status->{name} = 'stringtable_status';
+$stringtable_status->{next_status} = {};
+$stringtable_status->{on_start} = 'releaseables::on_start_stringtable';
+$stringtable_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$stringtable_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$archive_status->{name} = 'archive_status';
+$archive_status->{next_status} = {member=>$archive_member_status};
+
+$archive_member_status->{name} = 'archive_member_status';
+$archive_member_status->{next_status} = {};
+$archive_member_status->{on_start} = 'releaseables::on_start_archive_member';
+$archive_member_status->{on_end} = 'releaseables::on_end_whatlog_subtag';
+$archive_member_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag';
+
+$export_status->{name} = 'export_status';
+$export_status->{next_status} = {};
+$export_status->{on_start} = 'releaseables::on_start_export';
+
+$whatlog_default_status->{name} = 'whatlog_default_status';
+$whatlog_default_status->{next_status} = {};
+$whatlog_default_status->{on_start} = 'releaseables::on_start_whatlog_default';
+
+my $whatlog_info = {};
+my $curbldinf = 'unknown';
+my $curconfig = 'unknown';
+my $curfiletype = 'unknown';
+my $characters = '';
+
+sub on_start_buildlog
+{
+	system("rmdir /S /Q $::raptorbitsdir/releaseables") if (-d "$::basedir/releaseables");
+	mkdir("$::raptorbitsdir/releaseables");
+}
+
+sub on_start_whatlog
+{
+	my ($el) = @_;
+	
+	$whatlog_info = {};
+	
+	my $bldinf = '';
+	my $config = '';
+	my $attributes = $el->{Attributes};
+	for (keys %{$attributes})
+	{
+		#print "reading attribute $_\n";
+		if ($attributes->{$_}->{'LocalName'} eq 'bldinf')
+		{
+			$bldinf = $attributes->{$_}->{'Value'};
+			#print "bldinf=$bldinf\n";
+		}
+		elsif ($attributes->{$_}->{'LocalName'} eq 'config')
+		{
+			$config = $attributes->{$_}->{'Value'};
+			$config =~ s,\.whatlog$,,;
+		}
+	}
+	
+	if ($bldinf eq '')
+	{
+		print "WARNING: whatlog tag with no bldinf attribute. Skipping\n";
+		return;
+	}
+	
+	$curbldinf = $bldinf;
+	$curconfig = $config;
+	$whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf});
+	$whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig});
+}
+
+sub on_start_whatlog_subtag
+{
+	my ($ft) = @_;
+	
+	$curfiletype = $ft;
+	$characters = '';
+	$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype});
+}
+
+sub on_chars_whatlog_subtag
+{
+	my ($ch) = @_;
+	
+	$characters .= $ch->{Data};
+	
+	#print "characters is now -->$characters<--\n";
+}
+
+sub on_end_whatlog_subtag
+{
+	$characters = normalize_filepath($characters);
+	
+	push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters);
+	
+	$curfiletype = 'unknown';
+	$characters = '';
+}
+
+sub on_start_bitmap
+{
+	on_start_whatlog_subtag('bitmap');
+}
+
+sub on_start_resource
+{
+	on_start_whatlog_subtag('resource');
+}
+
+sub on_start_build
+{
+	on_start_whatlog_subtag('build');
+}
+
+sub on_start_stringtable
+{
+	on_start_whatlog_subtag('stringtable');
+}
+
+sub on_start_archive_member
+{
+	on_start_whatlog_subtag('export');
+}
+
+sub on_start_export
+{
+	my ($el) = @_;
+	
+	$whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export});
+	
+	my $destination = '';
+	my $attributes = $el->{Attributes};
+	for (keys %{$attributes})
+	{
+		#print "reading attribute $_\n";
+		if ($attributes->{$_}->{'LocalName'} eq 'destination')
+		{
+			$destination = $attributes->{$_}->{'Value'};
+			#print "destination=$destination\n";
+			last;
+		}
+	}
+	
+	if ($destination eq '')
+	{
+		print "WARNING: export tag with no destination attribute. Skipping\n";
+		return;
+	}
+	
+	$destination = normalize_filepath($destination);
+	
+	push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination);
+}
+
+sub on_end_whatlog
+{
+	my $unknown_counter = 0;
+	
+	for my $bldinf (keys %{$whatlog_info})
+	{
+		for my $config (keys %{$whatlog_info->{$bldinf}})
+		{
+			my $normalized = lc($bldinf);
+			$normalized =~ s,^[A-Za-z]:,,;
+			$normalized =~ s,[\\],/,g;
+			
+			$normalized =~ m,^/sf/([^/]+)/([^/]+)/,;
+			my $layer = $1;
+			my $package = $2;
+			
+			mkdir("$::raptorbitsdir/releaseables/$layer");
+			mkdir("$::raptorbitsdir/releaseables/$layer/$package");
+			
+			my $filename = "$::raptorbitsdir/releaseables/$layer/$package/info.tsv";
+			
+			print "Writing info file $filename\n" if (!-f$filename);
+			open(FILE, ">>$filename");
+			
+			for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}})
+			{
+				for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}}))
+				{
+					print FILE "$_\t$filetype\t$config\n";
+				}
+			}
+			
+			close(FILE);
+		}
+	}
+}
+
+sub normalize_filepath
+{
+	my ($filepath) = @_;
+	
+	if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,)
+	{
+		print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n";
+	}
+	
+	# strip all CR's and NL's
+	$filepath =~ s,[\r\n],,g;
+	
+	# strip all whitespaces at string start/end
+	$filepath =~ s,^\s+,,g;
+	$filepath =~ s,\s+$,,g;
+	
+	# remove drive letter and colon from the beginning of the string
+	$filepath =~ s,^[A-Za-z]:,,;
+	
+	# normalize slashes
+	$filepath =~ s,\\,/,g;
+	$filepath =~ s,//,/,g;
+	
+	if ($filepath !~ m,^/epoc32/,i)
+	{
+		print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n";
+	}
+	
+	return $filepath;
+}
+
+sub on_start_whatlog_default
+{
+	my ($el) = @_;
+	
+	my $tagname = $el->{LocalName};
+	
+	print "WARNING: unsupported tag '$tagname' in <whatlog> context\n";
+}
+
+1;
\ No newline at end of file
--- a/common/tools/raptor/summarize.pl	Mon Nov 16 14:47:25 2009 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,311 +0,0 @@
-# Copyright (c) 2009 Symbian Foundation Ltd
-# This component and the accompanying materials are made available
-# under the terms of the License "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Symbian Foundation Ltd - initial contribution.
-#
-# Contributors:
-#
-# Description:
-# Generate an HTML summary of the Raptor build from the output of the raptor parser
-
-use strict;
-use FindBin;
-use lib $FindBin::Bin;
-use Getopt::Long;
-
-my $raptorbitsdir = 0;
-my $outputdir = '.';
-my $help = 0;
-GetOptions((
-	'raptorbitsdir=s' => \$raptorbitsdir,
-	'outputdir=s' => \$outputdir,
-	'help!' => \$help
-));
-
-$help = 1 if (!$raptorbitsdir);
-
-if ($help)
-{
-	print "Generate an HTML summary of the Raptor build from a summary.csv file\n";
-	print "Usage: perl summarize.pl --raptorbitsdir=DIR [--outputdir=DIR]\n";
-	exit(0);
-}
-
-$outputdir = "$outputdir/html";
-
-system("rd /S /Q $outputdir") if (-d $outputdir);
-mkdir ($outputdir);
-
-my $raptor_errors = {};
-my $raptor_warnings = {};
-my $raptor_unreciped = {};
-my $general_failures_num_by_severity = {};
-my $general_failures_by_category_severity = {};
-my $recipe_failures_num_by_severity = {};
-my $recipe_failures_by_package_severity = {};
-#my $severities = {};
-my @severities = ('critical', 'major', 'minor', 'unknown');
-
-# READ SUMMARY.CSV FILE
-my $csv_file = "$raptorbitsdir/summary.csv";
-my $csv_linenum = 0;
-open(CSV, $csv_file);
-while(<CSV>)
-{
-	$csv_linenum ++;
-	my $line = $_;
-	
-	if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/)
-	{
-		my $failure = {};
-		$failure->{category} = $1;
-		$failure->{subcategory} = $2;
-		$failure->{severity} = $3;
-		$failure->{config} = $4;
-		$failure->{component} = $5;
-		$failure->{phase} = $6;
-		$failure->{recipe} = $7;
-		$failure->{file} = $8;
-		$failure->{linenum} = $9;
-		
-		my $failure_package = '';
-		
-		if (!$failure->{category})
-		{
-			print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n";
-			next;
-		}
-		
-		if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component})
-		{
-			print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n";
-			next;
-		}
-		if ($failure->{component})
-		{
-			if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
-			{
-				$failure_package = $1;
-			}
-			else
-			{
-				print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n";
-				next;
-			}
-		}
-		
-		$failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory});
-		$failure->{severity} = 'unknown' if (!$failure->{severity});
-		
-		# populate severities dynamically.
-		#$severities->{$failure->{severity}} = 1;
-		
-		# put failure items into their category container
-		if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i)
-		{
-			$general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}});
-			my $general_failure = $general_failures_num_by_severity->{$failure->{category}};
-			
-			if (!defined $general_failure->{$failure->{severity}})
-			{
-				$general_failure->{$failure->{severity}} = 1;
-			}
-			else
-			{
-				$general_failure->{$failure->{severity}} ++;
-			}
-			
-			$general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}});
-			$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}});
-			push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure);
-		}
-		elsif ($failure->{category} =~ /^recipe_failure$/i)
-		{
-			$recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package});
-			my $package_failure = $recipe_failures_num_by_severity->{$failure_package};
-			
-			if (!defined $package_failure->{$failure->{severity}})
-			{
-				$package_failure->{$failure->{severity}} = 1;
-			}
-			else
-			{
-				$package_failure->{$failure->{severity}} ++;
-			}
-			
-			$recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package});
-			$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}});
-			push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure);
-		}
-	}
-	else
-	{
-		print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n";
-	}
-}
-close(CSV);
-
-# PRINT HTML SUMMARY
-my $aggregated_html = "$outputdir/index.html";
-open(AGGREGATED, ">$aggregated_html");
-print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n";
-
-print AGGREGATED "<br/>GENERAL FAILURES<br/>\n";
-print AGGREGATED "<table border='1'>\n";
-my $tableheader = "<tr><th>category</th>";
-for (@severities) { $tableheader .= "<th>$_</th>"; }
-$tableheader .= "</tr>";
-print AGGREGATED "$tableheader\n";
-for my $category (keys %{$general_failures_num_by_severity})
-{
-	print_category_specific_summary($category, $general_failures_by_category_severity->{$category});
-	my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>";
-	for (@severities)
-	{
-		my $failuresbyseverity = 0;
-		$failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_});
-		$categoryline .= "<td>$failuresbyseverity</td>";
-	}
-	$categoryline .= "</tr>";
-	print AGGREGATED "$categoryline\n";
-}
-print AGGREGATED "</table>\n";
-print AGGREGATED "<br/>\n";
-
-print AGGREGATED "<br/>PACKGE-SPECIFIC FAILURES<br/>\n";
-print AGGREGATED "<table border='1'>\n";
-$tableheader = "<tr><th>package</th>";
-for (@severities) { $tableheader .= "<th>$_</th>"; }
-$tableheader .= "</tr>";
-print AGGREGATED "$tableheader\n";
-for my $package (keys %{$recipe_failures_num_by_severity})
-{
-	print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package});
-	my $packagesummaryhtml = $package;
-	$packagesummaryhtml =~ s,/,_,;
-	$packagesummaryhtml .= ".html";
-	my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>";
-	for (@severities)
-	{
-		my $failuresbyseverity = 0;
-		$failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_});
-		$packageline .= "<td>$failuresbyseverity</td>";
-	}
-	$packageline .= "</tr>";
-	print AGGREGATED "$packageline\n";
-}
-print AGGREGATED "</table>\n";
-close(AGGREGATED);
-
-translate_detail_files_to_html();
-
-
-sub print_category_specific_summary
-{
-	my ($category, $failures_by_severity) = @_;
-	
-	my $filenamebase = $category;
-	$filenamebase =~ s,/,_,;
-	
-	open(SPECIFIC, ">$outputdir/$filenamebase.html");
-	print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n";
-		
-	for my $severity (@severities)
-	{
-		if (defined $failures_by_severity->{$severity})
-		{
-			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
-			print SPECIFIC "<table border='1'>\n";
-			# $subcategory, $severity, $component, $phase, $recipe, $file, $line
-			my $tableheader = "<tr><th>category</th><th>configuration</th><th>log snippet</th></tr>";
-			print SPECIFIC "$tableheader\n";
-			
-			for my $failure (@{$failures_by_severity->{$severity}})
-			{
-				my $failureline = "<tr><td>$failure->{subcategory}</td>";
-				$failureline .= "<td>$failure->{config}</td>";
-				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
-				$failureline .= "</tr>";
-				print SPECIFIC "$failureline\n";
-			}
-			
-			print SPECIFIC "</table>\n";
-			print SPECIFIC "<br/>\n";
-		}
-	}
-	
-	close(SPECIFIC);
-}
-
-sub print_package_specific_summary
-{
-	my ($package, $failures_by_severity) = @_;
-	
-	my $filenamebase = $package;
-	$filenamebase =~ s,/,_,;
-	
-	open(SPECIFIC, ">$outputdir/$filenamebase.html");
-	print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n";
-		
-	for my $severity (@severities)
-	{
-		if (defined $failures_by_severity->{$severity})
-		{
-			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
-			print SPECIFIC "<table border='1'>\n";
-			# $subcategory, $severity, $component, $phase, $recipe, $file, $line
-			my $tableheader = "<tr><th>category</th><th>configuration</th><th>component</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>";
-			print SPECIFIC "$tableheader\n";
-			
-			for my $failure (@{$failures_by_severity->{$severity}})
-			{
-				my $failureline = "<tr><td>$failure->{subcategory}</td>";
-				$failureline .= "<td>$failure->{config}</td>";
-				$failureline .= "<td>$failure->{component}</td>";
-				$failureline .= "<td>$failure->{phase}</td>";
-				$failureline .= "<td>$failure->{recipe}</td>";
-				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
-				$failureline .= "</tr>";
-				print SPECIFIC "$failureline\n";
-			}
-			
-			print SPECIFIC "</table>\n";
-			print SPECIFIC "<br/>\n";
-		}
-	}
-	
-	close(SPECIFIC);
-}
-
-sub translate_detail_files_to_html
-{
-	opendir(DIR, $raptorbitsdir);
-	my @failurefiles = readdir(DIR);
-	closedir(DIR);	
-	@failurefiles = grep(/\.txt$/, @failurefiles);
-	
-	for my $file (@failurefiles)
-	{
-		$file =~ /(.*)\.txt$/;
-		my $filenamebase = $1;
-		
-		my $filecontent = '';
-		open(FILE, "$raptorbitsdir/$file");
-		{
-			local $/=undef;
-			$filecontent = <FILE>;
-		}
-		close(FILE);
-		
-		$filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g;
-		$filecontent = "<pre>$filecontent</pre>";
-		
-		open(FILE, ">$outputdir/$filenamebase\_failures.html");
-		print FILE $filecontent;
-		close(FILE);
-	}
-}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/uh.pl	Mon Nov 16 19:27:25 2009 +0000
@@ -0,0 +1,358 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Unite and HTML-ize Raptor log files
+
+use strict;
+use FindBin;
+use lib $FindBin::Bin;
+use RaptorError;
+use RaptorWarning;
+use RaptorInfo;
+use RaptorUnreciped;
+use RaptorRecipe;
+
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+our $raptorbitsdir = 'raptorbits';
+our $basedir = '';
+my $outputdir = "html";
+our $raptor_config = 'dummy_config';
+our $current_log_file = '';
+my $help = 0;
+GetOptions((
+	'basedir=s' => \$basedir,
+	'help!' => \$help
+));
+my @logfiles = @ARGV;
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+	print "Unite and HTML-ize Raptor log files.\n";
+	print "Usage: perl uh.pl [OPTIONS] FILE1 FILE2 ...\n";
+	print "where OPTIONS are:\n";
+	print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n";
+	exit(0);
+}
+
+if ($basedir)
+{
+	$raptorbitsdir = "$basedir/raptorbits";
+	$outputdir = "$basedir/html";
+}
+mkdir($basedir) if (!-d$basedir);
+
+system("rmdir /S /Q $raptorbitsdir") if (-d $raptorbitsdir);
+mkdir($raptorbitsdir);
+#print "Created dir $raptorbitsdir.\n";
+
+# create empty summary file anyway
+open(SUMMARY, ">$raptorbitsdir/summary.csv");
+close(SUMMARY);
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->add_observer('RaptorError', $RaptorError::reset_status);
+$saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status);
+$saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status);
+$saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status);
+$saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status);
+
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+	print "Reading file: $_\n";
+	$current_log_file = $_;
+	$parser->parse_uri($_);
+}
+
+print "Generating HTML...\n";
+
+system("rd /S /Q $outputdir") if (-d $outputdir);
+mkdir ($outputdir);
+
+my $raptor_errors = {};
+my $raptor_warnings = {};
+my $raptor_unreciped = {};
+my $general_failures_num_by_severity = {};
+my $general_failures_by_category_severity = {};
+my $recipe_failures_num_by_severity = {};
+my $recipe_failures_by_package_severity = {};
+#my $severities = {};
+my @severities = ('critical', 'major', 'minor', 'unknown');
+
+# READ SUMMARY.CSV FILE
+my $csv_file = "$raptorbitsdir/summary.csv";
+my $csv_linenum = 0;
+open(CSV, $csv_file);
+while(<CSV>)
+{
+	$csv_linenum ++;
+	my $line = $_;
+	
+	if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/)
+	{
+		my $failure = {};
+		$failure->{category} = $1;
+		$failure->{subcategory} = $2;
+		$failure->{severity} = $3;
+		$failure->{config} = $4;
+		$failure->{component} = $5;
+		$failure->{mmp} = $6;
+		$failure->{phase} = $7;
+		$failure->{recipe} = $8;
+		$failure->{file} = $9;
+		$failure->{linenum} = $10;
+		
+		my $failure_package = '';
+		
+		if (!$failure->{category})
+		{
+			print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n";
+			next;
+		}
+		
+		if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component})
+		{
+			print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n";
+			next;
+		}
+		if ($failure->{component})
+		{
+			if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
+			{
+				$failure_package = $1;
+			}
+			else
+			{
+				print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n";
+				next;
+			}
+		}
+		
+		$failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory});
+		$failure->{severity} = 'unknown' if (!$failure->{severity});
+		$failure->{mmp} = '-' if (!$failure->{mmp});
+		
+		# populate severities dynamically.
+		#$severities->{$failure->{severity}} = 1;
+		
+		# put failure items into their category container
+		if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i)
+		{
+			$general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}});
+			my $general_failure = $general_failures_num_by_severity->{$failure->{category}};
+			
+			if (!defined $general_failure->{$failure->{severity}})
+			{
+				$general_failure->{$failure->{severity}} = 1;
+			}
+			else
+			{
+				$general_failure->{$failure->{severity}} ++;
+			}
+			
+			$general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}});
+			$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}});
+			push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure);
+		}
+		elsif ($failure->{category} =~ /^recipe_failure$/i)
+		{
+			$recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package});
+			my $package_failure = $recipe_failures_num_by_severity->{$failure_package};
+			
+			if (!defined $package_failure->{$failure->{severity}})
+			{
+				$package_failure->{$failure->{severity}} = 1;
+			}
+			else
+			{
+				$package_failure->{$failure->{severity}} ++;
+			}
+			
+			$recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package});
+			$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}});
+			push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure);
+		}
+	}
+	else
+	{
+		print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n";
+	}
+}
+close(CSV);
+
+# PRINT HTML SUMMARY
+my $aggregated_html = "$outputdir/index.html";
+open(AGGREGATED, ">$aggregated_html");
+print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n";
+
+print AGGREGATED "<br/>GENERAL FAILURES<br/>\n";
+print AGGREGATED "<table border='1'>\n";
+my $tableheader = "<tr><th>category</th>";
+for (@severities) { $tableheader .= "<th>$_</th>"; }
+$tableheader .= "</tr>";
+print AGGREGATED "$tableheader\n";
+for my $category (keys %{$general_failures_num_by_severity})
+{
+	print_category_specific_summary($category, $general_failures_by_category_severity->{$category});
+	my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>";
+	for (@severities)
+	{
+		my $failuresbyseverity = 0;
+		$failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_});
+		$categoryline .= "<td>$failuresbyseverity</td>";
+	}
+	$categoryline .= "</tr>";
+	print AGGREGATED "$categoryline\n";
+}
+print AGGREGATED "</table>\n";
+print AGGREGATED "<br/>\n";
+
+print AGGREGATED "<br/>PACKGE-SPECIFIC FAILURES<br/>\n";
+print AGGREGATED "<table border='1'>\n";
+$tableheader = "<tr><th>package</th>";
+for (@severities) { $tableheader .= "<th>$_</th>"; }
+$tableheader .= "</tr>";
+print AGGREGATED "$tableheader\n";
+for my $package (keys %{$recipe_failures_num_by_severity})
+{
+	print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package});
+	my $packagesummaryhtml = $package;
+	$packagesummaryhtml =~ s,/,_,;
+	$packagesummaryhtml .= ".html";
+	my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>";
+	for (@severities)
+	{
+		my $failuresbyseverity = 0;
+		$failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_});
+		$packageline .= "<td>$failuresbyseverity</td>";
+	}
+	$packageline .= "</tr>";
+	print AGGREGATED "$packageline\n";
+}
+print AGGREGATED "</table>\n";
+close(AGGREGATED);
+
+translate_detail_files_to_html();
+
+print "OK, done. Please open $outputdir/index.html.\n";
+
+
+sub print_category_specific_summary
+{
+	my ($category, $failures_by_severity) = @_;
+	
+	my $filenamebase = $category;
+	$filenamebase =~ s,/,_,;
+	
+	open(SPECIFIC, ">$outputdir/$filenamebase.html");
+	print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n";
+		
+	for my $severity (@severities)
+	{
+		if (defined $failures_by_severity->{$severity})
+		{
+			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
+			print SPECIFIC "<table border='1'>\n";
+			# $subcategory, $severity, $mmp, $phase, $recipe, $file, $line
+			my $tableheader = "<tr><th>category</th><th>log file</th><th>log snippet</th></tr>";
+			print SPECIFIC "$tableheader\n";
+			
+			for my $failure (@{$failures_by_severity->{$severity}})
+			{
+				my $failureline = "<tr><td>$failure->{subcategory}</td>";
+				$failureline .= "<td>$failure->{config}</td>";
+				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
+				$failureline .= "</tr>";
+				print SPECIFIC "$failureline\n";
+			}
+			
+			print SPECIFIC "</table>\n";
+			print SPECIFIC "<br/>\n";
+		}
+	}
+	
+	close(SPECIFIC);
+}
+
+sub print_package_specific_summary
+{
+	my ($package, $failures_by_severity) = @_;
+	
+	my $filenamebase = $package;
+	$filenamebase =~ s,/,_,;
+	
+	open(SPECIFIC, ">$outputdir/$filenamebase.html");
+	print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n";
+		
+	for my $severity (@severities)
+	{
+		if (defined $failures_by_severity->{$severity})
+		{
+			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
+			print SPECIFIC "<table border='1'>\n";
+			# $subcategory, $severity, $mmp, $phase, $recipe, $file, $line
+			my $tableheader = "<tr><th>category</th><th>configuration</th><th>mmp</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>";
+			print SPECIFIC "$tableheader\n";
+			
+			for my $failure (@{$failures_by_severity->{$severity}})
+			{
+				my $failureline = "<tr><td>$failure->{subcategory}</td>";
+				$failureline .= "<td>$failure->{config}</td>";
+				$failureline .= "<td>$failure->{mmp}</td>";
+				$failureline .= "<td>$failure->{phase}</td>";
+				$failureline .= "<td>$failure->{recipe}</td>";
+				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
+				$failureline .= "</tr>";
+				print SPECIFIC "$failureline\n";
+			}
+			
+			print SPECIFIC "</table>\n";
+			print SPECIFIC "<br/>\n";
+		}
+	}
+	
+	close(SPECIFIC);
+}
+
+sub translate_detail_files_to_html
+{
+	opendir(DIR, $raptorbitsdir);
+	my @failurefiles = readdir(DIR);
+	closedir(DIR);	
+	@failurefiles = grep(/\.txt$/, @failurefiles);
+	
+	for my $file (@failurefiles)
+	{
+		$file =~ /(.*)\.txt$/;
+		my $filenamebase = $1;
+		
+		my $filecontent = '';
+		open(FILE, "$raptorbitsdir/$file");
+		{
+			local $/=undef;
+			$filecontent = <FILE>;
+		}
+		close(FILE);
+		
+		$filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g;
+		$filecontent = "<pre>$filecontent</pre>";
+		
+		open(FILE, ">$outputdir/$filenamebase\_failures.html");
+		print FILE $filecontent;
+		close(FILE);
+	}
+}
\ No newline at end of file