# HG changeset patch
# User Dario Sestito <darios@symbian.org>
# Date 1251286259 -3600
# Node ID 52675b624b66719cb73396f2347cd9700e205ca0
# Parent  cc469b3fe54705b466e25a7163918e855638f9ba
Add raptor build html summary computation

diff -r cc469b3fe547 -r 52675b624b66 common/build.postbuild.xml
--- a/common/build.postbuild.xml	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/build.postbuild.xml	Wed Aug 26 12:30:59 2009 +0100
@@ -301,8 +301,8 @@
           </fileset>
         </path>
         <sequential>
-		<propertyregex override="yes" property="preprocessedlogfile" input="@{logfile}" regexp=".*[\\/](.*)_compile\.log" replace="\1_compile_preprocessed.log"/>
-	        <echo message="Preprocessing @{logfile}..."/>
+		  <propertyregex override="yes" property="preprocessedlogfile" input="@{logfile}" regexp=".*[\\/](.*)_compile\.log" replace="\1_compile_preprocessed.log"/>
+	      <echo message="Preprocessing @{logfile}..."/>
           <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" input="@{logfile}" output="${build.log.dir}/analysis/${preprocessedlogfile}" error="${build.log.dir}/analysis/${build.id}_preprocess.log" append="true">
             <arg value="${sf.common.config.dir}/tools/raptor/preprocess_log.pl"/>
           </exec>
@@ -318,6 +318,7 @@
         </path>
         <sequential>
           <echo message="Extracting whatlog info from @{whatlogfile}..."/>
+          <echo message="Exec: [${sf.common.config.dir}/tools/raptor] perl ${sf.common.config.dir}/tools/raptor/parse.pl --releaseable --log=@{whatlogfile} --basedir=${build.log.dir} --append"/>
           <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" append="true">
             <arg value="${sf.common.config.dir}/tools/raptor/parse.pl"/>
             <arg value="--releaseable"/>
@@ -337,17 +338,30 @@
         </path>
         <sequential>
           <echo message="Extracting error info from @{raptorlogfile}..."/>
+          <propertyregex override="yes" property="raptorconfig" input="@{raptorlogfile}" regexp=".*[\\/].*__(.*)\.whatlog_(single|multiple)_thread.*" replace="\1" casesensitive="false" defaultValue="noconfig"/>
+          <echo message="Exec: [${sf.common.config.dir}/tools/raptor] perl ${sf.common.config.dir}/tools/raptor/parse.pl --error --warning --unreciped --recipe --config=${raptorconfig} --log=@{raptorlogfile} --basedir=${build.log.dir}/raptorbits --append"/>
           <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_raptorparse.log" append="true">
             <arg value="${sf.common.config.dir}/tools/raptor/parse.pl"/>
             <arg value="--error"/>
             <arg value="--warning"/>
+            <arg value="--unreciped"/>
+            <arg value="--recipe"/>
+            <arg value="--config=${raptorconfig}"/>
             <arg value="--log=@{raptorlogfile}"/>
             <arg value="--basedir=${build.log.dir}/raptorbits"/>
             <arg value="--append"/>
           </exec>
         </sequential>
       </for>
-            <!-- Now iterate through the generated TSVs to sort them and remove duplicates -->
+      
+      <echo message="Exec: [${sf.common.config.dir}/tools/raptor] perl ${sf.common.config.dir}/tools/raptor/summarize.pl --raptorbitsdir=${build.log.dir}/raptorbits --outputdir=${build.log.dir}"/>
+      <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_raptorparse.log" append="true">
+        <arg value="${sf.common.config.dir}/tools/raptor/summarize.pl"/>
+        <arg value="--raptorbitsdir=${build.log.dir}/raptorbits"/>
+        <arg value="--outputdir=${build.log.dir}"/>
+      </exec>
+          
+      <!-- Now iterate through the generated TSVs to sort them and remove duplicates -->
       <for param="tsv">
         <path>
           <fileset dir="${build.log.dir}">
@@ -456,7 +470,7 @@
       <fmpp sourceFile="${sf.common.config.dir}/diamonds/sf-run-analysis.xml.ftl" outputFile="${build.drive}/output/logs/sf-run-analysis.xml">
         <data expandProperties="yes">
           ant: antProperties()
-          raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,component,phase,recipe,file,line]})
+          raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,config,component,phase,recipe,file,line]})
           files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]})
         </data>
       </fmpp>
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/RaptorCommon.pm
--- a/common/tools/raptor/RaptorCommon.pm	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/RaptorCommon.pm	Wed Aug 26 12:30:59 2009 +0100
@@ -14,22 +14,8 @@
 
 package RaptorCommon;
 
-our $CATEGORY_GENERAL = 'general';
-
-our $CATEGORY_RAPTORERROR = 'raptor_error';
-our $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION = 'cannot_process_schema_version';
-our $CATEGORY_RAPTORERROR_NOBLDINFFOUND = 'no_bld_inf_found';
-our $CATEGORY_RAPTORERROR_CANTFINDMMPFILE = 'cant_find_mmp_file';
-our $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS = 'make_exited_with_errors';
-our $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION = 'tool_didnot_return_version';
-
-our $CATEGORY_RAPTORWARNING = 'raptor_warning';
-our $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2 = 'missing_enable_abiv2_mode';
-
-our $SEVERITY_UNKNOWN = 'unknown';
 our $SEVERITY_CRITICAL = 'critical';
 our $SEVERITY_MAJOR = 'major';
-our $SEVERITY_NORMAL = 'normal';
 our $SEVERITY_MINOR = 'minor';
 
 sub init
@@ -48,7 +34,7 @@
 	my ($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line) = @_;
 	
 	open(SUMMARY, ">>$::basedir/summary.csv");
-	print SUMMARY "$category,$subcategory,$severity,$component,$phase,$recipe,$file,$line\n";
+	print SUMMARY "$category,$subcategory,$severity,$::raptor_config,$component,$phase,$recipe,$file,$line\n";
 	close(SUMMARY);
 }
 
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/RaptorError.pm
--- a/common/tools/raptor/RaptorError.pm	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/RaptorError.pm	Wed Aug 26 12:30:59 2009 +0100
@@ -36,51 +36,57 @@
 $buildlog_error_status->{on_chars} = 'RaptorError::on_chars_buildlog_error';
 
 my $filename = '';
+my $failure_item = 0;
 
 my $characters = '';
 
-my $category = $RaptorCommon::CATEGORY_RAPTORERROR;
+my $CATEGORY_RAPTORERROR = 'raptor_error';
+my $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION = 'cannot_process_schema_version';
+my $CATEGORY_RAPTORERROR_NOBLDINFFOUND = 'no_bld_inf_found';
+my $CATEGORY_RAPTORERROR_CANTFINDMMPFILE = 'cant_find_mmp_file';
+my $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS = 'make_exited_with_errors';
+my $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION = 'tool_didnot_return_version';
 
 sub process
 {
 	my ($text, $component, $phase, $recipe, $file, $line) = @_;
 	
-	my $severity = $RaptorCommon::SEVERITY_UNKNOWN;
+	my $category = $CATEGORY_RAPTORERROR;
+	my $severity = '';
+	my $subcategory = '';
 	
 	if ($text =~ m,Cannot process schema version .* of file,)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
-		my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION;
+		$subcategory = $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION;
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,No bld\.inf found at,)
 	{
 		$severity = $RaptorCommon::SEVERITY_MAJOR;
-		my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_NOBLDINFFOUND;
+		$subcategory = $CATEGORY_RAPTORERROR_NOBLDINFFOUND;
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,Can't find mmp file,)
 	{
-		$severity = $RaptorCommon::SEVERITY_NORMAL;
-		my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_CANTFINDMMPFILE;
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+		$subcategory = $CATEGORY_RAPTORERROR_CANTFINDMMPFILE;
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,The make-engine exited with errors,)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
-		my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS;
+		$subcategory = $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS;
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 	elsif ($text =~ m,tool .* from config .* did not return version .* as required,)
 	{
 		$severity = $RaptorCommon::SEVERITY_CRITICAL;
-		my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION;
+		$subcategory = $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION;
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 	else # log everything by default
 	{
-		$severity = $RaptorCommon::SEVERITY_NORMAL;
-		my $subcategory = '';
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 }
@@ -89,7 +95,7 @@
 {
 	RaptorCommon::init();
 	
-	$filename = "$::basedir/errors.txt";
+	$filename = "$::basedir/raptor_error.txt";
 	if (!-f$filename)
 	{
 		print "Writing errors file $filename\n";
@@ -100,7 +106,6 @@
 
 sub on_start_buildlog_error
 {
-	open(FILE, ">>$filename");
 }
 
 sub on_chars_buildlog_error
@@ -118,17 +123,31 @@
 {
 	#print "on_end_buildlog_error\n";
 	
-	print FILE $characters if ($characters =~ m,[^\s^\r^\n],);
-	print FILE "\n" if ($characters !~ m,[\r\n]$, );
-	close(FILE);
+	$characters =~ s,^[\r\n]*,,;
+	$characters =~ s,[\r\n]*$,,;
 	
-	# get the line number - not really optimized
-	my $linecount = 0;
-	open(FILE, "$filename");
-	for ($linecount = 0; <FILE>; $linecount++) { }
-	close(FILE);
+	if ($characters =~ m,[^\s^\r^\n],)
+	{	
+		if ($failure_item == 0 and -f "$filename")
+		{
+			open(FILE, "$filename");
+			{
+				local $/ = undef;
+				my $filecontent = <FILE>;
+				$failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s);
+			}
+			close(FILE);
+		}
+		
+		$failure_item++;
 	
-	process($characters, '', '', '', "errors.txt", $linecount);
+		open(FILE, ">>$filename");
+		print FILE "---failure_item_$failure_item\---\n";
+		print FILE "$characters\n\n";
+		close(FILE);
+		
+		process($characters, '', '', '', "raptor_error.txt", $failure_item);
+	}
 	
 	$characters = '';
 }
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/RaptorInfo.pm
--- a/common/tools/raptor/RaptorInfo.pm	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/RaptorInfo.pm	Wed Aug 26 12:30:59 2009 +0100
@@ -42,7 +42,7 @@
 {
 	my ($text) = @_;
 	
-	my $severity = $RaptorCommon::SEVERITY_UNKNOWN;
+	my $severity = '';
 	
 	if ($text =~ m,unmatchable,)
 	{
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/RaptorRecipe.pm
--- a/common/tools/raptor/RaptorRecipe.pm	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/RaptorRecipe.pm	Wed Aug 26 12:30:59 2009 +0100
@@ -21,70 +21,92 @@
 our $reset_status = {};
 my $buildlog_status = {};
 my $buildlog_recipe_status = {};
+my $buildlog_recipe_status_status = {};
 
 $reset_status->{name} = 'reset_status';
 $reset_status->{next_status} = {buildlog=>$buildlog_status};
 
 $buildlog_status->{name} = 'buildlog_status';
 $buildlog_status->{next_status} = {recipe=>$buildlog_recipe_status};
+$buildlog_status->{on_start} = 'RaptorRecipe::on_start_buildlog';
+$buildlog_status->{on_end} = 'RaptorRecipe::on_end_buildlog';
 
 $buildlog_recipe_status->{name} = 'buildlog_recipe_status';
-$buildlog_recipe_status->{next_status} = {};
+$buildlog_recipe_status->{next_status} = {status=>$buildlog_recipe_status_status};
 $buildlog_recipe_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe';
 $buildlog_recipe_status->{on_end} = 'RaptorRecipe::on_end_buildlog_recipe';
 $buildlog_recipe_status->{on_chars} = 'RaptorRecipe::on_chars_buildlog_recipe';
 
-my $characters = '';
+$buildlog_recipe_status_status->{name} = 'buildlog_recipe_status_status';
+$buildlog_recipe_status_status->{next_status} = {};
+$buildlog_recipe_status_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe_status';
+
+
+my $filename = '';
+my $failure_item = 0;
+
 my $recipe_info = {};
 
-my $category = $RaptorCommon::CATEGORY_RAPTORRECIPE;
+my $characters = '';
+
+my $CATEGORY_RECIPEFAILURE = 'recipe_failure';
+my $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE = 'armcc_cannot_open_source_input_file';
+my $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE = 'armlink_could_not_open_file';
+my $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE = 'elf2e32_could_not_open_file';
 
 sub process
 {
-	my ($text) = @_;
+	my ($text, $component, $phase, $recipe, $file, $line) = @_;
+	
+	my $category = $CATEGORY_RECIPEFAILURE;
+	my $severity = '';
+	my $subcategory = '';
 	
-	my $severity = $RaptorCommon::SEVERITY_UNKNOWN;
-	
-	if ($text =~ m,Cannot process schema version .* of file,)
+	if ($text =~ m,Error:  #5: cannot open source input file .*: No such file or directory,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MAJOR;
+		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,Fatal error: L6002U: Could not open file .*: No such file or directory,)
 	{
-		$severity = $RaptorCommon::SEVERITY_CRITICAL;
-		
-		#dump_recipe($category, $severity, $text);
-		print "$category, $severity, $text\n";
+		$severity = $RaptorCommon::SEVERITY_MAJOR;
+		my $subcategory = $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,elf2e32 : Error: E1001: Could not open file : .*.,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MAJOR;
+		my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
+	else # log everything by default
+	{
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+}
+
+sub on_start_buildlog
+{
+	#print FILE "line,layer,component,name,armlicence,platform,phase,code,bldinf,mmp,target,source,\n";
+	
+	RaptorCommon::init();
 }
 
 sub on_start_buildlog_recipe
 {
 	my ($el) = @_;
 	
-	$recipe_info = {};
-	$characters = '';
+	#print "on_start_buildlog_recipe\n";
 	
-	my $attrstring = '';
-	my $bldinf = '';
+	$recipe_info = {};
 	
 	my $attributes = $el->{Attributes};
 	for (keys %{$attributes})
 	{
-		if ($attributes->{$_}->{'LocalName'} eq 'bldinf')
-		{
-			$bldinf = $attributes->{$_}->{'Value'};
-		}
-		
-		$attrstring .= "$attributes->{$_}->{'LocalName'}}='$attributes->{$_}->{'Value'}' ";
-		
+		$recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+		#print "$_ -> $attributes->{$_}->{'Value'}\n";
 	}
-	
-	if ($bldinf eq '')
-	{
-		print "WARNING: recipe tag with no bldinf attribute. Associating to package unknown/unknown\n";
-		$bldinf = "/sf/unknown/unknown/group/bld.inf";
-	}
-	
-	$recipe_info->{bldinf} = $bldinf;
-	
-	$characters = "<recipe $attrstring>\n";
 }
 
 sub on_chars_buildlog_recipe
@@ -98,30 +120,88 @@
 	#print "characters is now -->$characters<--\n";
 }
 
+sub on_start_buildlog_recipe_status
+{
+	my ($el) = @_;
+	
+	my $attributes = $el->{Attributes};
+	for (keys %{$attributes})
+	{
+		if ($attributes->{$_}->{'LocalName'} eq 'code')
+		{
+			$recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+		}
+		elsif ($attributes->{$_}->{'LocalName'} eq 'exit')
+		{
+			$recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+		}
+		elsif ($attributes->{$_}->{'LocalName'} eq 'attempt')
+		{
+			$recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'};
+		}
+	}
+}
+
 sub on_end_buildlog_recipe
 {
-	#print "on_end_buildlog_recipe\n";
-	
-	$characters .= "\n</recipe>\n";
-	
-	my $normalized = lc($recipe_info->{bldinf});
-	$normalized =~ s,^[A-Za-z]:,,;
-	$normalized =~ s,[\\],/,g;
-	
-	$normalized =~ m,^/sf/([^/]+)/([^/]+)/,;
-	my $layer = $1;
-	my $package = $2;
-	
-	mkdir("$::basedir/recipes");
-	mkdir("$::basedir/recipes/$layer");
-	mkdir("$::basedir/recipes/$layer/$package");
-	
-	my $filename = "$::basedir/recipes/$layer/$package/recipes.txt";
-	
-	print "Writing recipes file $filename\n" if (!-f$filename);
-	open(FILE, ">>$filename");
-	print FILE $characters;
-	close(FILE);
+	if ($recipe_info->{exit} =~ /failed/)
+	{
+		# normalize bldinf path
+		$recipe_info->{bldinf} = lc($recipe_info->{bldinf});
+		$recipe_info->{bldinf} =~ s,^[A-Za-z]:,,;
+		$recipe_info->{bldinf} =~ s,[\\],/,g;
+		
+		my $package = '';
+		if ($recipe_info->{bldinf} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
+		{
+			$package = $1;
+			$package =~ s,/,_,;
+		}
+		else
+		{
+			print "WARNING: can't understand bldinf attribute of recipe: $recipe_info->{bldinf}. Won't dump to failed recipes file.\n";
+		}
+		
+		$characters =~ s,^[\r\n]*,,;
+		$characters =~ s,[\r\n]*$,,;
+		
+		if ($package)
+		{
+			$filename = "$::basedir/$package.txt";
+			if (!-f$filename)
+			{
+				print "Writing recipe file $filename\n";
+				open(FILE, ">$filename");
+				close(FILE);
+			}
+			
+			if ($failure_item == 0 and -f "$filename")
+			{
+				open(FILE, "$filename");
+				{
+					local $/ = undef;
+					my $filecontent = <FILE>;
+					$failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s);
+				}
+				close(FILE);
+			}
+			
+			$failure_item++;
+			
+			open(FILE, ">>$filename");
+			print FILE "---failure_item_$failure_item\---\n";
+			print FILE "$characters\n\n";
+			close(FILE);
+		}
+		
+		process($characters, $recipe_info->{bldinf}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt", $failure_item);
+	}
+
+	$characters = '';
+}
+
+sub on_end_buildlog
+{
 }
 
 
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/RaptorUnreciped.pm
--- a/common/tools/raptor/RaptorUnreciped.pm	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/RaptorUnreciped.pm	Wed Aug 26 12:30:59 2009 +0100
@@ -33,30 +33,154 @@
 
 $buildlog_subtag_status->{name} = 'buildlog_subtag_status';
 $buildlog_subtag_status->{next_status} = {};
+$buildlog_subtag_status->{on_start} = 'RaptorUnreciped::on_start_buildlog_subtag';
+$buildlog_subtag_status->{on_end} = 'RaptorUnreciped::on_end_buildlog_subtag';
+
+my $filename = '';
+my $failure_item = 0;
 
 my $characters = '';
+my $store_chars = 1;
+
+my $CATEGORY_RAPTORUNRECIPED = 'raptor_unreciped';
+my $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET = 'no_rule_to_make_target';
+my $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS = 'target_not_remade_for_errors';
+my $CATEGORY_RAPTORUNRECIPED_NOTHINGTOBEDONEFOR = 'nothing_to_be_done_for';
+my $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET = 'ignoring_old_commands_for_target';
+my $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET = 'overriding_commands_for_target';
+
+sub process
+{
+	my ($text, $component, $phase, $recipe, $file, $line) = @_;
+
+	my $category = $CATEGORY_RAPTORUNRECIPED;	
+	my $severity = '';
+	my $subcategory = '';
+	
+	if ($text =~ m,make\.exe: \*\*\* No rule to make target,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MAJOR;
+		my $subcategory = $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,make\.exe: Target .* not remade because of errors,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+		my $subcategory = $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,: warning: ignoring old commands for target,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+		my $subcategory = $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,: warning: overriding commands for target,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+		my $subcategory = $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,make\.exe: Nothing to be done for .*,)
+	{
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+		my $subcategory = $CATEGORY_RAPTORUNRECIPED_NOTHINGTOBEDONEFOR;
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+	elsif ($text =~ m,^(true|false)$,)
+	{
+		# don't dump
+	}
+	else # log everything by default
+	{
+		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
+	}
+}
 
 sub on_start_buildlog
 {
-	my $filename = "$::basedir/unreciped.txt";
-	print "Writing unreciped file $filename\n" if (!-f$filename);
-	open(FILE, ">>$filename");
+	RaptorCommon::init();
+	
+	$filename = "$::basedir/raptor_unreciped.txt";
+	if (!-f$filename)
+	{
+		print "Writing unreciped file $filename\n";
+		open(FILE, ">$filename");
+		close(FILE);
+	}
 }
 
 sub on_chars_buildlog
 {
 	my ($ch) = @_;
 	
-	my $characters = $ch->{Data};
+	#print "on_chars_buildlog\n";
+	
+	if ($store_chars)
+	{
+		$characters .= $ch->{Data};
+		
+		#print "characters is now -->$characters<--\n";
+	}
+}
+
+sub on_end_buildlog_subtag
+{
+	$store_chars = 1;
+}
+
+sub process_characters
+{
+	#print "process_characters\n";
+	
+	$characters =~ s,^[\r\n]*,,;
+	$characters =~ s,[\r\n]*$,,;
+	
+	#print "characters is -->$characters<--\n";
 	
-	print FILE $characters if ($characters =~ m,[^\s^\r^\n],);
+	my @lines = split(/[\r\n]/, $characters);
+	for my $line (@lines)
+	{
+		if ($line =~ m,[^\s^\r^\n],)
+		{
+			#print "dumping chars\n";
+			
+			if ($failure_item == 0 and -f "$filename")
+			{
+				open(FILE, "$filename");
+				{
+					local $/ = undef;
+					my $filecontent = <FILE>;
+					$failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s);
+				}
+				close(FILE);
+			}
+			
+			$failure_item++;
+							
+			open(FILE, ">>$filename");
+			print FILE "---failure_item_$failure_item\---\n";
+			print FILE "$line\n\n";
+			close(FILE);
+			
+			process($line, '', '', '', "raptor_unreciped.txt", $failure_item);
+		}
+	}
 	
-	#print "characters is now -->$characters<--\n";
+	$characters = '';
+	$store_chars = 0;
+}
+
+sub on_start_buildlog_subtag
+{
+	#print "on_start_buildlog_subtag\n";
+	
+	process_characters();
 }
 
 sub on_end_buildlog
 {
-	close(FILE);
+	process_characters();
 }
 
 
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/RaptorWarning.pm
--- a/common/tools/raptor/RaptorWarning.pm	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/RaptorWarning.pm	Wed Aug 26 12:30:59 2009 +0100
@@ -36,27 +36,29 @@
 $buildlog_warning_status->{on_chars} = 'RaptorWarning::on_chars_buildlog_warning';
 
 my $filename = '';
+my $failure_item = 0;
 
 my $characters = '';
 
-my $category = $RaptorCommon::CATEGORY_RAPTORWARNING;
+my $CATEGORY_RAPTORWARNING = 'raptor_warning';
+my $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2 = 'missing_enable_abiv2_mode';
 
 sub process
 {
 	my ($text, $component, $phase, $recipe, $file, $line) = @_;
 	
-	my $severity = $RaptorCommon::SEVERITY_UNKNOWN;
+	my $category = $CATEGORY_RAPTORWARNING;
+	my $severity = '';
+	my $subcategory = '';
 	
 	if ($text =~ m,missing flag ENABLE_ABIV2_MODE,)
 	{
-		$severity = $RaptorCommon::SEVERITY_NORMAL;
-		my $subcategory = $RaptorCommon::CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2;
+		$severity = $RaptorCommon::SEVERITY_MINOR;
+		my $subcategory = $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2;
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 	else # log everything by default
 	{
-		$severity = $RaptorCommon::SEVERITY_NORMAL;
-		my $subcategory = '';
 		RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line);
 	}
 }
@@ -65,7 +67,7 @@
 {
 	RaptorCommon::init();
 	
-	$filename = "$::basedir/warnings.txt";
+	$filename = "$::basedir/raptor_warning.txt";
 	if (!-f$filename)
 	{
 		print "Writing warnings file $filename\n";
@@ -93,17 +95,31 @@
 {
 	#print "on_end_buildlog_warning\n";
 	
-	print FILE $characters if ($characters =~ m,[^\s^\r^\n],);
-	print FILE "\n" if ($characters !~ m,[\r\n]$, );
-	close(FILE);
+	$characters =~ s,^[\r\n]*,,;
+	$characters =~ s,[\r\n]*$,,;
 	
-	# get the line number - not really optimized
-	my $linecount = 0;
-	open(FILE, "$filename");
-	for ($linecount = 0; <FILE>; $linecount++) { }
-	close(FILE);
+	if ($characters =~ m,[^\s^\r^\n],)
+	{
+		if ($failure_item == 0 and -f "$filename")
+		{
+			open(FILE, "$filename");
+			{
+				local $/ = undef;
+				my $filecontent = <FILE>;
+				$failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s);
+			}
+			close(FILE);
+		}
+		
+		$failure_item++;
 	
-	process($characters, '', '', '', "warnings.txt", $linecount);
+		open(FILE, ">>$filename");
+		print FILE "---failure_item_$failure_item\---\n";
+		print FILE "$characters\n\n";
+		close(FILE);
+		
+		process($characters, '', '', '', "raptor_warning.txt", $failure_item);
+	}
 	
 	$characters = '';
 }
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/parse.pl
--- a/common/tools/raptor/parse.pl	Mon Aug 24 15:09:22 2009 +0100
+++ b/common/tools/raptor/parse.pl	Wed Aug 26 12:30:59 2009 +0100
@@ -13,6 +13,8 @@
 # Run the raptor parsers
 
 use strict;
+use FindBin;
+use lib $FindBin::Bin;
 use RaptorReleaseable;
 use RaptorError;
 use RaptorWarning;
@@ -32,6 +34,7 @@
 my $unreciped_module = 0;
 my $recipe_module = 0;
 our $basedir = '';
+our $raptor_config = '';
 my $append = 0;
 my $help = 0;
 GetOptions((
@@ -42,7 +45,8 @@
 	'info!' => \$info_module,
 	'unreciped!' => \$unreciped_module,
 	'recipe!' => \$recipe_module,
-	'basedir:s' => \$basedir,
+	'basedir=s' => \$basedir,
+	'config=s' => \$raptor_config,
 	'append!' => \$append,
 	'help!' => \$help
 ));
diff -r cc469b3fe547 -r 52675b624b66 common/tools/raptor/summarize.pl
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/summarize.pl	Wed Aug 26 12:30:59 2009 +0100
@@ -0,0 +1,311 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Generate an HTML summary of the Raptor build from the output of the raptor parser
+
+use strict;
+use FindBin;
+use lib $FindBin::Bin;
+use Getopt::Long;
+
+my $raptorbitsdir = 0;
+my $outputdir = '.';
+my $help = 0;
+GetOptions((
+	'raptorbitsdir=s' => \$raptorbitsdir,
+	'outputdir=s' => \$outputdir,
+	'help!' => \$help
+));
+
+$help = 1 if (!$raptorbitsdir);
+
+if ($help)
+{
+	print "Generate an HTML summary of the Raptor build from a summary.csv file\n";
+	print "Usage: perl summarize.pl --raptorbitsdir=DIR [--outputdir=DIR]\n";
+	exit(0);
+}
+
+$outputdir = "$outputdir/html";
+
+system("rd /S /Q $outputdir") if (-d $outputdir);
+mkdir ($outputdir);
+
+my $raptor_errors = {};
+my $raptor_warnings = {};
+my $raptor_unreciped = {};
+my $general_failures_num_by_severity = {};
+my $general_failures_by_category_severity = {};
+my $recipe_failures_num_by_severity = {};
+my $recipe_failures_by_package_severity = {};
+#my $severities = {};
+my @severities = ('critical', 'major', 'minor', 'unknown');
+
+# READ SUMMARY.CSV FILE
+my $csv_file = "$raptorbitsdir/summary.csv";
+my $csv_linenum = 0;
+open(CSV, $csv_file);
+while(<CSV>)
+{
+	$csv_linenum ++;
+	my $line = $_;
+	
+	if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/)
+	{
+		my $failure = {};
+		$failure->{category} = $1;
+		$failure->{subcategory} = $2;
+		$failure->{severity} = $3;
+		$failure->{config} = $4;
+		$failure->{component} = $5;
+		$failure->{phase} = $6;
+		$failure->{recipe} = $7;
+		$failure->{file} = $8;
+		$failure->{linenum} = $9;
+		
+		my $failure_package = '';
+		
+		if (!$failure->{category})
+		{
+			print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n";
+			next;
+		}
+		
+		if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component})
+		{
+			print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n";
+			next;
+		}
+		if ($failure->{component})
+		{
+			if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
+			{
+				$failure_package = $1;
+			}
+			else
+			{
+				print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n";
+				next;
+			}
+		}
+		
+		$failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory});
+		$failure->{severity} = 'unknown' if (!$failure->{severity});
+		
+		# populate severities dynamically.
+		#$severities->{$failure->{severity}} = 1;
+		
+		# put failure items into their category container
+		if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i)
+		{
+			$general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}});
+			my $general_failure = $general_failures_num_by_severity->{$failure->{category}};
+			
+			if (!defined $general_failure->{$failure->{severity}})
+			{
+				$general_failure->{$failure->{severity}} = 1;
+			}
+			else
+			{
+				$general_failure->{$failure->{severity}} ++;
+			}
+			
+			$general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}});
+			$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}});
+			push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure);
+		}
+		elsif ($failure->{category} =~ /^recipe_failure$/i)
+		{
+			$recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package});
+			my $package_failure = $recipe_failures_num_by_severity->{$failure_package};
+			
+			if (!defined $package_failure->{$failure->{severity}})
+			{
+				$package_failure->{$failure->{severity}} = 1;
+			}
+			else
+			{
+				$package_failure->{$failure->{severity}} ++;
+			}
+			
+			$recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package});
+			$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}});
+			push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure);
+		}
+	}
+	else
+	{
+		print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n";
+	}
+}
+close(CSV);
+
+# PRINT HTML SUMMARY
+my $aggregated_html = "$outputdir/index.html";
+open(AGGREGATED, ">$aggregated_html");
+print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n";
+
+print AGGREGATED "<br/>GENERAL FAILURES<br/>\n";
+print AGGREGATED "<table border='1'>\n";
+my $tableheader = "<tr><th>category</th>";
+for (@severities) { $tableheader .= "<th>$_</th>"; }
+$tableheader .= "</tr>";
+print AGGREGATED "$tableheader\n";
+for my $category (keys %{$general_failures_num_by_severity})
+{
+	print_category_specific_summary($category, $general_failures_by_category_severity->{$category});
+	my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>";
+	for (@severities)
+	{
+		my $failuresbyseverity = 0;
+		$failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_});
+		$categoryline .= "<td>$failuresbyseverity</td>";
+	}
+	$categoryline .= "</tr>";
+	print AGGREGATED "$categoryline\n";
+}
+print AGGREGATED "</table>\n";
+print AGGREGATED "<br/>\n";
+
+print AGGREGATED "<br/>PACKGE-SPECIFIC FAILURES<br/>\n";
+print AGGREGATED "<table border='1'>\n";
+$tableheader = "<tr><th>package</th>";
+for (@severities) { $tableheader .= "<th>$_</th>"; }
+$tableheader .= "</tr>";
+print AGGREGATED "$tableheader\n";
+for my $package (keys %{$recipe_failures_num_by_severity})
+{
+	print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package});
+	my $packagesummaryhtml = $package;
+	$packagesummaryhtml =~ s,/,_,;
+	$packagesummaryhtml .= ".html";
+	my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>";
+	for (@severities)
+	{
+		my $failuresbyseverity = 0;
+		$failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_});
+		$packageline .= "<td>$failuresbyseverity</td>";
+	}
+	$packageline .= "</tr>";
+	print AGGREGATED "$packageline\n";
+}
+print AGGREGATED "</table>\n";
+close(AGGREGATED);
+
+translate_detail_files_to_html();
+
+
+sub print_category_specific_summary
+{
+	my ($category, $failures_by_severity) = @_;
+	
+	my $filenamebase = $category;
+	$filenamebase =~ s,/,_,;
+	
+	open(SPECIFIC, ">$outputdir/$filenamebase.html");
+	print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n";
+		
+	for my $severity (@severities)
+	{
+		if (defined $failures_by_severity->{$severity})
+		{
+			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
+			print SPECIFIC "<table border='1'>\n";
+			# $subcategory, $severity, $component, $phase, $recipe, $file, $line
+			my $tableheader = "<tr><th>category</th><th>configuration</th><th>log snippet</th></tr>";
+			print SPECIFIC "$tableheader\n";
+			
+			for my $failure (@{$failures_by_severity->{$severity}})
+			{
+				my $failureline = "<tr><td>$failure->{subcategory}</td>";
+				$failureline .= "<td>$failure->{config}</td>";
+				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
+				$failureline .= "</tr>";
+				print SPECIFIC "$failureline\n";
+			}
+			
+			print SPECIFIC "</table>\n";
+			print SPECIFIC "<br/>\n";
+		}
+	}
+	
+	close(SPECIFIC);
+}
+
+sub print_package_specific_summary
+{
+	my ($package, $failures_by_severity) = @_;
+	
+	my $filenamebase = $package;
+	$filenamebase =~ s,/,_,;
+	
+	open(SPECIFIC, ">$outputdir/$filenamebase.html");
+	print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n";
+		
+	for my $severity (@severities)
+	{
+		if (defined $failures_by_severity->{$severity})
+		{
+			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
+			print SPECIFIC "<table border='1'>\n";
+			# $subcategory, $severity, $component, $phase, $recipe, $file, $line
+			my $tableheader = "<tr><th>category</th><th>configuration</th><th>component</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>";
+			print SPECIFIC "$tableheader\n";
+			
+			for my $failure (@{$failures_by_severity->{$severity}})
+			{
+				my $failureline = "<tr><td>$failure->{subcategory}</td>";
+				$failureline .= "<td>$failure->{config}</td>";
+				$failureline .= "<td>$failure->{component}</td>";
+				$failureline .= "<td>$failure->{phase}</td>";
+				$failureline .= "<td>$failure->{recipe}</td>";
+				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
+				$failureline .= "</tr>";
+				print SPECIFIC "$failureline\n";
+			}
+			
+			print SPECIFIC "</table>\n";
+			print SPECIFIC "<br/>\n";
+		}
+	}
+	
+	close(SPECIFIC);
+}
+
+sub translate_detail_files_to_html
+{
+	opendir(DIR, $raptorbitsdir);
+	my @failurefiles = readdir(DIR);
+	closedir(DIR);	
+	@failurefiles = grep(/\.txt$/, @failurefiles);
+	
+	for my $file (@failurefiles)
+	{
+		$file =~ /(.*)\.txt$/;
+		my $filenamebase = $1;
+		
+		my $filecontent = '';
+		open(FILE, "$raptorbitsdir/$file");
+		{
+			local $/=undef;
+			$filecontent = <FILE>;
+		}
+		close(FILE);
+		
+		$filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g;
+		$filecontent = "<pre>$filecontent</pre>";
+		
+		open(FILE, ">$outputdir/$filenamebase\_failures.html");
+		print FILE $filecontent;
+		close(FILE);
+	}
+}