--- a/common/build.xml	Fri Jun 26 17:33:05 2009 +0100
+++ b/common/build.xml	Tue Jun 30 13:58:29 2009 +0100
@@ -650,6 +650,31 @@
         files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]})
       </data>
     </fmpp>
+    
+    <echo message="Extracting whatlog information from *whatlog*_compile.log files"/>
+    <for param="logfile">
+      <path>
+        <fileset dir="${build.log.dir}">
+          <include name="*whatlog*_compile.log"/>
+        </fileset>
+      </path>
+      <sequential>
+        <propertyregex override="yes" property="whatlogfile"  input="@{logfile}" regexp=".*[\\|\/].*\.\d+_+(.*)_compile\.log" replace="\1_preprocessed.log"/>
+        <echo message="Preprocessing @{logfile}..."/>
+        <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" append="true">
+          <arg value="${sf.common.config.dir}/tools/raptor/preprocess_log.pl"/>
+          <arg value="--in=@{logfile}"/>
+          <arg value="--out=${build.log.dir}/analysis/${whatlogfile}"/>
+        </exec>
+        <echo message="Extracting whatlog info from ${whatlogfile}..."/>
+        <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" append="true">
+          <arg value="${sf.common.config.dir}/tools/raptor/package_what.pl"/>
+          <arg value="--log=${build.log.dir}/analysis/${whatlogfile}"/>
+          <arg value="--basedir=${build.log.dir}/deliverables"/>
+          <arg value="--append"/>
+        </exec>
+      </sequential>
+    </for>
 
     </target>
   

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/RaptorSAXHandler.pm	Tue Jun 30 13:58:29 2009 +0100
@@ -0,0 +1,93 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# SAX Handler for the Raptor log
+
+package RaptorSAXHandler;
+use base qw(XML::SAX::Base);
+
+sub new
+{
+    my ($type) = @_;
+    
+    return bless {}, $type;
+}
+
+sub set_init_status
+{
+	my ($self, $initialstatus) = @_;
+	
+	$self->{status} = $initialstatus;
+}
+
+sub start_document
+{
+	my ($self, $doc) = @_;
+	# process document start event
+	
+	#print "start_document\n";
+}
+  
+sub start_element
+{
+	my ($self, $el) = @_;
+	# process element start event
+	
+	my $tagname = $el->{LocalName};
+	
+	#print "start_element($tagname)\n";
+	
+	
+	if (defined $self->{status}->{next_status}->{$tagname})
+	{
+		my $oldstatus = $self->{status};
+		$self->{status} = $self->{status}->{next_status}->{$tagname};
+		#print "status is now $self->{status}->{name}\n";
+		$self->{status}->{next_status}->{$tagname} = $oldstatus;
+		&{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start});
+	}
+	elsif (defined $self->{status}->{next_status}->{'?default?'})
+	{
+		#print "changing to default status\n";
+		my $oldstatus = $self->{status};
+		$self->{status} = $self->{status}->{next_status}->{'?default?'};
+		#print "status is now ?default?\n";
+		$self->{status}->{next_status}->{$tagname} = $oldstatus;
+		&{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start});
+	}
+}
+
+sub end_element
+{
+	my ($self, $el) = @_;
+	# process element start event
+	
+	my $tagname = $el->{LocalName};
+	
+	#print "end_element($tagname)\n";
+	
+	if (defined $self->{status}->{next_status}->{$tagname})
+	{
+		&{$self->{status}->{on_end}}($el) if (defined $self->{status}->{on_end});
+		$self->{status} = $self->{status}->{next_status}->{$tagname};
+		#print "status is now $self->{status}->{name}\n";
+	}
+}
+
+sub characters
+{
+	my ($self, $ch) = @_;
+	
+	&{$self->{status}->{on_chars}}($ch) if (defined $self->{status}->{on_chars});
+}
+
+1;

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/package_what.pl	Tue Jun 30 13:58:29 2009 +0100
@@ -0,0 +1,340 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extracts whatlog information from a raptor log file
+
+use strict;
+
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+my @logfiles;
+my $basedir = '';
+my $append = 0;
+my $help = 0;
+GetOptions((
+	'log:s' => \@logfiles,
+	'basedir:s' => \$basedir,
+	'append!' => \$append,
+	'help!' => \$help
+));
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+	print "Extracts whatlog information from a raptor log file\n";
+	print "Usage: perl package_what.pl --log=FILE1 --log=FILE2 [OPTIONS]\n";
+	print "where OPTIONS are:\n";
+	print "\t--basedir=DIR Generate info files under DIR\n";
+	print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n";
+	exit(0);
+}
+
+my $reset_status = {};
+my $buildlog_status = {};
+my $whatlog_status = {};
+my $bitmap_status = {};
+my $resource_status = {};
+my $build_status = {};
+my $export_status = {};
+my $stringtable_status = {};
+my $archive_status = {};
+my $archive_member_status = {};
+my $whatlog_default_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {whatlog=>$whatlog_status};
+
+$whatlog_status->{name} = 'whatlog_status';
+$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status};
+$whatlog_status->{on_start} = 'main::on_start_whatlog';
+$whatlog_status->{on_end} = 'main::on_end_whatlog';
+
+$bitmap_status->{name} = 'bitmap_status';
+$bitmap_status->{next_status} = {};
+$bitmap_status->{on_start} = 'main::on_start_bitmap';
+$bitmap_status->{on_end} = 'main::on_end_whatlog_subtag';
+$bitmap_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$resource_status->{name} = 'resource_status';
+$resource_status->{next_status} = {};
+$resource_status->{on_start} = 'main::on_start_resource';
+$resource_status->{on_end} = 'main::on_end_whatlog_subtag';
+$resource_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$build_status->{name} = 'build_status';
+$build_status->{next_status} = {};
+$build_status->{on_start} = 'main::on_start_build';
+$build_status->{on_end} = 'main::on_end_whatlog_subtag';
+$build_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$stringtable_status->{name} = 'stringtable_status';
+$stringtable_status->{next_status} = {};
+$stringtable_status->{on_start} = 'main::on_start_stringtable';
+$stringtable_status->{on_end} = 'main::on_end_whatlog_subtag';
+$stringtable_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$archive_status->{name} = 'archive_status';
+$archive_status->{next_status} = {member=>$archive_member_status};
+
+$archive_member_status->{name} = 'archive_member_status';
+$archive_member_status->{next_status} = {};
+$archive_member_status->{on_start} = 'main::on_start_archive_member';
+$archive_member_status->{on_end} = 'main::on_end_whatlog_subtag';
+$archive_member_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$export_status->{name} = 'export_status';
+$export_status->{next_status} = {};
+$export_status->{on_start} = 'main::on_start_export';
+
+$whatlog_default_status->{name} = 'whatlog_default_status';
+$whatlog_default_status->{next_status} = {};
+$whatlog_default_status->{on_start} = 'main::on_start_whatlog_default';
+
+my $whatlog_info = {};
+my $curbldinf = 'unknown';
+my $curconfig = 'unknown';
+my $curfiletype = 'unknown';
+my $characters = '';
+
+if (!$basedir)
+{
+	$basedir = time;
+	
+	print "Using $basedir as basedir.\n";
+}
+if (-d $basedir)
+{
+	if ($append)
+	{
+		print "Directory $basedir exists. Appending new info to it.\n";
+	}
+	else
+	{
+		print "Directory $basedir exists. Quitting.\n";
+		exit(1);
+	}
+}
+mkdir($basedir);
+#print "Created dir $basedir.\n";
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->set_init_status($reset_status);
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+	$parser->parse_uri($_);
+}
+
+
+sub on_start_whatlog
+{
+	my ($el) = @_;
+	
+	#print "on_start_whatlog\n";
+	
+	$whatlog_info = {};
+	
+	my $bldinf = '';
+	my $config = '';
+	my $attributes = $el->{Attributes};
+	for (keys %{$attributes})
+	{
+		#print "reading attribute $_\n";
+		if ($attributes->{$_}->{'LocalName'} eq 'bldinf')
+		{
+			$bldinf = $attributes->{$_}->{'Value'};
+			#print "bldinf=$bldinf\n";
+		}
+		elsif ($attributes->{$_}->{'LocalName'} eq 'config')
+		{
+			$config = $attributes->{$_}->{'Value'};
+			$config =~ s,\.whatlog$,,;
+		}
+	}
+	
+	if ($bldinf eq '')
+	{
+		print "WARNING: whatlog tag with no bldinf attribute. Skipping\n";
+		return;
+	}
+	
+	$curbldinf = $bldinf;
+	$curconfig = $config;
+	$whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf});
+	$whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig});
+}
+
+sub on_start_whatlog_subtag
+{
+	my ($ft) = @_;
+	
+	$curfiletype = $ft;
+	$characters = '';
+	$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype});
+}
+
+sub on_chars_whatlog_subtag
+{
+	my ($ch) = @_;
+	
+	$characters .= $ch->{Data};
+	
+	#print "characters is now -->$characters<--\n";
+}
+
+sub on_end_whatlog_subtag
+{
+	$characters = normalize_filepath($characters);
+	
+	push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters);
+	
+	$curfiletype = 'unknown';
+	$characters = '';
+}
+
+sub on_start_bitmap
+{
+	on_start_whatlog_subtag('bitmap');
+}
+
+sub on_start_resource
+{
+	on_start_whatlog_subtag('resource');
+}
+
+sub on_start_build
+{
+	on_start_whatlog_subtag('build');
+}
+
+sub on_start_stringtable
+{
+	on_start_whatlog_subtag('stringtable');
+}
+
+sub on_start_archive_member
+{
+	on_start_whatlog_subtag('export');
+}
+
+sub on_start_export
+{
+	my ($el) = @_;
+	
+	$whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export});
+	
+	my $destination = '';
+	my $attributes = $el->{Attributes};
+	for (keys %{$attributes})
+	{
+		#print "reading attribute $_\n";
+		if ($attributes->{$_}->{'LocalName'} eq 'destination')
+		{
+			$destination = $attributes->{$_}->{'Value'};
+			#print "destination=$destination\n";
+			last;
+		}
+	}
+	
+	if ($destination eq '')
+	{
+		print "WARNING: export tag with no destination attribute. Skipping\n";
+		return;
+	}
+	
+	$destination = normalize_filepath($destination);
+	
+	push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination);
+}
+
+sub on_end_whatlog
+{
+	my $unknown_counter = 0;
+	
+	for my $bldinf (keys %{$whatlog_info})
+	{
+		for my $config (keys %{$whatlog_info->{$bldinf}})
+		{
+			my $normalized = lc($bldinf);
+			$normalized =~ s,^[A-Za-z]:,,;
+			$normalized =~ s,[\\],/,g;
+			
+			$normalized =~ m,^/sf/([^/]+)/([^/]+)/,;
+			my $layer = $1;
+			my $package = $2;
+			
+			mkdir("$basedir/$layer");
+			mkdir("$basedir/$layer/$package");
+			
+			my $filename = "$basedir/$layer/$package/info.tsv";
+			
+			print "Writing info file $filename\n" if (!-f$filename);
+			open(FILE, ">>$filename");
+			
+			for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}})
+			{
+				for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}}))
+				{
+					print FILE "$_\t$filetype\t$config\n";
+				}
+			}
+			
+			close(FILE);
+		}
+	}
+}
+
+sub normalize_filepath
+{
+	my ($filepath) = @_;
+	
+	if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,)
+	{
+		print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n";
+	}
+	
+	# strip all CR's and NL's
+	$filepath =~ s,[\r\n],,g;
+	
+	# strip all whitespaces at string start/end
+	$filepath =~ s,^\s+,,g;
+	$filepath =~ s,\s+$,,g;
+	
+	# remove drive letter and colon from the beginning of the string
+	$filepath =~ s,^[A-Za-z]:,,;
+	
+	# normalize slashes
+	$filepath =~ s,\\,/,g;
+	$filepath =~ s,//,/,g;
+	
+	if ($filepath !~ m,^/epoc32/,i)
+	{
+		print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n";
+	}
+	
+	return $filepath;
+}
+
+sub on_start_whatlog_default
+{
+	my ($el) = @_;
+	
+	my $tagname = $el->{LocalName};
+	
+	print "WARNING: unsupported tag '$tagname' in <whatlog> context\n";
+}
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/preprocess_log.pl	Tue Jun 30 13:58:29 2009 +0100
@@ -0,0 +1,83 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Preprocess a raptor log, trying to countermeasure a list of known anomalies
+
+use strict;
+
+use Getopt::Long;
+
+my $infile = '';
+my $outfile = '';
+my $basedir = '';
+my $help = 0;
+GetOptions((
+	'in:s' => \$infile,
+	'out:s' => \$outfile,
+	'help!' => \$help
+));
+
+$help = 1 if (!$infile);
+
+if ($help)
+{
+	print "Preprocess a raptor log, trying to countermeasure a list of known anomalies\n";
+	print "Usage: perl preprocess_log.pl --in=INFILE --out=OUTFILE\n";
+	exit(0);
+}
+
+open(INFILE, $infile);
+open(OUTFILE, ">$outfile");
+
+for my $line (<INFILE>)
+{
+	if ($line =~ m,<[^<^>]+>.*&.*</[^<^>]+>,)
+	{
+		$line = escape_ampersand($line);
+	}
+	elsif ($line =~ m,<\?xml\s.*encoding=.*\".*\?>,)
+	{
+		$line = set_encoding_utf8($line);
+	}
+	
+	print OUTFILE $line;
+}
+
+close(OUTFILE);
+close(INFILE);
+
+
+sub escape_ampersand
+{
+	my ($line) = @_;
+	
+	print "escape_ampersand\n";
+	print "in: $line";
+	
+	$line =~ s,&,&amp;,g;
+	
+	print "out: $line";
+	return $line;
+}
+
+sub set_encoding_utf8
+{
+	my ($line) = @_;
+	
+	print "set_encoding_utf8\n";
+	print "in: $line";
+	
+	$line =~ s,encoding=".*",encoding="utf-8",;
+	
+	print "out: $line";
+	return $line;
+}
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/unreciped_text.pl	Tue Jun 30 13:58:29 2009 +0100
@@ -0,0 +1,103 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extracts output text in <buildlog> context which doesn't belong to <recipe>'s
+
+use strict;
+
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+my @logfiles;
+my $basedir = '';
+my $help = 0;
+GetOptions((
+	'log:s' => \@logfiles,
+	'basedir:s' => \$basedir,
+	'help!' => \$help
+));
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+	print "Extracts text which doesn't belong to recipes from a raptor log file\n";
+	print "Usage: perl unreciped_text.pl --log=FILE1 --log=FILE2 [OPTIONS]\n";
+	print "where OPTIONS are:\n";
+	print "\t--basedir=DIR Generate output file under DIR\n";
+	exit(0);
+}
+
+my $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_subtag_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {'?default?'=>$buildlog_subtag_status};
+$buildlog_status->{on_start} = 'main::on_start_buildlog';
+$buildlog_status->{on_end} = 'main::on_end_buildlog';
+$buildlog_status->{on_chars} = 'main::on_chars_buildlog';
+
+$buildlog_subtag_status->{name} = 'buildlog_subtag_status';
+$buildlog_subtag_status->{next_status} = {};
+
+my $characters = '';
+
+if (!$basedir)
+{
+	$basedir = time;
+	
+	print "Using $basedir as basedir.\n";
+}
+if (-d $basedir)
+{
+	print "Directory $basedir exists. Quitting.\n";
+	exit(1);
+}
+mkdir($basedir);
+#print "Created dir $basedir.\n";
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->set_init_status($reset_status);
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+	$parser->parse_uri($_);
+}
+
+
+sub on_start_buildlog
+{
+	my $filename = "$basedir/unreciped.txt";
+	print "Writing unreciped file $filename\n" if (!-f$filename);
+	open(FILE, ">>$filename");
+}
+
+sub on_chars_buildlog
+{
+	my ($ch) = @_;
+	
+	my $characters = $ch->{Data};
+	
+	print FILE $characters if ($characters =~ m,[^\s^\r^\n],);
+	
+	#print "characters is now -->$characters<--\n";
+}
+
+sub on_end_buildlog
+{
+	close(FILE);
+}
\ No newline at end of file