--- a/common/build.xml Tue Jun 30 16:46:32 2009 +0100
+++ b/common/build.xml Tue Jun 30 17:20:00 2009 +0100
@@ -755,6 +755,31 @@
files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]})
</data>
</fmpp>
+
+ <echo message="Extracting whatlog information from *whatlog*_compile.log files"/>
+ <for param="logfile">
+ <path>
+ <fileset dir="${build.log.dir}">
+ <include name="*whatlog*_compile.log"/>
+ </fileset>
+ </path>
+ <sequential>
+ <propertyregex override="yes" property="whatlogfile" input="@{logfile}" regexp=".*[\\|\/].*\.\d+_+(.*)_compile\.log" replace="\1_preprocessed.log"/>
+ <echo message="Preprocessing @{logfile}..."/>
+ <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" append="true">
+ <arg value="${sf.common.config.dir}/tools/raptor/preprocess_log.pl"/>
+ <arg value="--in=@{logfile}"/>
+ <arg value="--out=${build.log.dir}/analysis/${whatlogfile}"/>
+ </exec>
+ <echo message="Extracting whatlog info from ${whatlogfile}..."/>
+ <exec executable="perl" dir="${sf.common.config.dir}/tools/raptor" failonerror="false" output="${build.log.dir}/analysis/${build.id}_whatlog.log" append="true">
+ <arg value="${sf.common.config.dir}/tools/raptor/package_what.pl"/>
+ <arg value="--log=${build.log.dir}/analysis/${whatlogfile}"/>
+ <arg value="--basedir=${build.log.dir}/deliverables"/>
+ <arg value="--append"/>
+ </exec>
+ </sequential>
+ </for>
</target>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/RaptorSAXHandler.pm Tue Jun 30 17:20:00 2009 +0100
@@ -0,0 +1,93 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# SAX Handler for the Raptor log
+
+package RaptorSAXHandler;
+use base qw(XML::SAX::Base);
+
+sub new
+{
+ my ($type) = @_;
+
+ return bless {}, $type;
+}
+
+sub set_init_status
+{
+ my ($self, $initialstatus) = @_;
+
+ $self->{status} = $initialstatus;
+}
+
+sub start_document
+{
+ my ($self, $doc) = @_;
+ # process document start event
+
+ #print "start_document\n";
+}
+
+sub start_element
+{
+ my ($self, $el) = @_;
+ # process element start event
+
+ my $tagname = $el->{LocalName};
+
+ #print "start_element($tagname)\n";
+
+
+ if (defined $self->{status}->{next_status}->{$tagname})
+ {
+ my $oldstatus = $self->{status};
+ $self->{status} = $self->{status}->{next_status}->{$tagname};
+ #print "status is now $self->{status}->{name}\n";
+ $self->{status}->{next_status}->{$tagname} = $oldstatus;
+ &{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start});
+ }
+ elsif (defined $self->{status}->{next_status}->{'?default?'})
+ {
+ #print "changing to default status\n";
+ my $oldstatus = $self->{status};
+ $self->{status} = $self->{status}->{next_status}->{'?default?'};
+ #print "status is now ?default?\n";
+ $self->{status}->{next_status}->{$tagname} = $oldstatus;
+ &{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start});
+ }
+}
+
+sub end_element
+{
+ my ($self, $el) = @_;
+ # process element start event
+
+ my $tagname = $el->{LocalName};
+
+ #print "end_element($tagname)\n";
+
+ if (defined $self->{status}->{next_status}->{$tagname})
+ {
+ &{$self->{status}->{on_end}}($el) if (defined $self->{status}->{on_end});
+ $self->{status} = $self->{status}->{next_status}->{$tagname};
+ #print "status is now $self->{status}->{name}\n";
+ }
+}
+
+sub characters
+{
+ my ($self, $ch) = @_;
+
+ &{$self->{status}->{on_chars}}($ch) if (defined $self->{status}->{on_chars});
+}
+
+1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/package_what.pl Tue Jun 30 17:20:00 2009 +0100
@@ -0,0 +1,340 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extracts whatlog information from a raptor log file
+
+use strict;
+
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+my @logfiles;
+my $basedir = '';
+my $append = 0;
+my $help = 0;
+GetOptions((
+ 'log:s' => \@logfiles,
+ 'basedir:s' => \$basedir,
+ 'append!' => \$append,
+ 'help!' => \$help
+));
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+ print "Extracts whatlog information from a raptor log file\n";
+ print "Usage: perl package_what.pl --log=FILE1 --log=FILE2 [OPTIONS]\n";
+ print "where OPTIONS are:\n";
+ print "\t--basedir=DIR Generate info files under DIR\n";
+ print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n";
+ exit(0);
+}
+
+my $reset_status = {};
+my $buildlog_status = {};
+my $whatlog_status = {};
+my $bitmap_status = {};
+my $resource_status = {};
+my $build_status = {};
+my $export_status = {};
+my $stringtable_status = {};
+my $archive_status = {};
+my $archive_member_status = {};
+my $whatlog_default_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {whatlog=>$whatlog_status};
+
+$whatlog_status->{name} = 'whatlog_status';
+$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status};
+$whatlog_status->{on_start} = 'main::on_start_whatlog';
+$whatlog_status->{on_end} = 'main::on_end_whatlog';
+
+$bitmap_status->{name} = 'bitmap_status';
+$bitmap_status->{next_status} = {};
+$bitmap_status->{on_start} = 'main::on_start_bitmap';
+$bitmap_status->{on_end} = 'main::on_end_whatlog_subtag';
+$bitmap_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$resource_status->{name} = 'resource_status';
+$resource_status->{next_status} = {};
+$resource_status->{on_start} = 'main::on_start_resource';
+$resource_status->{on_end} = 'main::on_end_whatlog_subtag';
+$resource_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$build_status->{name} = 'build_status';
+$build_status->{next_status} = {};
+$build_status->{on_start} = 'main::on_start_build';
+$build_status->{on_end} = 'main::on_end_whatlog_subtag';
+$build_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$stringtable_status->{name} = 'stringtable_status';
+$stringtable_status->{next_status} = {};
+$stringtable_status->{on_start} = 'main::on_start_stringtable';
+$stringtable_status->{on_end} = 'main::on_end_whatlog_subtag';
+$stringtable_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$archive_status->{name} = 'archive_status';
+$archive_status->{next_status} = {member=>$archive_member_status};
+
+$archive_member_status->{name} = 'archive_member_status';
+$archive_member_status->{next_status} = {};
+$archive_member_status->{on_start} = 'main::on_start_archive_member';
+$archive_member_status->{on_end} = 'main::on_end_whatlog_subtag';
+$archive_member_status->{on_chars} = 'main::on_chars_whatlog_subtag';
+
+$export_status->{name} = 'export_status';
+$export_status->{next_status} = {};
+$export_status->{on_start} = 'main::on_start_export';
+
+$whatlog_default_status->{name} = 'whatlog_default_status';
+$whatlog_default_status->{next_status} = {};
+$whatlog_default_status->{on_start} = 'main::on_start_whatlog_default';
+
+my $whatlog_info = {};
+my $curbldinf = 'unknown';
+my $curconfig = 'unknown';
+my $curfiletype = 'unknown';
+my $characters = '';
+
+if (!$basedir)
+{
+ $basedir = time;
+
+ print "Using $basedir as basedir.\n";
+}
+if (-d $basedir)
+{
+ if ($append)
+ {
+ print "Directory $basedir exists. Appending new info to it.\n";
+ }
+ else
+ {
+ print "Directory $basedir exists. Quitting.\n";
+ exit(1);
+ }
+}
+mkdir($basedir);
+#print "Created dir $basedir.\n";
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->set_init_status($reset_status);
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+ $parser->parse_uri($_);
+}
+
+
+sub on_start_whatlog
+{
+ my ($el) = @_;
+
+ #print "on_start_whatlog\n";
+
+ $whatlog_info = {};
+
+ my $bldinf = '';
+ my $config = '';
+ my $attributes = $el->{Attributes};
+ for (keys %{$attributes})
+ {
+ #print "reading attribute $_\n";
+ if ($attributes->{$_}->{'LocalName'} eq 'bldinf')
+ {
+ $bldinf = $attributes->{$_}->{'Value'};
+ #print "bldinf=$bldinf\n";
+ }
+ elsif ($attributes->{$_}->{'LocalName'} eq 'config')
+ {
+ $config = $attributes->{$_}->{'Value'};
+ $config =~ s,\.whatlog$,,;
+ }
+ }
+
+ if ($bldinf eq '')
+ {
+ print "WARNING: whatlog tag with no bldinf attribute. Skipping\n";
+ return;
+ }
+
+ $curbldinf = $bldinf;
+ $curconfig = $config;
+ $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf});
+ $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig});
+}
+
+sub on_start_whatlog_subtag
+{
+ my ($ft) = @_;
+
+ $curfiletype = $ft;
+ $characters = '';
+ $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype});
+}
+
+sub on_chars_whatlog_subtag
+{
+ my ($ch) = @_;
+
+ $characters .= $ch->{Data};
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_end_whatlog_subtag
+{
+ $characters = normalize_filepath($characters);
+
+ push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters);
+
+ $curfiletype = 'unknown';
+ $characters = '';
+}
+
+sub on_start_bitmap
+{
+ on_start_whatlog_subtag('bitmap');
+}
+
+sub on_start_resource
+{
+ on_start_whatlog_subtag('resource');
+}
+
+sub on_start_build
+{
+ on_start_whatlog_subtag('build');
+}
+
+sub on_start_stringtable
+{
+ on_start_whatlog_subtag('stringtable');
+}
+
+sub on_start_archive_member
+{
+ on_start_whatlog_subtag('export');
+}
+
+sub on_start_export
+{
+ my ($el) = @_;
+
+ $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export});
+
+ my $destination = '';
+ my $attributes = $el->{Attributes};
+ for (keys %{$attributes})
+ {
+ #print "reading attribute $_\n";
+ if ($attributes->{$_}->{'LocalName'} eq 'destination')
+ {
+ $destination = $attributes->{$_}->{'Value'};
+ #print "destination=$destination\n";
+ last;
+ }
+ }
+
+ if ($destination eq '')
+ {
+ print "WARNING: export tag with no destination attribute. Skipping\n";
+ return;
+ }
+
+ $destination = normalize_filepath($destination);
+
+ push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination);
+}
+
+sub on_end_whatlog
+{
+ my $unknown_counter = 0;
+
+ for my $bldinf (keys %{$whatlog_info})
+ {
+ for my $config (keys %{$whatlog_info->{$bldinf}})
+ {
+ my $normalized = lc($bldinf);
+ $normalized =~ s,^[A-Za-z]:,,;
+ $normalized =~ s,[\\],/,g;
+
+ $normalized =~ m,^/sf/([^/]+)/([^/]+)/,;
+ my $layer = $1;
+ my $package = $2;
+
+ mkdir("$basedir/$layer");
+ mkdir("$basedir/$layer/$package");
+
+ my $filename = "$basedir/$layer/$package/info.tsv";
+
+ print "Writing info file $filename\n" if (!-f$filename);
+ open(FILE, ">>$filename");
+
+ for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}})
+ {
+ for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}}))
+ {
+ print FILE "$_\t$filetype\t$config\n";
+ }
+ }
+
+ close(FILE);
+ }
+ }
+}
+
+sub normalize_filepath
+{
+ my ($filepath) = @_;
+
+ if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,)
+ {
+ print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n";
+ }
+
+ # strip all CR's and NL's
+ $filepath =~ s,[\r\n],,g;
+
+ # strip all whitespaces at string start/end
+ $filepath =~ s,^\s+,,g;
+ $filepath =~ s,\s+$,,g;
+
+ # remove drive letter and colon from the beginning of the string
+ $filepath =~ s,^[A-Za-z]:,,;
+
+ # normalize slashes
+ $filepath =~ s,\\,/,g;
+ $filepath =~ s,//,/,g;
+
+ if ($filepath !~ m,^/epoc32/,i)
+ {
+ print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n";
+ }
+
+ return $filepath;
+}
+
+sub on_start_whatlog_default
+{
+ my ($el) = @_;
+
+ my $tagname = $el->{LocalName};
+
+ print "WARNING: unsupported tag '$tagname' in <whatlog> context\n";
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/preprocess_log.pl Tue Jun 30 17:20:00 2009 +0100
@@ -0,0 +1,83 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Preprocess a raptor log, trying to countermeasure a list of known anomalies
+
+use strict;
+
+use Getopt::Long;
+
+my $infile = '';
+my $outfile = '';
+my $basedir = '';
+my $help = 0;
+GetOptions((
+ 'in:s' => \$infile,
+ 'out:s' => \$outfile,
+ 'help!' => \$help
+));
+
+$help = 1 if (!$infile);
+
+if ($help)
+{
+ print "Preprocess a raptor log, trying to countermeasure a list of known anomalies\n";
+ print "Usage: perl preprocess_log.pl --in=INFILE --out=OUTFILE\n";
+ exit(0);
+}
+
+open(INFILE, $infile);
+open(OUTFILE, ">$outfile");
+
+for my $line (<INFILE>)
+{
+ if ($line =~ m,<[^<^>]+>.*&.*</[^<^>]+>,)
+ {
+ $line = escape_ampersand($line);
+ }
+ elsif ($line =~ m,<\?xml\s.*encoding=.*\".*\?>,)
+ {
+ $line = set_encoding_utf8($line);
+ }
+
+ print OUTFILE $line;
+}
+
+close(OUTFILE);
+close(INFILE);
+
+
+sub escape_ampersand
+{
+ my ($line) = @_;
+
+ print "escape_ampersand\n";
+ print "in: $line";
+
+ $line =~ s,&,&,g;
+
+ print "out: $line";
+ return $line;
+}
+
+sub set_encoding_utf8
+{
+ my ($line) = @_;
+
+ print "set_encoding_utf8\n";
+ print "in: $line";
+
+ $line =~ s,encoding=".*",encoding="utf-8",;
+
+ print "out: $line";
+ return $line;
+}
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/common/tools/raptor/unreciped_text.pl Tue Jun 30 17:20:00 2009 +0100
@@ -0,0 +1,103 @@
+# Copyright (c) 2009 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+#
+# Description:
+# Extracts output text in <buildlog> context which doesn't belong to <recipe>'s
+
+use strict;
+
+use XML::SAX;
+use RaptorSAXHandler;
+use Getopt::Long;
+
+my @logfiles;
+my $basedir = '';
+my $help = 0;
+GetOptions((
+ 'log:s' => \@logfiles,
+ 'basedir:s' => \$basedir,
+ 'help!' => \$help
+));
+
+$help = 1 if (!@logfiles);
+
+if ($help)
+{
+ print "Extracts text which doesn't belong to recipes from a raptor log file\n";
+ print "Usage: perl unreciped_text.pl --log=FILE1 --log=FILE2 [OPTIONS]\n";
+ print "where OPTIONS are:\n";
+ print "\t--basedir=DIR Generate output file under DIR\n";
+ exit(0);
+}
+
+my $reset_status = {};
+my $buildlog_status = {};
+my $buildlog_subtag_status = {};
+
+$reset_status->{name} = 'reset_status';
+$reset_status->{next_status} = {buildlog=>$buildlog_status};
+
+$buildlog_status->{name} = 'buildlog_status';
+$buildlog_status->{next_status} = {'?default?'=>$buildlog_subtag_status};
+$buildlog_status->{on_start} = 'main::on_start_buildlog';
+$buildlog_status->{on_end} = 'main::on_end_buildlog';
+$buildlog_status->{on_chars} = 'main::on_chars_buildlog';
+
+$buildlog_subtag_status->{name} = 'buildlog_subtag_status';
+$buildlog_subtag_status->{next_status} = {};
+
+my $characters = '';
+
+if (!$basedir)
+{
+ $basedir = time;
+
+ print "Using $basedir as basedir.\n";
+}
+if (-d $basedir)
+{
+ print "Directory $basedir exists. Quitting.\n";
+ exit(1);
+}
+mkdir($basedir);
+#print "Created dir $basedir.\n";
+
+my $saxhandler = RaptorSAXHandler->new();
+$saxhandler->set_init_status($reset_status);
+my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
+for (@logfiles)
+{
+ $parser->parse_uri($_);
+}
+
+
+sub on_start_buildlog
+{
+ my $filename = "$basedir/unreciped.txt";
+ print "Writing unreciped file $filename\n" if (!-f$filename);
+ open(FILE, ">>$filename");
+}
+
+sub on_chars_buildlog
+{
+ my ($ch) = @_;
+
+ my $characters = $ch->{Data};
+
+ print FILE $characters if ($characters =~ m,[^\s^\r^\n],);
+
+ #print "characters is now -->$characters<--\n";
+}
+
+sub on_end_buildlog
+{
+ close(FILE);
+}
\ No newline at end of file
--- a/sf-package/package_props.ant.xml Tue Jun 30 16:46:32 2009 +0100
+++ b/sf-package/package_props.ant.xml Tue Jun 30 17:20:00 2009 +0100
@@ -4,8 +4,8 @@
<property name="sf.spec.baseline.enable" value="true"/>
<property name="sf.spec.baseline.select" value="location"/>
- <property name="sf.spec.baseline.location" value="\\v800008.ad-sfpd.intra\g$\Releases\PDK_candidate_2.0.b_flat"/>
- <property name="sf.spec.baseline.getenv_options" value="-i emu -i rnd"/>
+ <property name="sf.spec.baseline.location" value="\\bishare\Releases\PDK_candidate_2.0.c_flat"/>
+ <property name="sf.spec.baseline.getenv_options" value="-i emu -i rnd -i tools"/>
<property name="sf.spec.systemdefinition.assemble" value="false"/>
<property name="sf.spec.sysdef.configurations.list" value="SF"/>