# HG changeset patch # User Dario Sestito # Date 1246366709 -3600 # Node ID 1d9c60a4e3083387801d479a548ebcef2c4dd659 # Parent 3f8d2ea13886a2ff6b0b58eea323405edeccf48f Adding extraction of whatlog information into "deliverables" files diff -r 3f8d2ea13886 -r 1d9c60a4e308 common/build.xml --- a/common/build.xml Fri Jun 26 17:33:05 2009 +0100 +++ b/common/build.xml Tue Jun 30 13:58:29 2009 +0100 @@ -650,6 +650,31 @@ files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]}) + + + + + + + + + + + + + + + + + + + + + + + + + diff -r 3f8d2ea13886 -r 1d9c60a4e308 common/tools/raptor/RaptorSAXHandler.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorSAXHandler.pm Tue Jun 30 13:58:29 2009 +0100 @@ -0,0 +1,93 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# SAX Handler for the Raptor log + +package RaptorSAXHandler; +use base qw(XML::SAX::Base); + +sub new +{ + my ($type) = @_; + + return bless {}, $type; +} + +sub set_init_status +{ + my ($self, $initialstatus) = @_; + + $self->{status} = $initialstatus; +} + +sub start_document +{ + my ($self, $doc) = @_; + # process document start event + + #print "start_document\n"; +} + +sub start_element +{ + my ($self, $el) = @_; + # process element start event + + my $tagname = $el->{LocalName}; + + #print "start_element($tagname)\n"; + + + if (defined $self->{status}->{next_status}->{$tagname}) + { + my $oldstatus = $self->{status}; + $self->{status} = $self->{status}->{next_status}->{$tagname}; + #print "status is now $self->{status}->{name}\n"; + $self->{status}->{next_status}->{$tagname} = $oldstatus; + &{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start}); + } + elsif (defined $self->{status}->{next_status}->{'?default?'}) + { + #print "changing to default status\n"; + my $oldstatus = $self->{status}; + $self->{status} = $self->{status}->{next_status}->{'?default?'}; + #print "status is now ?default?\n"; + $self->{status}->{next_status}->{$tagname} = $oldstatus; + &{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start}); + } +} + +sub end_element +{ + my ($self, $el) = @_; + # process element start event + + my $tagname = $el->{LocalName}; + + #print "end_element($tagname)\n"; + + if (defined $self->{status}->{next_status}->{$tagname}) + { + &{$self->{status}->{on_end}}($el) if (defined $self->{status}->{on_end}); + $self->{status} = $self->{status}->{next_status}->{$tagname}; + #print "status is now $self->{status}->{name}\n"; + } +} + +sub characters +{ + my ($self, $ch) = @_; + + &{$self->{status}->{on_chars}}($ch) if (defined $self->{status}->{on_chars}); +} + +1; diff -r 3f8d2ea13886 -r 1d9c60a4e308 common/tools/raptor/package_what.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/package_what.pl Tue Jun 30 13:58:29 2009 +0100 @@ -0,0 +1,340 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Extracts whatlog information from a raptor log file + +use strict; + +use XML::SAX; +use RaptorSAXHandler; +use Getopt::Long; + +my @logfiles; +my $basedir = ''; +my $append = 0; +my $help = 0; +GetOptions(( + 'log:s' => \@logfiles, + 'basedir:s' => \$basedir, + 'append!' => \$append, + 'help!' => \$help +)); + +$help = 1 if (!@logfiles); + +if ($help) +{ + print "Extracts whatlog information from a raptor log file\n"; + print "Usage: perl package_what.pl --log=FILE1 --log=FILE2 [OPTIONS]\n"; + print "where OPTIONS are:\n"; + print "\t--basedir=DIR Generate info files under DIR\n"; + print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n"; + exit(0); +} + +my $reset_status = {}; +my $buildlog_status = {}; +my $whatlog_status = {}; +my $bitmap_status = {}; +my $resource_status = {}; +my $build_status = {}; +my $export_status = {}; +my $stringtable_status = {}; +my $archive_status = {}; +my $archive_member_status = {}; +my $whatlog_default_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {whatlog=>$whatlog_status}; + +$whatlog_status->{name} = 'whatlog_status'; +$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status}; +$whatlog_status->{on_start} = 'main::on_start_whatlog'; +$whatlog_status->{on_end} = 'main::on_end_whatlog'; + +$bitmap_status->{name} = 'bitmap_status'; +$bitmap_status->{next_status} = {}; +$bitmap_status->{on_start} = 'main::on_start_bitmap'; +$bitmap_status->{on_end} = 'main::on_end_whatlog_subtag'; +$bitmap_status->{on_chars} = 'main::on_chars_whatlog_subtag'; + +$resource_status->{name} = 'resource_status'; +$resource_status->{next_status} = {}; +$resource_status->{on_start} = 'main::on_start_resource'; +$resource_status->{on_end} = 'main::on_end_whatlog_subtag'; +$resource_status->{on_chars} = 'main::on_chars_whatlog_subtag'; + +$build_status->{name} = 'build_status'; +$build_status->{next_status} = {}; +$build_status->{on_start} = 'main::on_start_build'; +$build_status->{on_end} = 'main::on_end_whatlog_subtag'; +$build_status->{on_chars} = 'main::on_chars_whatlog_subtag'; + +$stringtable_status->{name} = 'stringtable_status'; +$stringtable_status->{next_status} = {}; +$stringtable_status->{on_start} = 'main::on_start_stringtable'; +$stringtable_status->{on_end} = 'main::on_end_whatlog_subtag'; +$stringtable_status->{on_chars} = 'main::on_chars_whatlog_subtag'; + +$archive_status->{name} = 'archive_status'; +$archive_status->{next_status} = {member=>$archive_member_status}; + +$archive_member_status->{name} = 'archive_member_status'; +$archive_member_status->{next_status} = {}; +$archive_member_status->{on_start} = 'main::on_start_archive_member'; +$archive_member_status->{on_end} = 'main::on_end_whatlog_subtag'; +$archive_member_status->{on_chars} = 'main::on_chars_whatlog_subtag'; + +$export_status->{name} = 'export_status'; +$export_status->{next_status} = {}; +$export_status->{on_start} = 'main::on_start_export'; + +$whatlog_default_status->{name} = 'whatlog_default_status'; +$whatlog_default_status->{next_status} = {}; +$whatlog_default_status->{on_start} = 'main::on_start_whatlog_default'; + +my $whatlog_info = {}; +my $curbldinf = 'unknown'; +my $curconfig = 'unknown'; +my $curfiletype = 'unknown'; +my $characters = ''; + +if (!$basedir) +{ + $basedir = time; + + print "Using $basedir as basedir.\n"; +} +if (-d $basedir) +{ + if ($append) + { + print "Directory $basedir exists. Appending new info to it.\n"; + } + else + { + print "Directory $basedir exists. Quitting.\n"; + exit(1); + } +} +mkdir($basedir); +#print "Created dir $basedir.\n"; + +my $saxhandler = RaptorSAXHandler->new(); +$saxhandler->set_init_status($reset_status); +my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); +for (@logfiles) +{ + $parser->parse_uri($_); +} + + +sub on_start_whatlog +{ + my ($el) = @_; + + #print "on_start_whatlog\n"; + + $whatlog_info = {}; + + my $bldinf = ''; + my $config = ''; + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + #print "reading attribute $_\n"; + if ($attributes->{$_}->{'LocalName'} eq 'bldinf') + { + $bldinf = $attributes->{$_}->{'Value'}; + #print "bldinf=$bldinf\n"; + } + elsif ($attributes->{$_}->{'LocalName'} eq 'config') + { + $config = $attributes->{$_}->{'Value'}; + $config =~ s,\.whatlog$,,; + } + } + + if ($bldinf eq '') + { + print "WARNING: whatlog tag with no bldinf attribute. Skipping\n"; + return; + } + + $curbldinf = $bldinf; + $curconfig = $config; + $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf}); + $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig}); +} + +sub on_start_whatlog_subtag +{ + my ($ft) = @_; + + $curfiletype = $ft; + $characters = ''; + $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}); +} + +sub on_chars_whatlog_subtag +{ + my ($ch) = @_; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_whatlog_subtag +{ + $characters = normalize_filepath($characters); + + push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters); + + $curfiletype = 'unknown'; + $characters = ''; +} + +sub on_start_bitmap +{ + on_start_whatlog_subtag('bitmap'); +} + +sub on_start_resource +{ + on_start_whatlog_subtag('resource'); +} + +sub on_start_build +{ + on_start_whatlog_subtag('build'); +} + +sub on_start_stringtable +{ + on_start_whatlog_subtag('stringtable'); +} + +sub on_start_archive_member +{ + on_start_whatlog_subtag('export'); +} + +sub on_start_export +{ + my ($el) = @_; + + $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export}); + + my $destination = ''; + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + #print "reading attribute $_\n"; + if ($attributes->{$_}->{'LocalName'} eq 'destination') + { + $destination = $attributes->{$_}->{'Value'}; + #print "destination=$destination\n"; + last; + } + } + + if ($destination eq '') + { + print "WARNING: export tag with no destination attribute. Skipping\n"; + return; + } + + $destination = normalize_filepath($destination); + + push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination); +} + +sub on_end_whatlog +{ + my $unknown_counter = 0; + + for my $bldinf (keys %{$whatlog_info}) + { + for my $config (keys %{$whatlog_info->{$bldinf}}) + { + my $normalized = lc($bldinf); + $normalized =~ s,^[A-Za-z]:,,; + $normalized =~ s,[\\],/,g; + + $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; + my $layer = $1; + my $package = $2; + + mkdir("$basedir/$layer"); + mkdir("$basedir/$layer/$package"); + + my $filename = "$basedir/$layer/$package/info.tsv"; + + print "Writing info file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); + + for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}}) + { + for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}})) + { + print FILE "$_\t$filetype\t$config\n"; + } + } + + close(FILE); + } + } +} + +sub normalize_filepath +{ + my ($filepath) = @_; + + if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,) + { + print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n"; + } + + # strip all CR's and NL's + $filepath =~ s,[\r\n],,g; + + # strip all whitespaces at string start/end + $filepath =~ s,^\s+,,g; + $filepath =~ s,\s+$,,g; + + # remove drive letter and colon from the beginning of the string + $filepath =~ s,^[A-Za-z]:,,; + + # normalize slashes + $filepath =~ s,\\,/,g; + $filepath =~ s,//,/,g; + + if ($filepath !~ m,^/epoc32/,i) + { + print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n"; + } + + return $filepath; +} + +sub on_start_whatlog_default +{ + my ($el) = @_; + + my $tagname = $el->{LocalName}; + + print "WARNING: unsupported tag '$tagname' in context\n"; +} \ No newline at end of file diff -r 3f8d2ea13886 -r 1d9c60a4e308 common/tools/raptor/preprocess_log.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/preprocess_log.pl Tue Jun 30 13:58:29 2009 +0100 @@ -0,0 +1,83 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Preprocess a raptor log, trying to countermeasure a list of known anomalies + +use strict; + +use Getopt::Long; + +my $infile = ''; +my $outfile = ''; +my $basedir = ''; +my $help = 0; +GetOptions(( + 'in:s' => \$infile, + 'out:s' => \$outfile, + 'help!' => \$help +)); + +$help = 1 if (!$infile); + +if ($help) +{ + print "Preprocess a raptor log, trying to countermeasure a list of known anomalies\n"; + print "Usage: perl preprocess_log.pl --in=INFILE --out=OUTFILE\n"; + exit(0); +} + +open(INFILE, $infile); +open(OUTFILE, ">$outfile"); + +for my $line () +{ + if ($line =~ m,<[^<^>]+>.*&.*]+>,) + { + $line = escape_ampersand($line); + } + elsif ($line =~ m,<\?xml\s.*encoding=.*\".*\?>,) + { + $line = set_encoding_utf8($line); + } + + print OUTFILE $line; +} + +close(OUTFILE); +close(INFILE); + + +sub escape_ampersand +{ + my ($line) = @_; + + print "escape_ampersand\n"; + print "in: $line"; + + $line =~ s,&,&,g; + + print "out: $line"; + return $line; +} + +sub set_encoding_utf8 +{ + my ($line) = @_; + + print "set_encoding_utf8\n"; + print "in: $line"; + + $line =~ s,encoding=".*",encoding="utf-8",; + + print "out: $line"; + return $line; +} \ No newline at end of file diff -r 3f8d2ea13886 -r 1d9c60a4e308 common/tools/raptor/unreciped_text.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/unreciped_text.pl Tue Jun 30 13:58:29 2009 +0100 @@ -0,0 +1,103 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Extracts output text in context which doesn't belong to 's + +use strict; + +use XML::SAX; +use RaptorSAXHandler; +use Getopt::Long; + +my @logfiles; +my $basedir = ''; +my $help = 0; +GetOptions(( + 'log:s' => \@logfiles, + 'basedir:s' => \$basedir, + 'help!' => \$help +)); + +$help = 1 if (!@logfiles); + +if ($help) +{ + print "Extracts text which doesn't belong to recipes from a raptor log file\n"; + print "Usage: perl unreciped_text.pl --log=FILE1 --log=FILE2 [OPTIONS]\n"; + print "where OPTIONS are:\n"; + print "\t--basedir=DIR Generate output file under DIR\n"; + exit(0); +} + +my $reset_status = {}; +my $buildlog_status = {}; +my $buildlog_subtag_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {'?default?'=>$buildlog_subtag_status}; +$buildlog_status->{on_start} = 'main::on_start_buildlog'; +$buildlog_status->{on_end} = 'main::on_end_buildlog'; +$buildlog_status->{on_chars} = 'main::on_chars_buildlog'; + +$buildlog_subtag_status->{name} = 'buildlog_subtag_status'; +$buildlog_subtag_status->{next_status} = {}; + +my $characters = ''; + +if (!$basedir) +{ + $basedir = time; + + print "Using $basedir as basedir.\n"; +} +if (-d $basedir) +{ + print "Directory $basedir exists. Quitting.\n"; + exit(1); +} +mkdir($basedir); +#print "Created dir $basedir.\n"; + +my $saxhandler = RaptorSAXHandler->new(); +$saxhandler->set_init_status($reset_status); +my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); +for (@logfiles) +{ + $parser->parse_uri($_); +} + + +sub on_start_buildlog +{ + my $filename = "$basedir/unreciped.txt"; + print "Writing unreciped file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); +} + +sub on_chars_buildlog +{ + my ($ch) = @_; + + my $characters = $ch->{Data}; + + print FILE $characters if ($characters =~ m,[^\s^\r^\n],); + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_buildlog +{ + close(FILE); +} \ No newline at end of file