# HG changeset patch # User Brendan Donegan brendand@symbian.org # Date 1248270930 -3600 # Node ID 48a6e55f691be9c239724e73642699bb0ceee36f # Parent cdf0b7c501d2e7e9d5a4779a472f9b23b5dff177# Parent cf05cf95f5d35b7127d5718f27febda5ca81a62f Merge to tip diff -r cdf0b7c501d2 -r 48a6e55f691b .hgtags --- a/.hgtags Wed Jul 22 14:52:45 2009 +0100 +++ b/.hgtags Wed Jul 22 14:55:30 2009 +0100 @@ -1,2 +1,3 @@ 29394d95b3e1a63e73a3f3e31f9bab42cc1437f5 fbf_configs_0.0.a 36d82a2f1f927cbe91926e19c69fe74d9170b7de PDK_2.0.c +633b96a83005daab9464a00b8fcbcfa96a247c26 PDK_2.0.d diff -r cdf0b7c501d2 -r 48a6e55f691b common/build.xml --- a/common/build.xml Wed Jul 22 14:52:45 2009 +0100 +++ b/common/build.xml Wed Jul 22 14:55:30 2009 +0100 @@ -27,7 +27,7 @@ - + @@ -37,15 +37,15 @@ - - - - - - - - - + + + + + + + + + @@ -80,20 +80,20 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -108,79 +108,106 @@ - - + + + + + + + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - + + + - - - - + + + + + + + + + @@ -337,6 +364,9 @@ + + + @@ -382,10 +412,23 @@ - + - - + + + + + + + + + + + + + + + @@ -446,7 +489,7 @@ - + [SF-BUILD-NOPREP] @@ -464,7 +507,27 @@ - + + + + + + + + ant: antProperties() + data: csv(${sf.spec.sourcesync.sourcespecdir}/${sf.spec.sourcesync.sourcespecfile}, {separator:','}) + + + + + + + + + + + + + + + + - + - + - - - - - + - + - - - - + - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + @@ -599,12 +665,12 @@ - + - - + + @@ -615,8 +681,7 @@ - - + @@ -683,6 +748,7 @@ + @@ -714,7 +780,7 @@ - + @@ -730,71 +796,128 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ant: antProperties() - files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]}) - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ant: antProperties() + raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,component,phase,recipe,file,line]}) + files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]}) + + + + + + + + + + + + + + + + - + + + + + + + + + + + + diff -r cdf0b7c501d2 -r 48a6e55f691b common/common_props.ant.xml --- a/common/common_props.ant.xml Wed Jul 22 14:52:45 2009 +0100 +++ b/common/common_props.ant.xml Wed Jul 22 14:55:30 2009 +0100 @@ -15,7 +15,7 @@ - + @@ -26,7 +26,7 @@ - + @@ -94,8 +94,8 @@ - - + + diff -r cdf0b7c501d2 -r 48a6e55f691b common/diamonds/sf-run-analysis.xml.ftl --- a/common/diamonds/sf-run-analysis.xml.ftl Wed Jul 22 14:52:45 2009 +0100 +++ b/common/diamonds/sf-run-analysis.xml.ftl Wed Jul 22 14:55:30 2009 +0100 @@ -1,11 +1,17 @@ + 10 + +<#assign raptor_count=0/> +<#list raptor_summary as raptor_item> +<#assign raptor_count=raptor_count+1 /> + ${ant['sf.job.totalyarperrors']} 0 - 0 + ${raptor_count} 0 0 0 @@ -20,6 +26,11 @@ file:///${f.path} + + log + Raptor summary + file:///${ant['sf.spec.publish.networkdrive']}\${ant['sf.spec.job.name']}\builds\${ant['sf.spec.job.codeline']}\${ant['build.id']}\logs\raptorbits\summary.csv + \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/templates/source-spec.ant.xml.ftl --- a/common/templates/source-spec.ant.xml.ftl Wed Jul 22 14:52:45 2009 +0100 +++ b/common/templates/source-spec.ant.xml.ftl Wed Jul 22 14:55:30 2009 +0100 @@ -62,7 +62,7 @@ - + @@ -87,6 +87,7 @@ + diff -r cdf0b7c501d2 -r 48a6e55f691b common/templates/truclean.ant.xml.ftl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/templates/truclean.ant.xml.ftl Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,32 @@ + + + +<#assign target_depends=""/> +<#assign count=0/> +<#assign dollar="$"/> + +<#list data as pkg_detail> + + + + + + + + + + + + + <#if (count==0)> + <#assign target_depends="${target_depends}"+"sf-truclean-${count}"/> + + <#if (count>0)> + <#assign target_depends="${target_depends}"+","+"sf-truclean-${count}"/> + + <#assign count=count+1/> + + + + + \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/templates/zip.cfg.xml.ftl.template --- a/common/templates/zip.cfg.xml.ftl.template Wed Jul 22 14:52:45 2009 +0100 +++ b/common/templates/zip.cfg.xml.ftl.template Wed Jul 22 14:55:30 2009 +0100 @@ -38,6 +38,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/ats/bctest/bctest.pl --- a/common/tools/ats/bctest/bctest.pl Wed Jul 22 14:52:45 2009 +0100 +++ b/common/tools/ats/bctest/bctest.pl Wed Jul 22 14:55:30 2009 +0100 @@ -15,6 +15,7 @@ copy("bctest.xml", "temp/test.xml"); copy($epoc . "winscw/c/UIFW.bat", "temp/bctest/general/UIFW.bat"); copy($epoc . "winscw/c/BCTestLog/config.xml", "temp/bctest/BCTestLog/general/config.xml"); +copy($epoc . "winscw/c/BCTestLog/sdkversion.log", "temp/bctest/BCTestLog/general/sdkversion.log"); copy($epoc . "winscw/c/BCTestLog/release_note.txt", "temp/bctest/BCTestLog/general/release_note.txt"); copy($epoc . "data/z/resource/apps/bctestlauncher_aif.mbm", "temp/bctest/winscw_udeb/z/resource/apps/bctestlauncher_aif.mbm"); diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/ats/bctest/bctest.xml --- a/common/tools/ats/bctest/bctest.xml Wed Jul 22 14:52:45 2009 +0100 +++ b/common/tools/ats/bctest/bctest.xml Wed Jul 22 14:55:30 2009 +0100 @@ -40,6 +40,15 @@ install + + + + + + + install + + @@ -2036,6 +2045,7 @@ bctest/general/UIFW.bat bctest/BCTestLog/general/release_note.txt bctest/BCTestLog/general/config.xml + bctest/BCTestLog/general/sdkversion.log bctest/winscw_udeb/bctestutil.dll bctest/winscw_udeb/bctestlauncher.exe bctest/winscw_udeb/z/private/10003a3f/apps/bctestlauncher_reg.r01 diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/ats/smoketest/Group/smoketest.xml --- a/common/tools/ats/smoketest/Group/smoketest.xml Wed Jul 22 14:52:45 2009 +0100 +++ b/common/tools/ats/smoketest/Group/smoketest.xml Wed Jul 22 14:55:30 2009 +0100 @@ -208,4 +208,7 @@ smoketest/winscw_udeb/Smoketest_Timew_Server.exe smoketest/winscw_udeb/SmokeTest_Utils.dll + + DiamondsAction + diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/populateziptemplate.pl --- a/common/tools/populateziptemplate.pl Wed Jul 22 14:52:45 2009 +0100 +++ b/common/tools/populateziptemplate.pl Wed Jul 22 14:55:30 2009 +0100 @@ -25,7 +25,8 @@ my $sourcesCSV = shift or die "First arg must be source csv file"; my $template = shift or die "Second arg must be template file"; my $ftl = shift or die "Third arg must be output file"; -shift and die "No more than three arguments please"; +my $rndExcludes = shift or die "Fourth arg must be rnd-excludes file"; +shift and die "No more than four arguments please"; # Load CSV open my $csvText, "<", $sourcesCSV or die; @@ -91,7 +92,9 @@ elsif ($package->{source} =~ m{/rnd/([^/]+)/([^/]+)}) { # RnD repository - my $name = "bin_rnd_$1_$2"; + my $name = "binaries_$2"; + if ($1 eq "rndonly") { $name="bin_$1_$2";} + # Create a zip object push @{$zipConfig->{config}->{config}->{src}->{config}->{rnd}->{config}}, { @@ -141,6 +144,6 @@ $xml->XMLout($zipConfig, OutputFile => $ftl, XMLDecl => 1, RootName => 'build', KeyAttr => $keyAttr); # Output all rnd files into exclude list for later -open FILE, "> rnd_excludefile.txt" or die "Cannot write exludefile!"; -print FILE @allRndFiles; -close FILE; \ No newline at end of file +open my $fh, ">", $rndExcludes or die "Cannot write exlude file!"; +print $fh @allRndFiles; +close $fh; diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorCommon.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorCommon.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,35 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Common constants for the raptor parser suite + +package RaptorCommon; + +our $CATEGORY_GENERAL = 'general'; + +our $CATEGORY_RAPTORERROR = 'raptor_error'; +our $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION = 'cannot_process_schema_version'; +our $CATEGORY_RAPTORERROR_NOBLDINFFOUND = 'no_bld_inf_found'; + +our $SEVERITY_UNKNOWN = 'unknown'; +our $SEVERITY_CRITICAL = 'critical'; + +sub dump_fault +{ + my ($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line) = @_; + + open(SUMMARY, ">>$::basedir/summary.csv"); + print SUMMARY "$category,$subcategory,$severity,$component,$phase,$recipe,$file,$line\n"; + close(SUMMARY); +} + +1; diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorError.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorError.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,107 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract, analyzes and dumps raptor errors i.e. content of tags from a raptor log file + +package RaptorError; + +use strict; +use RaptorCommon; + +our $reset_status = {}; +my $buildlog_status = {}; +my $buildlog_error_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {error=>$buildlog_error_status}; + +$buildlog_error_status->{name} = 'buildlog_error_status'; +$buildlog_error_status->{next_status} = {}; +$buildlog_error_status->{on_start} = 'RaptorError::on_start_buildlog_error'; +$buildlog_error_status->{on_end} = 'RaptorError::on_end_buildlog_error'; +$buildlog_error_status->{on_chars} = 'RaptorError::on_chars_buildlog_error'; + +my $filename = ''; + +my $characters = ''; + +my $category = $RaptorCommon::CATEGORY_RAPTORERROR; + +sub process +{ + my ($text, $component, $phase, $recipe, $file, $line) = @_; + + my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + + if ($text =~ m,Cannot process schema version .* of file,) + { + $severity = $RaptorCommon::SEVERITY_CRITICAL; + my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,No bld\.inf found at,) + { + $severity = $RaptorCommon::SEVERITY_CRITICAL; + my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_NOBLDINFFOUND; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + else # log everything by default + { + $severity = $RaptorCommon::SEVERITY_UNKNOWN; + my $subcategory = ''; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } +} + +sub on_start_buildlog_error +{ + $filename = "$::basedir/errors.txt"; + print "Writing error file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); +} + +sub on_chars_buildlog_error +{ + my ($ch) = @_; + + #print "on_chars_buildlog_error\n"; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_buildlog_error +{ + #print "on_end_buildlog_error\n"; + + print FILE $characters if ($characters =~ m,[^\s^\r^\n],); + print FILE "\n" if ($characters !~ m,[\r\n]$, ); + close(FILE); + + # get the line number - not really optimized + my $linecount = 0; + open(FILE, "$filename"); + for ($linecount = 0; ; $linecount++) { } + close(FILE); + + process($characters, '', '', '', "errors.txt", $linecount); + + $characters = ''; +} + + +1; \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorInfo.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorInfo.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,89 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract, analyzes and dumps raptor info text i.e. content of tags from a raptor log file + +package RaptorInfo; + +use strict; +use RaptorCommon; + +our $reset_status = {}; +my $buildlog_status = {}; +my $buildlog_info_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {info=>$buildlog_info_status}; + +$buildlog_info_status->{name} = 'buildlog_info_status'; +$buildlog_info_status->{next_status} = {}; +$buildlog_info_status->{on_start} = 'RaptorInfo::on_start_buildlog_info'; +$buildlog_info_status->{on_end} = 'RaptorInfo::on_end_buildlog_info'; +$buildlog_info_status->{on_chars} = 'RaptorInfo::on_chars_buildlog_info'; + +my $characters = ''; + +my $category = $RaptorCommon::CATEGORY_RAPTORINFO; + +sub process +{ + my ($text) = @_; + + my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + + if ($text =~ m,unmatchable,) + { + $severity = $RaptorCommon::SEVERITY_CRITICAL; + + #dump_error($category, $severity, $text); + print "$category, $severity, $text\n"; + } +} + +sub on_start_buildlog_info +{ + my $filename = "$::basedir/info.txt"; + print "Writing info file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); +} + +sub on_chars_buildlog_info +{ + my ($ch) = @_; + + #print "on_chars_buildlog_info\n"; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_buildlog_info +{ + #print "on_end_buildlog_info\n"; + + process($characters); + + print FILE $characters if ($characters =~ m,[^\s^\r^\n],); + print FILE "\n" if ($characters !~ m,[\r\n]$, ); + + $characters = ''; + + close(FILE); +} + + +1; \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorRecipe.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorRecipe.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,128 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract, analyzes and dumps raptor recipes i.e. content of tags from a raptor log file + +package RaptorRecipe; + +use strict; +use RaptorCommon; + +our $reset_status = {}; +my $buildlog_status = {}; +my $buildlog_recipe_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {recipe=>$buildlog_recipe_status}; + +$buildlog_recipe_status->{name} = 'buildlog_recipe_status'; +$buildlog_recipe_status->{next_status} = {}; +$buildlog_recipe_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe'; +$buildlog_recipe_status->{on_end} = 'RaptorRecipe::on_end_buildlog_recipe'; +$buildlog_recipe_status->{on_chars} = 'RaptorRecipe::on_chars_buildlog_recipe'; + +my $characters = ''; +my $recipe_info = {}; + +my $category = $RaptorCommon::CATEGORY_RAPTORRECIPE; + +sub process +{ + my ($text) = @_; + + my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + + if ($text =~ m,Cannot process schema version .* of file,) + { + $severity = $RaptorCommon::SEVERITY_CRITICAL; + + #dump_recipe($category, $severity, $text); + print "$category, $severity, $text\n"; + } +} + +sub on_start_buildlog_recipe +{ + my ($el) = @_; + + $recipe_info = {}; + $characters = ''; + + my $attrstring = ''; + my $bldinf = ''; + + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + if ($attributes->{$_}->{'LocalName'} eq 'bldinf') + { + $bldinf = $attributes->{$_}->{'Value'}; + } + + $attrstring .= "$attributes->{$_}->{'LocalName'}}='$attributes->{$_}->{'Value'}' "; + + } + + if ($bldinf eq '') + { + print "WARNING: recipe tag with no bldinf attribute. Associating to package unknown/unknown\n"; + $bldinf = "/sf/unknown/unknown/group/bld.inf"; + } + + $recipe_info->{bldinf} = $bldinf; + + $characters = "\n"; +} + +sub on_chars_buildlog_recipe +{ + my ($ch) = @_; + + #print "on_chars_buildlog_recipe\n"; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_buildlog_recipe +{ + #print "on_end_buildlog_recipe\n"; + + $characters .= "\n\n"; + + my $normalized = lc($recipe_info->{bldinf}); + $normalized =~ s,^[A-Za-z]:,,; + $normalized =~ s,[\\],/,g; + + $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; + my $layer = $1; + my $package = $2; + + mkdir("$::basedir/recipes"); + mkdir("$::basedir/recipes/$layer"); + mkdir("$::basedir/recipes/$layer/$package"); + + my $filename = "$::basedir/recipes/$layer/$package/recipes.txt"; + + print "Writing recipes file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); + print FILE $characters; + close(FILE); +} + + +1; \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorReleaseable.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorReleaseable.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,288 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract releaseable (whatlog) information + +package RaptorReleaseable; + +use strict; +use RaptorCommon; + +our $reset_status = {}; +my $buildlog_status = {}; +my $whatlog_status = {}; +my $bitmap_status = {}; +my $resource_status = {}; +my $build_status = {}; +my $export_status = {}; +my $stringtable_status = {}; +my $archive_status = {}; +my $archive_member_status = {}; +my $whatlog_default_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {whatlog=>$whatlog_status}; + +$whatlog_status->{name} = 'whatlog_status'; +$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status}; +$whatlog_status->{on_start} = 'RaptorReleaseable::on_start_whatlog'; +$whatlog_status->{on_end} = 'RaptorReleaseable::on_end_whatlog'; + +$bitmap_status->{name} = 'bitmap_status'; +$bitmap_status->{next_status} = {}; +$bitmap_status->{on_start} = 'RaptorReleaseable::on_start_bitmap'; +$bitmap_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; +$bitmap_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; + +$resource_status->{name} = 'resource_status'; +$resource_status->{next_status} = {}; +$resource_status->{on_start} = 'RaptorReleaseable::on_start_resource'; +$resource_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; +$resource_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; + +$build_status->{name} = 'build_status'; +$build_status->{next_status} = {}; +$build_status->{on_start} = 'RaptorReleaseable::on_start_build'; +$build_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; +$build_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; + +$stringtable_status->{name} = 'stringtable_status'; +$stringtable_status->{next_status} = {}; +$stringtable_status->{on_start} = 'RaptorReleaseable::on_start_stringtable'; +$stringtable_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; +$stringtable_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; + +$archive_status->{name} = 'archive_status'; +$archive_status->{next_status} = {member=>$archive_member_status}; + +$archive_member_status->{name} = 'archive_member_status'; +$archive_member_status->{next_status} = {}; +$archive_member_status->{on_start} = 'RaptorReleaseable::on_start_archive_member'; +$archive_member_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; +$archive_member_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; + +$export_status->{name} = 'export_status'; +$export_status->{next_status} = {}; +$export_status->{on_start} = 'RaptorReleaseable::on_start_export'; + +$whatlog_default_status->{name} = 'whatlog_default_status'; +$whatlog_default_status->{next_status} = {}; +$whatlog_default_status->{on_start} = 'RaptorReleaseable::on_start_whatlog_default'; + +my $whatlog_info = {}; +my $curbldinf = 'unknown'; +my $curconfig = 'unknown'; +my $curfiletype = 'unknown'; +my $characters = ''; + +sub on_start_whatlog +{ + my ($el) = @_; + + $whatlog_info = {}; + + my $bldinf = ''; + my $config = ''; + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + #print "reading attribute $_\n"; + if ($attributes->{$_}->{'LocalName'} eq 'bldinf') + { + $bldinf = $attributes->{$_}->{'Value'}; + #print "bldinf=$bldinf\n"; + } + elsif ($attributes->{$_}->{'LocalName'} eq 'config') + { + $config = $attributes->{$_}->{'Value'}; + $config =~ s,\.whatlog$,,; + } + } + + if ($bldinf eq '') + { + print "WARNING: whatlog tag with no bldinf attribute. Skipping\n"; + return; + } + + $curbldinf = $bldinf; + $curconfig = $config; + $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf}); + $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig}); +} + +sub on_start_whatlog_subtag +{ + my ($ft) = @_; + + $curfiletype = $ft; + $characters = ''; + $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}); +} + +sub on_chars_whatlog_subtag +{ + my ($ch) = @_; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_whatlog_subtag +{ + $characters = normalize_filepath($characters); + + push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters); + + $curfiletype = 'unknown'; + $characters = ''; +} + +sub on_start_bitmap +{ + on_start_whatlog_subtag('bitmap'); +} + +sub on_start_resource +{ + on_start_whatlog_subtag('resource'); +} + +sub on_start_build +{ + on_start_whatlog_subtag('build'); +} + +sub on_start_stringtable +{ + on_start_whatlog_subtag('stringtable'); +} + +sub on_start_archive_member +{ + on_start_whatlog_subtag('export'); +} + +sub on_start_export +{ + my ($el) = @_; + + $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export}); + + my $destination = ''; + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + #print "reading attribute $_\n"; + if ($attributes->{$_}->{'LocalName'} eq 'destination') + { + $destination = $attributes->{$_}->{'Value'}; + #print "destination=$destination\n"; + last; + } + } + + if ($destination eq '') + { + print "WARNING: export tag with no destination attribute. Skipping\n"; + return; + } + + $destination = normalize_filepath($destination); + + push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination); +} + +sub on_end_whatlog +{ + my $unknown_counter = 0; + + for my $bldinf (keys %{$whatlog_info}) + { + for my $config (keys %{$whatlog_info->{$bldinf}}) + { + my $normalized = lc($bldinf); + $normalized =~ s,^[A-Za-z]:,,; + $normalized =~ s,[\\],/,g; + + $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; + my $layer = $1; + my $package = $2; + + mkdir("$::basedir/releaseables"); + mkdir("$::basedir/releaseables/$layer"); + mkdir("$::basedir/releaseables/$layer/$package"); + + my $filename = "$::basedir/releaseables/$layer/$package/info.tsv"; + + print "Writing info file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); + + for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}}) + { + for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}})) + { + print FILE "$_\t$filetype\t$config\n"; + } + } + + close(FILE); + } + } +} + +sub normalize_filepath +{ + my ($filepath) = @_; + + if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,) + { + print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n"; + } + + # strip all CR's and NL's + $filepath =~ s,[\r\n],,g; + + # strip all whitespaces at string start/end + $filepath =~ s,^\s+,,g; + $filepath =~ s,\s+$,,g; + + # remove drive letter and colon from the beginning of the string + $filepath =~ s,^[A-Za-z]:,,; + + # normalize slashes + $filepath =~ s,\\,/,g; + $filepath =~ s,//,/,g; + + if ($filepath !~ m,^/epoc32/,i) + { + print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n"; + } + + return $filepath; +} + +sub on_start_whatlog_default +{ + my ($el) = @_; + + my $tagname = $el->{LocalName}; + + print "WARNING: unsupported tag '$tagname' in context\n"; +} + +1; \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorSAXHandler.pm --- a/common/tools/raptor/RaptorSAXHandler.pm Wed Jul 22 14:52:45 2009 +0100 +++ b/common/tools/raptor/RaptorSAXHandler.pm Wed Jul 22 14:55:30 2009 +0100 @@ -22,11 +22,13 @@ return bless {}, $type; } -sub set_init_status +sub add_observer { - my ($self, $initialstatus) = @_; + my ($self, $name, $initialstatus) = @_; - $self->{status} = $initialstatus; + $self->{observers} = {} if (!defined $self->{observers}); + + $self->{observers}->{$name} = $initialstatus; } sub start_document @@ -46,23 +48,30 @@ #print "start_element($tagname)\n"; - - if (defined $self->{status}->{next_status}->{$tagname}) + for my $observer (keys %{$self->{observers}}) { - my $oldstatus = $self->{status}; - $self->{status} = $self->{status}->{next_status}->{$tagname}; - #print "status is now $self->{status}->{name}\n"; - $self->{status}->{next_status}->{$tagname} = $oldstatus; - &{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start}); - } - elsif (defined $self->{status}->{next_status}->{'?default?'}) - { - #print "changing to default status\n"; - my $oldstatus = $self->{status}; - $self->{status} = $self->{status}->{next_status}->{'?default?'}; - #print "status is now ?default?\n"; - $self->{status}->{next_status}->{$tagname} = $oldstatus; - &{$self->{status}->{on_start}}($el) if (defined $self->{status}->{on_start}); + #print "processing observer $observer: $self->{observers}->{$observer} $self->{observers}->{$observer}->{name}\n"; + #for (keys %{$self->{observers}->{$observer}->{next_status}}) {print "$_\n";} + + if (defined $self->{observers}->{$observer}->{next_status}->{$tagname}) + { + #print "processing observer $observer\n"; + my $oldstatus = $self->{observers}->{$observer}; + $self->{observers}->{$observer} = $self->{observers}->{$observer}->{next_status}->{$tagname}; + #print "$observer: status is now $self->{observers}->{$observer}->{name}\n"; + $self->{observers}->{$observer}->{next_status}->{$tagname} = $oldstatus; + &{$self->{observers}->{$observer}->{on_start}}($el) if (defined $self->{observers}->{$observer}->{on_start}); + } + elsif (defined $self->{observers}->{$observer}->{next_status}->{'?default?'}) + { + #print "processing observer $observer\n"; + #print "changing to default status\n"; + my $oldstatus = $self->{observers}->{$observer}; + $self->{observers}->{$observer} = $self->{observers}->{$observer}->{next_status}->{'?default?'}; + #print "status is now ?default?\n"; + $self->{observers}->{$observer}->{next_status}->{$tagname} = $oldstatus; + &{$self->{observers}->{$observer}->{on_start}}($el) if (defined $self->{observers}->{$observer}->{on_start}); + } } } @@ -75,11 +84,14 @@ #print "end_element($tagname)\n"; - if (defined $self->{status}->{next_status}->{$tagname}) + for my $observer (keys %{$self->{observers}}) { - &{$self->{status}->{on_end}}($el) if (defined $self->{status}->{on_end}); - $self->{status} = $self->{status}->{next_status}->{$tagname}; - #print "status is now $self->{status}->{name}\n"; + if (defined $self->{observers}->{$observer}->{next_status}->{$tagname}) + { + &{$self->{observers}->{$observer}->{on_end}}($el) if (defined $self->{observers}->{$observer}->{on_end}); + $self->{observers}->{$observer} = $self->{observers}->{$observer}->{next_status}->{$tagname}; + #print "status is now $self->{observers}->{$observer}->{name}\n"; + } } } @@ -87,7 +99,10 @@ { my ($self, $ch) = @_; - &{$self->{status}->{on_chars}}($ch) if (defined $self->{status}->{on_chars}); + for my $observer (keys %{$self->{observers}}) + { + &{$self->{observers}->{$observer}->{on_chars}}($ch) if (defined $self->{observers}->{$observer}->{on_chars}); + } } 1; diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorUnreciped.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorUnreciped.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,63 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract, analyzes and dumps text in context which doesn't belong to any tags + +package RaptorUnreciped; + +use strict; +use RaptorCommon; + +our $reset_status = {}; +my $buildlog_status = {}; +my $buildlog_subtag_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {'?default?'=>$buildlog_subtag_status}; +$buildlog_status->{on_start} = 'RaptorUnreciped::on_start_buildlog'; +$buildlog_status->{on_end} = 'RaptorUnreciped::on_end_buildlog'; +$buildlog_status->{on_chars} = 'RaptorUnreciped::on_chars_buildlog'; + +$buildlog_subtag_status->{name} = 'buildlog_subtag_status'; +$buildlog_subtag_status->{next_status} = {}; + +my $characters = ''; + +sub on_start_buildlog +{ + my $filename = "$::basedir/unreciped.txt"; + print "Writing unreciped file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); +} + +sub on_chars_buildlog +{ + my ($ch) = @_; + + my $characters = $ch->{Data}; + + print FILE $characters if ($characters =~ m,[^\s^\r^\n],); + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_buildlog +{ + close(FILE); +} + + +1; \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/RaptorWarning.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/RaptorWarning.pm Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,89 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract, analyzes and dumps raptor warnings i.e. content of tags from a raptor log file + +package RaptorWarning; + +use strict; +use RaptorCommon; + +our $reset_status = {}; +my $buildlog_status = {}; +my $buildlog_warning_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {warning=>$buildlog_warning_status}; + +$buildlog_warning_status->{name} = 'buildlog_warning_status'; +$buildlog_warning_status->{next_status} = {}; +$buildlog_warning_status->{on_start} = 'RaptorWarning::on_start_buildlog_warning'; +$buildlog_warning_status->{on_end} = 'RaptorWarning::on_end_buildlog_warning'; +$buildlog_warning_status->{on_chars} = 'RaptorWarning::on_chars_buildlog_warning'; + +my $characters = ''; + +my $category = $RaptorCommon::CATEGORY_RAPTORWARNING; + +sub process +{ + my ($text) = @_; + + my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + + if ($text =~ m,unmatchable,) + { + $severity = $RaptorCommon::SEVERITY_CRITICAL; + + #dump_error($category, $severity, $text); + print "$category, $severity, $text\n"; + } +} + +sub on_start_buildlog_warning +{ + my $filename = "$::basedir/warnings.txt"; + print "Writing warning file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); +} + +sub on_chars_buildlog_warning +{ + my ($ch) = @_; + + #print "on_chars_buildlog_warning\n"; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_buildlog_warning +{ + #print "on_end_buildlog_warning\n"; + + process($characters); + + print FILE $characters if ($characters =~ m,[^\s^\r^\n],); + print FILE "\n" if ($characters !~ m,[\r\n]$, ); + + $characters = ''; + + close(FILE); +} + + +1; \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/package_what.pl --- a/common/tools/raptor/package_what.pl Wed Jul 22 14:52:45 2009 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,340 +0,0 @@ -# Copyright (c) 2009 Symbian Foundation Ltd -# This component and the accompanying materials are made available -# under the terms of the License "Eclipse Public License v1.0" -# which accompanies this distribution, and is available -# at the URL "http://www.eclipse.org/legal/epl-v10.html". -# -# Initial Contributors: -# Symbian Foundation Ltd - initial contribution. -# -# Contributors: -# -# Description: -# Extracts whatlog information from a raptor log file - -use strict; - -use XML::SAX; -use RaptorSAXHandler; -use Getopt::Long; - -my @logfiles; -my $basedir = ''; -my $append = 0; -my $help = 0; -GetOptions(( - 'log:s' => \@logfiles, - 'basedir:s' => \$basedir, - 'append!' => \$append, - 'help!' => \$help -)); - -$help = 1 if (!@logfiles); - -if ($help) -{ - print "Extracts whatlog information from a raptor log file\n"; - print "Usage: perl package_what.pl --log=FILE1 --log=FILE2 [OPTIONS]\n"; - print "where OPTIONS are:\n"; - print "\t--basedir=DIR Generate info files under DIR\n"; - print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n"; - exit(0); -} - -my $reset_status = {}; -my $buildlog_status = {}; -my $whatlog_status = {}; -my $bitmap_status = {}; -my $resource_status = {}; -my $build_status = {}; -my $export_status = {}; -my $stringtable_status = {}; -my $archive_status = {}; -my $archive_member_status = {}; -my $whatlog_default_status = {}; - -$reset_status->{name} = 'reset_status'; -$reset_status->{next_status} = {buildlog=>$buildlog_status}; - -$buildlog_status->{name} = 'buildlog_status'; -$buildlog_status->{next_status} = {whatlog=>$whatlog_status}; - -$whatlog_status->{name} = 'whatlog_status'; -$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status}; -$whatlog_status->{on_start} = 'main::on_start_whatlog'; -$whatlog_status->{on_end} = 'main::on_end_whatlog'; - -$bitmap_status->{name} = 'bitmap_status'; -$bitmap_status->{next_status} = {}; -$bitmap_status->{on_start} = 'main::on_start_bitmap'; -$bitmap_status->{on_end} = 'main::on_end_whatlog_subtag'; -$bitmap_status->{on_chars} = 'main::on_chars_whatlog_subtag'; - -$resource_status->{name} = 'resource_status'; -$resource_status->{next_status} = {}; -$resource_status->{on_start} = 'main::on_start_resource'; -$resource_status->{on_end} = 'main::on_end_whatlog_subtag'; -$resource_status->{on_chars} = 'main::on_chars_whatlog_subtag'; - -$build_status->{name} = 'build_status'; -$build_status->{next_status} = {}; -$build_status->{on_start} = 'main::on_start_build'; -$build_status->{on_end} = 'main::on_end_whatlog_subtag'; -$build_status->{on_chars} = 'main::on_chars_whatlog_subtag'; - -$stringtable_status->{name} = 'stringtable_status'; -$stringtable_status->{next_status} = {}; -$stringtable_status->{on_start} = 'main::on_start_stringtable'; -$stringtable_status->{on_end} = 'main::on_end_whatlog_subtag'; -$stringtable_status->{on_chars} = 'main::on_chars_whatlog_subtag'; - -$archive_status->{name} = 'archive_status'; -$archive_status->{next_status} = {member=>$archive_member_status}; - -$archive_member_status->{name} = 'archive_member_status'; -$archive_member_status->{next_status} = {}; -$archive_member_status->{on_start} = 'main::on_start_archive_member'; -$archive_member_status->{on_end} = 'main::on_end_whatlog_subtag'; -$archive_member_status->{on_chars} = 'main::on_chars_whatlog_subtag'; - -$export_status->{name} = 'export_status'; -$export_status->{next_status} = {}; -$export_status->{on_start} = 'main::on_start_export'; - -$whatlog_default_status->{name} = 'whatlog_default_status'; -$whatlog_default_status->{next_status} = {}; -$whatlog_default_status->{on_start} = 'main::on_start_whatlog_default'; - -my $whatlog_info = {}; -my $curbldinf = 'unknown'; -my $curconfig = 'unknown'; -my $curfiletype = 'unknown'; -my $characters = ''; - -if (!$basedir) -{ - $basedir = time; - - print "Using $basedir as basedir.\n"; -} -if (-d $basedir) -{ - if ($append) - { - print "Directory $basedir exists. Appending new info to it.\n"; - } - else - { - print "Directory $basedir exists. Quitting.\n"; - exit(1); - } -} -mkdir($basedir); -#print "Created dir $basedir.\n"; - -my $saxhandler = RaptorSAXHandler->new(); -$saxhandler->set_init_status($reset_status); -my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); -for (@logfiles) -{ - $parser->parse_uri($_); -} - - -sub on_start_whatlog -{ - my ($el) = @_; - - #print "on_start_whatlog\n"; - - $whatlog_info = {}; - - my $bldinf = ''; - my $config = ''; - my $attributes = $el->{Attributes}; - for (keys %{$attributes}) - { - #print "reading attribute $_\n"; - if ($attributes->{$_}->{'LocalName'} eq 'bldinf') - { - $bldinf = $attributes->{$_}->{'Value'}; - #print "bldinf=$bldinf\n"; - } - elsif ($attributes->{$_}->{'LocalName'} eq 'config') - { - $config = $attributes->{$_}->{'Value'}; - $config =~ s,\.whatlog$,,; - } - } - - if ($bldinf eq '') - { - print "WARNING: whatlog tag with no bldinf attribute. Skipping\n"; - return; - } - - $curbldinf = $bldinf; - $curconfig = $config; - $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf}); - $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig}); -} - -sub on_start_whatlog_subtag -{ - my ($ft) = @_; - - $curfiletype = $ft; - $characters = ''; - $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}); -} - -sub on_chars_whatlog_subtag -{ - my ($ch) = @_; - - $characters .= $ch->{Data}; - - #print "characters is now -->$characters<--\n"; -} - -sub on_end_whatlog_subtag -{ - $characters = normalize_filepath($characters); - - push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters); - - $curfiletype = 'unknown'; - $characters = ''; -} - -sub on_start_bitmap -{ - on_start_whatlog_subtag('bitmap'); -} - -sub on_start_resource -{ - on_start_whatlog_subtag('resource'); -} - -sub on_start_build -{ - on_start_whatlog_subtag('build'); -} - -sub on_start_stringtable -{ - on_start_whatlog_subtag('stringtable'); -} - -sub on_start_archive_member -{ - on_start_whatlog_subtag('export'); -} - -sub on_start_export -{ - my ($el) = @_; - - $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export}); - - my $destination = ''; - my $attributes = $el->{Attributes}; - for (keys %{$attributes}) - { - #print "reading attribute $_\n"; - if ($attributes->{$_}->{'LocalName'} eq 'destination') - { - $destination = $attributes->{$_}->{'Value'}; - #print "destination=$destination\n"; - last; - } - } - - if ($destination eq '') - { - print "WARNING: export tag with no destination attribute. Skipping\n"; - return; - } - - $destination = normalize_filepath($destination); - - push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination); -} - -sub on_end_whatlog -{ - my $unknown_counter = 0; - - for my $bldinf (keys %{$whatlog_info}) - { - for my $config (keys %{$whatlog_info->{$bldinf}}) - { - my $normalized = lc($bldinf); - $normalized =~ s,^[A-Za-z]:,,; - $normalized =~ s,[\\],/,g; - - $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; - my $layer = $1; - my $package = $2; - - mkdir("$basedir/$layer"); - mkdir("$basedir/$layer/$package"); - - my $filename = "$basedir/$layer/$package/info.tsv"; - - print "Writing info file $filename\n" if (!-f$filename); - open(FILE, ">>$filename"); - - for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}}) - { - for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}})) - { - print FILE "$_\t$filetype\t$config\n"; - } - } - - close(FILE); - } - } -} - -sub normalize_filepath -{ - my ($filepath) = @_; - - if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,) - { - print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n"; - } - - # strip all CR's and NL's - $filepath =~ s,[\r\n],,g; - - # strip all whitespaces at string start/end - $filepath =~ s,^\s+,,g; - $filepath =~ s,\s+$,,g; - - # remove drive letter and colon from the beginning of the string - $filepath =~ s,^[A-Za-z]:,,; - - # normalize slashes - $filepath =~ s,\\,/,g; - $filepath =~ s,//,/,g; - - if ($filepath !~ m,^/epoc32/,i) - { - print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n"; - } - - return $filepath; -} - -sub on_start_whatlog_default -{ - my ($el) = @_; - - my $tagname = $el->{LocalName}; - - print "WARNING: unsupported tag '$tagname' in context\n"; -} \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/parse.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/parse.pl Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,126 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Run the raptor parsers + +use strict; +use RaptorReleaseable; +use RaptorError; +use RaptorWarning; +use RaptorInfo; +use RaptorUnreciped; +use RaptorRecipe; + +use XML::SAX; +use RaptorSAXHandler; +use Getopt::Long; + +my @logfiles; +my $releaseable_module = 0; +my $error_module = 0; +my $warning_module = 0; +my $info_module = 0; +my $unreciped_module = 0; +my $recipe_module = 0; +our $basedir = ''; +my $append = 0; +my $help = 0; +GetOptions(( + 'log:s' => \@logfiles, + 'releaseable!' => \$releaseable_module, + 'error!' => \$error_module, + 'warning!' => \$warning_module, + 'info!' => \$info_module, + 'unreciped!' => \$unreciped_module, + 'recipe!' => \$recipe_module, + 'basedir:s' => \$basedir, + 'append!' => \$append, + 'help!' => \$help +)); + +$help = 1 if (!@logfiles); + +if ($help) +{ + print "Run the raptor parsers\n"; + print "Usage: perl parse.pl [MODULES] --log=FILE1 --log=FILE2 ... [OPTIONS]\n"; + print "where MODULES are:\n"; + print "\t--releaseable Extract releaseable (whatlog) information\n"; + print "\t--error Extracts raptor errors, i.e. content of tags\n"; + print "\t--warning Extracts raptor warnings, i.e. content of tags\n"; + print "\t--info Extracts raptor info text i.e. content of tags from a raptor log file\n"; + print "\t--unreciped Extracts output text in context which doesn't belong to any tags\n"; + print "\t--recipe Extract, analyzes and dumps raptor recipes i.e. content of tags from a raptor log file\n"; + print "where OPTIONS are:\n"; + print "\t--basedir=DIR Generate output file under DIR\n"; + print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n"; + exit(0); +} + +if (!$basedir) +{ + $basedir = time; + + print "Using $basedir as basedir.\n"; +} +if (-d $basedir) +{ + if ($append) + { + print "Directory $basedir exists. Appending new info to it.\n"; + } + else + { + print "Directory $basedir exists. Quitting.\n"; + exit(1); + } +} +mkdir($basedir); +#print "Created dir $basedir.\n"; + +my $saxhandler = RaptorSAXHandler->new(); +if ($releaseable_module) +{ + print "Adding RaptorReleaseable module to the observers\n"; + $saxhandler->add_observer('RaptorReleaseable', $RaptorReleaseable::reset_status); +} +if ($error_module) +{ + print "Adding RaptorError module to the observers\n"; + $saxhandler->add_observer('RaptorError', $RaptorError::reset_status); +} +if ($warning_module) +{ + print "Adding RaptorWarning module to the observers\n"; + $saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status); +} +if ($info_module) +{ + print "Adding RaptorInfo module to the observers\n"; + $saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status); +} +if ($unreciped_module) +{ + print "Adding RaptorUnreciped module to the observers\n"; + $saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status); +} +if ($recipe_module) +{ + print "Adding RaptorRecipe module to the observers\n"; + $saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status); +} +my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); +for (@logfiles) +{ + $parser->parse_uri($_); +} + diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/preprocess_log.pl --- a/common/tools/raptor/preprocess_log.pl Wed Jul 22 14:52:45 2009 +0100 +++ b/common/tools/raptor/preprocess_log.pl Wed Jul 22 14:55:30 2009 +0100 @@ -1,3 +1,4 @@ +#!perl -w # Copyright (c) 2009 Symbian Foundation Ltd # This component and the accompanying materials are made available # under the terms of the License "Eclipse Public License v1.0" @@ -16,56 +17,49 @@ use Getopt::Long; -my $infile = ''; -my $outfile = ''; -my $basedir = ''; my $help = 0; -GetOptions(( - 'in:s' => \$infile, - 'out:s' => \$outfile, - 'help!' => \$help -)); - -$help = 1 if (!$infile); +GetOptions( + 'help!' => \$help, +); if ($help) { - print "Preprocess a raptor log, trying to countermeasure a list of known anomalies\n"; - print "Usage: perl preprocess_log.pl --in=INFILE --out=OUTFILE\n"; + warn <<"EOF"; +Preprocess a raptor log, trying to countermeasure a list of known anomalies + +Usage: perl preprocess_log.pl < INFILE > OUTFILE +EOF exit(0); } -open(INFILE, $infile); -open(OUTFILE, ">$outfile"); - -for my $line () +while (my $line = <>) { - if ($line =~ m,<[^<^>]+>.*&.*]+>,) + if ($line =~ m{<[^<^>]+>.*&.*]+>}) { $line = escape_ampersand($line); } - elsif ($line =~ m,<\?xml\s.*encoding=.*\".*\?>,) + elsif ($line =~ m{<\?xml\s.*encoding=.*\".*\?>}) { $line = set_encoding_utf8($line); } + elsif ($line =~ m{}) + { + $line = unterminated_archive_tag($line, scalar <>, $.) + } - print OUTFILE $line; + print $line; } -close(OUTFILE); -close(INFILE); - - sub escape_ampersand { my ($line) = @_; - print "escape_ampersand\n"; - print "in: $line"; + warn "escape_ampersand\n"; + warn "in: $line"; $line =~ s,&,&,g; - print "out: $line"; + warn "out: $line"; return $line; } @@ -73,11 +67,28 @@ { my ($line) = @_; - print "set_encoding_utf8\n"; - print "in: $line"; + warn "set_encoding_utf8\n"; + warn "in: $line"; $line =~ s,encoding=".*",encoding="utf-8",; - print "out: $line"; + warn "out: $line"; return $line; -} \ No newline at end of file +} + +sub unterminated_archive_tag +{ + my $line = shift; + my $nextLine = shift; + my $lineNum = shift; + + if ($nextLine !~ m{()|()}) + { + warn "unterminated_archive_tag\n"; + warn "in: $line"; + $line =~ s{>}{/>}; + warn "out: $line"; + } + + return $line . $nextLine; +} diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/truclean.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/truclean.pl Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,110 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Extracts output text in context which doesn't belong to 's + +use strict; +use Getopt::Long; + +my $RELEASEABLES_DIR = "/releaseables"; + +my $releaseablesdir = ""; +my $packageexpr = ''; +my $help = 0; +GetOptions(( + 'packageexpr:s' => \$packageexpr, + 'releaseablesdir:s' => \$RELEASEABLES_DIR, + 'help!' => \$help +)); + +$packageexpr =~ m,([^/^\\]+)[/\\]([^/^\\]+),; +my $layer_expr = $1; +my $package_expr = $2; +$help = 1 if (!$layer_expr or !$package_expr); + +if ($help) +{ + print "Extracts text which doesn't belong to recipes from a raptor log file\n"; + print "Usage: perl truclean.pl --packageexpr=LAYER_EXPR/PACKAGE_EXPR [OPTIONS]\n"; + print "where:\n"; + print "\tLAYER_EXPR can be * or the name of a layer\n"; + print "\tPACKAGE_EXPR can be * or the name of a package\n"; + print "and OPTIONS are:\n"; + print "\t--releaseablesdir=DIR Use DIR as the root of the releaseables dir (default: $RELEASEABLES_DIR\n"; + exit(0); +} + +$RELEASEABLES_DIR = $releaseablesdir if ($releaseablesdir); + +my @layers = (); +if ($layer_expr eq '*') +{ + opendir(DIR, $RELEASEABLES_DIR); + @layers = readdir(DIR); + closedir(DIR); + @layers = grep(!/^\.\.?$/, @layers); +} +else +{ + push(@layers, $layer_expr); +} +#for (@layers) {print "$_\n"}; + +for my $layer (@layers) +{ + my @packages = (); + if ($package_expr eq '*') + { + opendir(DIR, "$RELEASEABLES_DIR/$layer"); + @packages = readdir(DIR); + closedir(DIR); + @packages = grep(!/^\.\.?$/, @packages); + } + else + { + push(@packages, $package_expr); + } + #for (@pacakges) {print "$_\n"}; + + for my $package (@packages) + { + print "Processing package $layer/$package...\n"; + + open(FILE, "$RELEASEABLES_DIR/$layer/$package/info.tsv"); + while () + { + my $line = $_; + + if ($line =~ m,([^\t]*)\t([^\t]*)\t([^\t]*),) + { + my $file = $1; + my $type = $2; + my $config = $3; + + if (-f $file) + { + print "removing file: '$file'\n"; + unlink($file); + } + else + { + print "WARNING: file '$file' doesn't exist.\n"; + } + } + else + { + print "WARNING: line '$line' doesn't match the expected tab-separated pattern\n"; + } + } + close(FILE); + } +} \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/raptor/unreciped_text.pl --- a/common/tools/raptor/unreciped_text.pl Wed Jul 22 14:52:45 2009 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,103 +0,0 @@ -# Copyright (c) 2009 Symbian Foundation Ltd -# This component and the accompanying materials are made available -# under the terms of the License "Eclipse Public License v1.0" -# which accompanies this distribution, and is available -# at the URL "http://www.eclipse.org/legal/epl-v10.html". -# -# Initial Contributors: -# Symbian Foundation Ltd - initial contribution. -# -# Contributors: -# -# Description: -# Extracts output text in context which doesn't belong to 's - -use strict; - -use XML::SAX; -use RaptorSAXHandler; -use Getopt::Long; - -my @logfiles; -my $basedir = ''; -my $help = 0; -GetOptions(( - 'log:s' => \@logfiles, - 'basedir:s' => \$basedir, - 'help!' => \$help -)); - -$help = 1 if (!@logfiles); - -if ($help) -{ - print "Extracts text which doesn't belong to recipes from a raptor log file\n"; - print "Usage: perl unreciped_text.pl --log=FILE1 --log=FILE2 [OPTIONS]\n"; - print "where OPTIONS are:\n"; - print "\t--basedir=DIR Generate output file under DIR\n"; - exit(0); -} - -my $reset_status = {}; -my $buildlog_status = {}; -my $buildlog_subtag_status = {}; - -$reset_status->{name} = 'reset_status'; -$reset_status->{next_status} = {buildlog=>$buildlog_status}; - -$buildlog_status->{name} = 'buildlog_status'; -$buildlog_status->{next_status} = {'?default?'=>$buildlog_subtag_status}; -$buildlog_status->{on_start} = 'main::on_start_buildlog'; -$buildlog_status->{on_end} = 'main::on_end_buildlog'; -$buildlog_status->{on_chars} = 'main::on_chars_buildlog'; - -$buildlog_subtag_status->{name} = 'buildlog_subtag_status'; -$buildlog_subtag_status->{next_status} = {}; - -my $characters = ''; - -if (!$basedir) -{ - $basedir = time; - - print "Using $basedir as basedir.\n"; -} -if (-d $basedir) -{ - print "Directory $basedir exists. Quitting.\n"; - exit(1); -} -mkdir($basedir); -#print "Created dir $basedir.\n"; - -my $saxhandler = RaptorSAXHandler->new(); -$saxhandler->set_init_status($reset_status); -my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); -for (@logfiles) -{ - $parser->parse_uri($_); -} - - -sub on_start_buildlog -{ - my $filename = "$basedir/unreciped.txt"; - print "Writing unreciped file $filename\n" if (!-f$filename); - open(FILE, ">>$filename"); -} - -sub on_chars_buildlog -{ - my ($ch) = @_; - - my $characters = $ch->{Data}; - - print FILE $characters if ($characters =~ m,[^\s^\r^\n],); - - #print "characters is now -->$characters<--\n"; -} - -sub on_end_buildlog -{ - close(FILE); -} \ No newline at end of file diff -r cdf0b7c501d2 -r 48a6e55f691b common/tools/sortUnique.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/sortUnique.pl Wed Jul 22 14:55:30 2009 +0100 @@ -0,0 +1,13 @@ +#!perl -w +use strict; +# Sorts the input, removes duplicates, and outputs it + +# Read it +my @content = <>; + +# Sort it, and grep to remove duplicates +my $previous = "\n\n"; +@content = grep {$_ ne $previous && ($previous = $_, 1) } sort @content; + +# Write it +print @content; diff -r cdf0b7c501d2 -r 48a6e55f691b sf-package/package_props.ant.xml --- a/sf-package/package_props.ant.xml Wed Jul 22 14:52:45 2009 +0100 +++ b/sf-package/package_props.ant.xml Wed Jul 22 14:55:30 2009 +0100 @@ -5,7 +5,11 @@ - + + + + + diff -r cdf0b7c501d2 -r 48a6e55f691b sf-platform/build.xml --- a/sf-platform/build.xml Wed Jul 22 14:52:45 2009 +0100 +++ b/sf-platform/build.xml Wed Jul 22 14:55:30 2009 +0100 @@ -86,26 +86,6 @@ - - - - - - - - - - - - - - - - - - - - + +