# HG changeset patch # User Dario Sestito # Date 1258134470 0 # Node ID d00048f1b036c8e4917b9ed6df113142c8948b98 # Parent 5fdd5e70280dc1518cd3e0293a7b636a86b23a5e Improved Raptor parser, now "uh" diff -r 5fdd5e70280d -r d00048f1b036 common/build.postbuild.xml --- a/common/build.postbuild.xml Fri Nov 13 14:15:28 2009 +0000 +++ b/common/build.postbuild.xml Fri Nov 13 17:47:50 2009 +0000 @@ -323,8 +323,7 @@ - - + @@ -344,55 +343,23 @@ - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + @@ -527,7 +494,7 @@ ant: antProperties() - raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,config,component,phase,recipe,file,line]}) + raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,config,component,mmp,phase,recipe,file,line]}) diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorCommon.pm --- a/common/tools/raptor/RaptorCommon.pm Fri Nov 13 14:15:28 2009 +0000 +++ b/common/tools/raptor/RaptorCommon.pm Fri Nov 13 17:47:50 2009 +0000 @@ -20,7 +20,7 @@ sub init { - my $filename = "$::basedir/summary.csv"; + my $filename = "$::raptorbitsdir/summary.csv"; if (!-f$filename) { print "Writing summary file $filename\n"; @@ -31,10 +31,10 @@ sub dump_fault { - my ($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line) = @_; + my ($category, $subcategory, $severity, $location, $component, $mmp, $phase, $recipe, $file, $line) = @_; - open(SUMMARY, ">>$::basedir/summary.csv"); - print SUMMARY "$category,$subcategory,$severity,$::raptor_config,$component,$phase,$recipe,$file,$line\n"; + open(SUMMARY, ">>$::raptorbitsdir/summary.csv"); + print SUMMARY "$category,$subcategory,$severity,$location,$component,$mmp,$phase,$recipe,$file,$line\n"; close(SUMMARY); } diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorError.pm --- a/common/tools/raptor/RaptorError.pm Fri Nov 13 14:15:28 2009 +0000 +++ b/common/tools/raptor/RaptorError.pm Fri Nov 13 17:47:50 2009 +0000 @@ -51,7 +51,7 @@ sub process { - my ($text, $component, $phase, $recipe, $file, $line) = @_; + my ($text, $logfile, $component, $mmp, $phase, $recipe, $file, $line) = @_; my $category = $CATEGORY_RAPTORERROR; my $severity = ''; @@ -61,47 +61,47 @@ { $severity = $RaptorCommon::SEVERITY_CRITICAL; $subcategory = $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,No bld\.inf found at,) { $severity = $RaptorCommon::SEVERITY_MAJOR; $subcategory = $CATEGORY_RAPTORERROR_NOBLDINFFOUND; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,Can't find mmp file,) { $severity = $RaptorCommon::SEVERITY_MINOR; $subcategory = $CATEGORY_RAPTORERROR_CANTFINDMMPFILE; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,The make-engine exited with errors,) { $severity = $RaptorCommon::SEVERITY_CRITICAL; $subcategory = $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,tool .* from config .* did not return version .* as required,) { $severity = $RaptorCommon::SEVERITY_CRITICAL; $subcategory = $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,Unknown build configuration '.*',) { $severity = $RaptorCommon::SEVERITY_CRITICAL; $subcategory = $CATEGORY_RAPTORERROR_UNKNOWNBUILDCONFIG; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,No build configurations given,) { $severity = $RaptorCommon::SEVERITY_CRITICAL; $subcategory = $CATEGORY_RAPTORERROR_NOBUILDCONFIGSGIVEN; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } else # log everything by default { - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } } @@ -109,7 +109,7 @@ { RaptorCommon::init(); - $filename = "$::basedir/raptor_error.txt"; + $filename = "$::raptorbitsdir/raptor_error.txt"; if (!-f$filename) { print "Writing errors file $filename\n"; @@ -160,7 +160,7 @@ print FILE "$characters\n\n"; close(FILE); - process($characters, '', '', '', "raptor_error.txt", $failure_item); + process($characters, $::current_log_file, '', '', '', '', "raptor_error.txt", $failure_item); } $characters = ''; diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorInfo.pm --- a/common/tools/raptor/RaptorInfo.pm Fri Nov 13 14:15:28 2009 +0000 +++ b/common/tools/raptor/RaptorInfo.pm Fri Nov 13 17:47:50 2009 +0000 @@ -55,7 +55,7 @@ sub on_start_buildlog_info { - my $filename = "$::basedir/info.txt"; + my $filename = "$::raptorbitsdir/info.txt"; print "Writing info file $filename\n" if (!-f$filename); open(FILE, ">>$filename"); } diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorRecipe.pm --- a/common/tools/raptor/RaptorRecipe.pm Fri Nov 13 14:15:28 2009 +0000 +++ b/common/tools/raptor/RaptorRecipe.pm Fri Nov 13 17:47:50 2009 +0000 @@ -59,67 +59,79 @@ my $CATEGORY_RECIPEFAILURE_ARMCC_GENERICWARNINGSERRORS = 'armcc_generic_warnings_errors'; my $CATEGORY_RECIPEFAILURE_ELF2E32_SYMBOLMISSINGFROMELFFILE = 'elf2e32_symbol_missing_from_elf_file'; +my $mmp_with_issues = {}; + sub process { - my ($text, $component, $phase, $recipe, $file, $line) = @_; + my ($text, $config, $component, $mmp, $phase, $recipe, $file, $line) = @_; my $category = $CATEGORY_RECIPEFAILURE; my $severity = ''; my $subcategory = ''; - if ($text =~ m,Error: #5: cannot open source input file .*: No such file or directory,) + # if mmp is defined assign severity=MAJOR for the first failure + # then severity=MINOR to all other (for each logfile) + if ($mmp and defined $mmp_with_issues->{$::current_log_file}->{$mmp}) + { + $severity = $RaptorCommon::SEVERITY_MINOR; + } + elsif ($mmp) + { + $mmp_with_issues->{$::current_log_file} = {} if (!defined $mmp_with_issues->{$::current_log_file}); + $mmp_with_issues->{$::current_log_file}->{$mmp} = 1; + $severity = $RaptorCommon::SEVERITY_MAJOR; + } + else { $severity = $RaptorCommon::SEVERITY_MAJOR; + } + + + if ($text =~ m,Error: #5: cannot open source input file .*: No such file or directory,) + { my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,Fatal error: L6002U: Could not open file .*: No such file or directory,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,elf2e32 : Error: E1001: Could not open file : .*.,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,elf2e32 : Error: E1036: Symbol .* Missing from ELF File,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_SYMBOLMISSINGFROMELFFILE; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,Error: L6833E: File '.*' does not exist,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ARMAR_FILEDOESNOTEXIST; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,: Warning: #236-D: controlling expression is constant,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CONTROLLINGEXPRESSIONISCONSTANT; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,/armcc.exe , and $text =~ m,Internal fault: ,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_INTERNALFAULT; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } # the following captures generic armcc error/warnings, not captured by regexps above elsif ($text =~ m,/armcc.exe , and $text =~ m,: \d+ warnings\, \d+ errors$,) { - $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_GENERICWARNINGSERRORS; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } else # log everything by default { - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $config, $component, $mmp, $phase, $recipe, $file, $line); } } @@ -199,12 +211,20 @@ print "WARNING: can't understand bldinf attribute of recipe: $recipe_info->{bldinf}. Won't dump to failed recipes file.\n"; } + # also normalize mmp path if this exists + if ($recipe_info->{mmp}) + { + $recipe_info->{mmp} = lc($recipe_info->{mmp}); + $recipe_info->{mmp} =~ s,^[A-Za-z]:,,; + $recipe_info->{mmp} =~ s,[\\],/,g; + } + $characters =~ s,^[\r\n]*,,; $characters =~ s,[\r\n]*$,,; if ($package) { - $filename = "$::basedir/$package.txt"; + $filename = "$::raptorbitsdir/$package.txt"; if (!-f$filename) { print "Writing recipe file $filename\n"; @@ -231,7 +251,7 @@ close(FILE); } - process($characters, $recipe_info->{bldinf}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt", $failure_item); + process($characters, $recipe_info->{config}, $recipe_info->{bldinf}, $recipe_info->{mmp}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt", $failure_item); } $characters = ''; diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorReleaseable.pm --- a/common/tools/raptor/RaptorReleaseable.pm Fri Nov 13 14:15:28 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,293 +0,0 @@ -# Copyright (c) 2009 Symbian Foundation Ltd -# This component and the accompanying materials are made available -# under the terms of the License "Eclipse Public License v1.0" -# which accompanies this distribution, and is available -# at the URL "http://www.eclipse.org/legal/epl-v10.html". -# -# Initial Contributors: -# Symbian Foundation Ltd - initial contribution. -# -# Contributors: -# -# Description: -# Raptor parser module. -# Extract releaseable (whatlog) information - -package RaptorReleaseable; - -use strict; -use RaptorCommon; - -our $reset_status = {}; -my $buildlog_status = {}; -my $whatlog_status = {}; -my $bitmap_status = {}; -my $resource_status = {}; -my $build_status = {}; -my $export_status = {}; -my $stringtable_status = {}; -my $archive_status = {}; -my $archive_member_status = {}; -my $whatlog_default_status = {}; - -$reset_status->{name} = 'reset_status'; -$reset_status->{next_status} = {buildlog=>$buildlog_status}; - -$buildlog_status->{name} = 'buildlog_status'; -$buildlog_status->{next_status} = {whatlog=>$whatlog_status}; -$buildlog_status->{on_start} = 'RaptorReleaseable::on_start_buildlog'; - -$whatlog_status->{name} = 'whatlog_status'; -$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status}; -$whatlog_status->{on_start} = 'RaptorReleaseable::on_start_whatlog'; -$whatlog_status->{on_end} = 'RaptorReleaseable::on_end_whatlog'; - -$bitmap_status->{name} = 'bitmap_status'; -$bitmap_status->{next_status} = {}; -$bitmap_status->{on_start} = 'RaptorReleaseable::on_start_bitmap'; -$bitmap_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; -$bitmap_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; - -$resource_status->{name} = 'resource_status'; -$resource_status->{next_status} = {}; -$resource_status->{on_start} = 'RaptorReleaseable::on_start_resource'; -$resource_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; -$resource_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; - -$build_status->{name} = 'build_status'; -$build_status->{next_status} = {}; -$build_status->{on_start} = 'RaptorReleaseable::on_start_build'; -$build_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; -$build_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; - -$stringtable_status->{name} = 'stringtable_status'; -$stringtable_status->{next_status} = {}; -$stringtable_status->{on_start} = 'RaptorReleaseable::on_start_stringtable'; -$stringtable_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; -$stringtable_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; - -$archive_status->{name} = 'archive_status'; -$archive_status->{next_status} = {member=>$archive_member_status}; - -$archive_member_status->{name} = 'archive_member_status'; -$archive_member_status->{next_status} = {}; -$archive_member_status->{on_start} = 'RaptorReleaseable::on_start_archive_member'; -$archive_member_status->{on_end} = 'RaptorReleaseable::on_end_whatlog_subtag'; -$archive_member_status->{on_chars} = 'RaptorReleaseable::on_chars_whatlog_subtag'; - -$export_status->{name} = 'export_status'; -$export_status->{next_status} = {}; -$export_status->{on_start} = 'RaptorReleaseable::on_start_export'; - -$whatlog_default_status->{name} = 'whatlog_default_status'; -$whatlog_default_status->{next_status} = {}; -$whatlog_default_status->{on_start} = 'RaptorReleaseable::on_start_whatlog_default'; - -my $whatlog_info = {}; -my $curbldinf = 'unknown'; -my $curconfig = 'unknown'; -my $curfiletype = 'unknown'; -my $characters = ''; - -sub on_start_buildlog -{ - mkdir("$::basedir/releaseables"); -} - -sub on_start_whatlog -{ - my ($el) = @_; - - $whatlog_info = {}; - - my $bldinf = ''; - my $config = ''; - my $attributes = $el->{Attributes}; - for (keys %{$attributes}) - { - #print "reading attribute $_\n"; - if ($attributes->{$_}->{'LocalName'} eq 'bldinf') - { - $bldinf = $attributes->{$_}->{'Value'}; - #print "bldinf=$bldinf\n"; - } - elsif ($attributes->{$_}->{'LocalName'} eq 'config') - { - $config = $attributes->{$_}->{'Value'}; - $config =~ s,\.whatlog$,,; - } - } - - if ($bldinf eq '') - { - print "WARNING: whatlog tag with no bldinf attribute. Skipping\n"; - return; - } - - $curbldinf = $bldinf; - $curconfig = $config; - $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf}); - $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig}); -} - -sub on_start_whatlog_subtag -{ - my ($ft) = @_; - - $curfiletype = $ft; - $characters = ''; - $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}); -} - -sub on_chars_whatlog_subtag -{ - my ($ch) = @_; - - $characters .= $ch->{Data}; - - #print "characters is now -->$characters<--\n"; -} - -sub on_end_whatlog_subtag -{ - $characters = normalize_filepath($characters); - - push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters); - - $curfiletype = 'unknown'; - $characters = ''; -} - -sub on_start_bitmap -{ - on_start_whatlog_subtag('bitmap'); -} - -sub on_start_resource -{ - on_start_whatlog_subtag('resource'); -} - -sub on_start_build -{ - on_start_whatlog_subtag('build'); -} - -sub on_start_stringtable -{ - on_start_whatlog_subtag('stringtable'); -} - -sub on_start_archive_member -{ - on_start_whatlog_subtag('export'); -} - -sub on_start_export -{ - my ($el) = @_; - - $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export}); - - my $destination = ''; - my $attributes = $el->{Attributes}; - for (keys %{$attributes}) - { - #print "reading attribute $_\n"; - if ($attributes->{$_}->{'LocalName'} eq 'destination') - { - $destination = $attributes->{$_}->{'Value'}; - #print "destination=$destination\n"; - last; - } - } - - if ($destination eq '') - { - print "WARNING: export tag with no destination attribute. Skipping\n"; - return; - } - - $destination = normalize_filepath($destination); - - push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination); -} - -sub on_end_whatlog -{ - my $unknown_counter = 0; - - for my $bldinf (keys %{$whatlog_info}) - { - for my $config (keys %{$whatlog_info->{$bldinf}}) - { - my $normalized = lc($bldinf); - $normalized =~ s,^[A-Za-z]:,,; - $normalized =~ s,[\\],/,g; - - $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; - my $layer = $1; - my $package = $2; - - mkdir("$::basedir/releaseables/$layer"); - mkdir("$::basedir/releaseables/$layer/$package"); - - my $filename = "$::basedir/releaseables/$layer/$package/info.tsv"; - - print "Writing info file $filename\n" if (!-f$filename); - open(FILE, ">>$filename"); - - for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}}) - { - for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}})) - { - print FILE "$_\t$filetype\t$config\n"; - } - } - - close(FILE); - } - } -} - -sub normalize_filepath -{ - my ($filepath) = @_; - - if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,) - { - print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n"; - } - - # strip all CR's and NL's - $filepath =~ s,[\r\n],,g; - - # strip all whitespaces at string start/end - $filepath =~ s,^\s+,,g; - $filepath =~ s,\s+$,,g; - - # remove drive letter and colon from the beginning of the string - $filepath =~ s,^[A-Za-z]:,,; - - # normalize slashes - $filepath =~ s,\\,/,g; - $filepath =~ s,//,/,g; - - if ($filepath !~ m,^/epoc32/,i) - { - print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n"; - } - - return $filepath; -} - -sub on_start_whatlog_default -{ - my ($el) = @_; - - my $tagname = $el->{LocalName}; - - print "WARNING: unsupported tag '$tagname' in context\n"; -} - -1; \ No newline at end of file diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorUnreciped.pm --- a/common/tools/raptor/RaptorUnreciped.pm Fri Nov 13 14:15:28 2009 +0000 +++ b/common/tools/raptor/RaptorUnreciped.pm Fri Nov 13 17:47:50 2009 +0000 @@ -51,7 +51,7 @@ sub process { - my ($text, $component, $phase, $recipe, $file, $line) = @_; + my ($text, $logfile, $component, $mmp, $phase, $recipe, $file, $line) = @_; my $category = $CATEGORY_RAPTORUNRECIPED; my $severity = ''; @@ -61,31 +61,31 @@ { $severity = $RaptorCommon::SEVERITY_MAJOR; my $subcategory = $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,make\.exe: Target .* not remade because of errors,) { $severity = $RaptorCommon::SEVERITY_MINOR; my $subcategory = $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,: warning: ignoring old commands for target,) { $severity = $RaptorCommon::SEVERITY_MINOR; my $subcategory = $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,: warning: overriding commands for target,) { $severity = $RaptorCommon::SEVERITY_MINOR; my $subcategory = $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,make\.exe: Nothing to be done for .*,) { $severity = $RaptorCommon::SEVERITY_MINOR; my $subcategory = $CATEGORY_RAPTORUNRECIPED_NOTHINGTOBEDONEFOR; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } elsif ($text =~ m,^(true|false)$,) { @@ -93,7 +93,7 @@ } else # log everything by default { - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } } @@ -101,7 +101,7 @@ { RaptorCommon::init(); - $filename = "$::basedir/raptor_unreciped.txt"; + $filename = "$::raptorbitsdir/raptor_unreciped.txt"; if (!-f$filename) { print "Writing unreciped file $filename\n"; @@ -163,7 +163,7 @@ print FILE "$line\n\n"; close(FILE); - process($line, '', '', '', "raptor_unreciped.txt", $failure_item); + process($line, $::current_log_file, '', '', '', '', "raptor_unreciped.txt", $failure_item); } } diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/RaptorWarning.pm --- a/common/tools/raptor/RaptorWarning.pm Fri Nov 13 14:15:28 2009 +0000 +++ b/common/tools/raptor/RaptorWarning.pm Fri Nov 13 17:47:50 2009 +0000 @@ -45,7 +45,7 @@ sub process { - my ($text, $component, $phase, $recipe, $file, $line) = @_; + my ($text, $logfile, $component, $mmp, $phase, $recipe, $file, $line) = @_; my $category = $CATEGORY_RAPTORWARNING; my $severity = ''; @@ -55,11 +55,11 @@ { $severity = $RaptorCommon::SEVERITY_MINOR; my $subcategory = $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2; - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } else # log everything by default { - RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + RaptorCommon::dump_fault($category, $subcategory, $severity, $logfile, $component, $mmp, $phase, $recipe, $file, $line); } } @@ -67,7 +67,7 @@ { RaptorCommon::init(); - $filename = "$::basedir/raptor_warning.txt"; + $filename = "$::raptorbitsdir/raptor_warning.txt"; if (!-f$filename) { print "Writing warnings file $filename\n"; @@ -118,7 +118,7 @@ print FILE "$characters\n\n"; close(FILE); - process($characters, '', '', '', "raptor_warning.txt", $failure_item); + process($characters, $::current_log_file, '', '', '', '', "raptor_warning.txt", $failure_item); } $characters = ''; diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/parse.pl --- a/common/tools/raptor/parse.pl Fri Nov 13 14:15:28 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,134 +0,0 @@ -# Copyright (c) 2009 Symbian Foundation Ltd -# This component and the accompanying materials are made available -# under the terms of the License "Eclipse Public License v1.0" -# which accompanies this distribution, and is available -# at the URL "http://www.eclipse.org/legal/epl-v10.html". -# -# Initial Contributors: -# Symbian Foundation Ltd - initial contribution. -# -# Contributors: -# -# Description: -# Run the raptor parsers - -use strict; -use FindBin; -use lib $FindBin::Bin; -use RaptorReleaseable; -use RaptorError; -use RaptorWarning; -use RaptorInfo; -use RaptorUnreciped; -use RaptorRecipe; - -use XML::SAX; -use RaptorSAXHandler; -use Getopt::Long; - -my @logfiles; -my $releaseable_module = 0; -my $error_module = 0; -my $warning_module = 0; -my $info_module = 0; -my $unreciped_module = 0; -my $recipe_module = 0; -our $basedir = ''; -our $raptor_config = ''; -my $append = 0; -my $help = 0; -GetOptions(( - 'log:s' => \@logfiles, - 'releaseable!' => \$releaseable_module, - 'error!' => \$error_module, - 'warning!' => \$warning_module, - 'info!' => \$info_module, - 'unreciped!' => \$unreciped_module, - 'recipe!' => \$recipe_module, - 'basedir=s' => \$basedir, - 'config=s' => \$raptor_config, - 'append!' => \$append, - 'help!' => \$help -)); - -$help = 1 if (!@logfiles); - -if ($help) -{ - print "Run the raptor parsers\n"; - print "Usage: perl parse.pl [MODULES] --log=FILE1 --log=FILE2 ... [OPTIONS]\n"; - print "where MODULES are:\n"; - print "\t--releaseable Extract releaseable (whatlog) information\n"; - print "\t--error Extracts raptor errors, i.e. content of tags\n"; - print "\t--warning Extracts raptor warnings, i.e. content of tags\n"; - print "\t--info Extracts raptor info text i.e. content of tags from a raptor log file\n"; - print "\t--unreciped Extracts output text in context which doesn't belong to any tags\n"; - print "\t--recipe Extract, analyzes and dumps raptor recipes i.e. content of tags from a raptor log file\n"; - print "where OPTIONS are:\n"; - print "\t--basedir=DIR Generate output file under DIR\n"; - print "\t--append Do not stop if basedir exists but append newly extracted info to already existing.\n"; - exit(0); -} - -if (!$basedir) -{ - $basedir = time; - - print "Using $basedir as basedir.\n"; -} -if (-d $basedir) -{ - if ($append) - { - print "Directory $basedir exists. Appending new info to it.\n"; - } - else - { - print "Directory $basedir exists. Quitting.\n"; - exit(1); - } -} -mkdir($basedir); -#print "Created dir $basedir.\n"; - -# create empty summary file anyway -open(SUMMARY, ">$basedir/summary.csv"); -close(SUMMARY); - -my $saxhandler = RaptorSAXHandler->new(); -if ($releaseable_module) -{ - print "Adding RaptorReleaseable module to the observers\n"; - $saxhandler->add_observer('RaptorReleaseable', $RaptorReleaseable::reset_status); -} -if ($error_module) -{ - print "Adding RaptorError module to the observers\n"; - $saxhandler->add_observer('RaptorError', $RaptorError::reset_status); -} -if ($warning_module) -{ - print "Adding RaptorWarning module to the observers\n"; - $saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status); -} -if ($info_module) -{ - print "Adding RaptorInfo module to the observers\n"; - $saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status); -} -if ($unreciped_module) -{ - print "Adding RaptorUnreciped module to the observers\n"; - $saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status); -} -if ($recipe_module) -{ - print "Adding RaptorRecipe module to the observers\n"; - $saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status); -} -my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); -for (@logfiles) -{ - $parser->parse_uri($_); -} - diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/releaseables.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/releaseables.pl Fri Nov 13 17:47:50 2009 +0000 @@ -0,0 +1,50 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Extract releaseable (whatlog) information from Raptor log files + +use strict; +use releaseables; +use FindBin; +use lib $FindBin::Bin; +use XML::SAX; +use RaptorSAXHandler; +use Getopt::Long; + +our $basedir = '.'; +my $help = 0; +GetOptions(( + 'basedir=s' => \$basedir, + 'help!' => \$help +)); +my @logfiles = @ARGV; + +$help = 1 if (!@logfiles); + +if ($help) +{ + print "Extract releaseable (whatlog) information from Raptor log files\n"; + print "Usage: perl releaseables.pl [OPTIONS] FILE1 FILE2 ...\n"; + print "where OPTIONS are:\n"; + print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n"; + exit(0); +} + +my $saxhandler = RaptorSAXHandler->new(); +$saxhandler->add_observer('releaseables', $releaseables::reset_status); + +my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); +for (@logfiles) +{ + $parser->parse_uri($_); +} + diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/releaseables.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/releaseables.pm Fri Nov 13 17:47:50 2009 +0000 @@ -0,0 +1,293 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Raptor parser module. +# Extract releaseable (whatlog) information + +package releaseables; + +use strict; + +our $reset_status = {}; +my $buildlog_status = {}; +my $whatlog_status = {}; +my $bitmap_status = {}; +my $resource_status = {}; +my $build_status = {}; +my $export_status = {}; +my $stringtable_status = {}; +my $archive_status = {}; +my $archive_member_status = {}; +my $whatlog_default_status = {}; + +$reset_status->{name} = 'reset_status'; +$reset_status->{next_status} = {buildlog=>$buildlog_status}; + +$buildlog_status->{name} = 'buildlog_status'; +$buildlog_status->{next_status} = {whatlog=>$whatlog_status}; +$buildlog_status->{on_start} = 'releaseables::on_start_buildlog'; + +$whatlog_status->{name} = 'whatlog_status'; +$whatlog_status->{next_status} = {bitmap=>$bitmap_status, resource=>$resource_status, build=>$build_status, export=>$export_status, stringtable=>$stringtable_status, archive=>$archive_status, '?default?'=>$whatlog_default_status}; +$whatlog_status->{on_start} = 'releaseables::on_start_whatlog'; +$whatlog_status->{on_end} = 'releaseables::on_end_whatlog'; + +$bitmap_status->{name} = 'bitmap_status'; +$bitmap_status->{next_status} = {}; +$bitmap_status->{on_start} = 'releaseables::on_start_bitmap'; +$bitmap_status->{on_end} = 'releaseables::on_end_whatlog_subtag'; +$bitmap_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag'; + +$resource_status->{name} = 'resource_status'; +$resource_status->{next_status} = {}; +$resource_status->{on_start} = 'releaseables::on_start_resource'; +$resource_status->{on_end} = 'releaseables::on_end_whatlog_subtag'; +$resource_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag'; + +$build_status->{name} = 'build_status'; +$build_status->{next_status} = {}; +$build_status->{on_start} = 'releaseables::on_start_build'; +$build_status->{on_end} = 'releaseables::on_end_whatlog_subtag'; +$build_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag'; + +$stringtable_status->{name} = 'stringtable_status'; +$stringtable_status->{next_status} = {}; +$stringtable_status->{on_start} = 'releaseables::on_start_stringtable'; +$stringtable_status->{on_end} = 'releaseables::on_end_whatlog_subtag'; +$stringtable_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag'; + +$archive_status->{name} = 'archive_status'; +$archive_status->{next_status} = {member=>$archive_member_status}; + +$archive_member_status->{name} = 'archive_member_status'; +$archive_member_status->{next_status} = {}; +$archive_member_status->{on_start} = 'releaseables::on_start_archive_member'; +$archive_member_status->{on_end} = 'releaseables::on_end_whatlog_subtag'; +$archive_member_status->{on_chars} = 'releaseables::on_chars_whatlog_subtag'; + +$export_status->{name} = 'export_status'; +$export_status->{next_status} = {}; +$export_status->{on_start} = 'releaseables::on_start_export'; + +$whatlog_default_status->{name} = 'whatlog_default_status'; +$whatlog_default_status->{next_status} = {}; +$whatlog_default_status->{on_start} = 'releaseables::on_start_whatlog_default'; + +my $whatlog_info = {}; +my $curbldinf = 'unknown'; +my $curconfig = 'unknown'; +my $curfiletype = 'unknown'; +my $characters = ''; + +sub on_start_buildlog +{ + system("rmdir /S /Q $::raptorbitsdir/releaseables") if (-d "$::basedir/releaseables"); + mkdir("$::raptorbitsdir/releaseables"); +} + +sub on_start_whatlog +{ + my ($el) = @_; + + $whatlog_info = {}; + + my $bldinf = ''; + my $config = ''; + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + #print "reading attribute $_\n"; + if ($attributes->{$_}->{'LocalName'} eq 'bldinf') + { + $bldinf = $attributes->{$_}->{'Value'}; + #print "bldinf=$bldinf\n"; + } + elsif ($attributes->{$_}->{'LocalName'} eq 'config') + { + $config = $attributes->{$_}->{'Value'}; + $config =~ s,\.whatlog$,,; + } + } + + if ($bldinf eq '') + { + print "WARNING: whatlog tag with no bldinf attribute. Skipping\n"; + return; + } + + $curbldinf = $bldinf; + $curconfig = $config; + $whatlog_info->{$curbldinf} = {} if (!defined $whatlog_info->{$curbldinf}); + $whatlog_info->{$curbldinf}->{$curconfig} = {} if (!defined $whatlog_info->{$curbldinf}->{$curconfig}); +} + +sub on_start_whatlog_subtag +{ + my ($ft) = @_; + + $curfiletype = $ft; + $characters = ''; + $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}); +} + +sub on_chars_whatlog_subtag +{ + my ($ch) = @_; + + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; +} + +sub on_end_whatlog_subtag +{ + $characters = normalize_filepath($characters); + + push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{$curfiletype}}, $characters); + + $curfiletype = 'unknown'; + $characters = ''; +} + +sub on_start_bitmap +{ + on_start_whatlog_subtag('bitmap'); +} + +sub on_start_resource +{ + on_start_whatlog_subtag('resource'); +} + +sub on_start_build +{ + on_start_whatlog_subtag('build'); +} + +sub on_start_stringtable +{ + on_start_whatlog_subtag('stringtable'); +} + +sub on_start_archive_member +{ + on_start_whatlog_subtag('export'); +} + +sub on_start_export +{ + my ($el) = @_; + + $whatlog_info->{$curbldinf}->{$curconfig}->{export} = [] if (! defined $whatlog_info->{$curbldinf}->{$curconfig}->{export}); + + my $destination = ''; + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + #print "reading attribute $_\n"; + if ($attributes->{$_}->{'LocalName'} eq 'destination') + { + $destination = $attributes->{$_}->{'Value'}; + #print "destination=$destination\n"; + last; + } + } + + if ($destination eq '') + { + print "WARNING: export tag with no destination attribute. Skipping\n"; + return; + } + + $destination = normalize_filepath($destination); + + push(@{$whatlog_info->{$curbldinf}->{$curconfig}->{export}}, $destination); +} + +sub on_end_whatlog +{ + my $unknown_counter = 0; + + for my $bldinf (keys %{$whatlog_info}) + { + for my $config (keys %{$whatlog_info->{$bldinf}}) + { + my $normalized = lc($bldinf); + $normalized =~ s,^[A-Za-z]:,,; + $normalized =~ s,[\\],/,g; + + $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; + my $layer = $1; + my $package = $2; + + mkdir("$::raptorbitsdir/releaseables/$layer"); + mkdir("$::raptorbitsdir/releaseables/$layer/$package"); + + my $filename = "$::raptorbitsdir/releaseables/$layer/$package/info.tsv"; + + print "Writing info file $filename\n" if (!-f$filename); + open(FILE, ">>$filename"); + + for my $filetype (keys %{$whatlog_info->{$bldinf}->{$config}}) + { + for (sort(@{$whatlog_info->{$bldinf}->{$config}->{$filetype}})) + { + print FILE "$_\t$filetype\t$config\n"; + } + } + + close(FILE); + } + } +} + +sub normalize_filepath +{ + my ($filepath) = @_; + + if ($filepath =~ m,[^\s^\r^\n]+(.*)[\r\n]+(.*)[^\s^\r^\n]+,) + { + print "WARNING: file path string extends over multiple line: $filepath. Removing all NL's and CR's\n"; + } + + # strip all CR's and NL's + $filepath =~ s,[\r\n],,g; + + # strip all whitespaces at string start/end + $filepath =~ s,^\s+,,g; + $filepath =~ s,\s+$,,g; + + # remove drive letter and colon from the beginning of the string + $filepath =~ s,^[A-Za-z]:,,; + + # normalize slashes + $filepath =~ s,\\,/,g; + $filepath =~ s,//,/,g; + + if ($filepath !~ m,^/epoc32/,i) + { + print "WARNING: file '$filepath' doesn't seem valid. Writing to info file anyway\n"; + } + + return $filepath; +} + +sub on_start_whatlog_default +{ + my ($el) = @_; + + my $tagname = $el->{LocalName}; + + print "WARNING: unsupported tag '$tagname' in context\n"; +} + +1; \ No newline at end of file diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/summarize.pl --- a/common/tools/raptor/summarize.pl Fri Nov 13 14:15:28 2009 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,311 +0,0 @@ -# Copyright (c) 2009 Symbian Foundation Ltd -# This component and the accompanying materials are made available -# under the terms of the License "Eclipse Public License v1.0" -# which accompanies this distribution, and is available -# at the URL "http://www.eclipse.org/legal/epl-v10.html". -# -# Initial Contributors: -# Symbian Foundation Ltd - initial contribution. -# -# Contributors: -# -# Description: -# Generate an HTML summary of the Raptor build from the output of the raptor parser - -use strict; -use FindBin; -use lib $FindBin::Bin; -use Getopt::Long; - -my $raptorbitsdir = 0; -my $outputdir = '.'; -my $help = 0; -GetOptions(( - 'raptorbitsdir=s' => \$raptorbitsdir, - 'outputdir=s' => \$outputdir, - 'help!' => \$help -)); - -$help = 1 if (!$raptorbitsdir); - -if ($help) -{ - print "Generate an HTML summary of the Raptor build from a summary.csv file\n"; - print "Usage: perl summarize.pl --raptorbitsdir=DIR [--outputdir=DIR]\n"; - exit(0); -} - -$outputdir = "$outputdir/html"; - -system("rd /S /Q $outputdir") if (-d $outputdir); -mkdir ($outputdir); - -my $raptor_errors = {}; -my $raptor_warnings = {}; -my $raptor_unreciped = {}; -my $general_failures_num_by_severity = {}; -my $general_failures_by_category_severity = {}; -my $recipe_failures_num_by_severity = {}; -my $recipe_failures_by_package_severity = {}; -#my $severities = {}; -my @severities = ('critical', 'major', 'minor', 'unknown'); - -# READ SUMMARY.CSV FILE -my $csv_file = "$raptorbitsdir/summary.csv"; -my $csv_linenum = 0; -open(CSV, $csv_file); -while() -{ - $csv_linenum ++; - my $line = $_; - - if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/) - { - my $failure = {}; - $failure->{category} = $1; - $failure->{subcategory} = $2; - $failure->{severity} = $3; - $failure->{config} = $4; - $failure->{component} = $5; - $failure->{phase} = $6; - $failure->{recipe} = $7; - $failure->{file} = $8; - $failure->{linenum} = $9; - - my $failure_package = ''; - - if (!$failure->{category}) - { - print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n"; - next; - } - - if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component}) - { - print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n"; - next; - } - if ($failure->{component}) - { - if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),) - { - $failure_package = $1; - } - else - { - print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n"; - next; - } - } - - $failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory}); - $failure->{severity} = 'unknown' if (!$failure->{severity}); - - # populate severities dynamically. - #$severities->{$failure->{severity}} = 1; - - # put failure items into their category container - if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i) - { - $general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}}); - my $general_failure = $general_failures_num_by_severity->{$failure->{category}}; - - if (!defined $general_failure->{$failure->{severity}}) - { - $general_failure->{$failure->{severity}} = 1; - } - else - { - $general_failure->{$failure->{severity}} ++; - } - - $general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}}); - $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}); - push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure); - } - elsif ($failure->{category} =~ /^recipe_failure$/i) - { - $recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package}); - my $package_failure = $recipe_failures_num_by_severity->{$failure_package}; - - if (!defined $package_failure->{$failure->{severity}}) - { - $package_failure->{$failure->{severity}} = 1; - } - else - { - $package_failure->{$failure->{severity}} ++; - } - - $recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package}); - $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}); - push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure); - } - } - else - { - print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n"; - } -} -close(CSV); - -# PRINT HTML SUMMARY -my $aggregated_html = "$outputdir/index.html"; -open(AGGREGATED, ">$aggregated_html"); -print AGGREGATED "RAPTOR BUILD SUMMARY
\n"; - -print AGGREGATED "
GENERAL FAILURES
\n"; -print AGGREGATED "\n"; -my $tableheader = ""; -for (@severities) { $tableheader .= ""; } -$tableheader .= ""; -print AGGREGATED "$tableheader\n"; -for my $category (keys %{$general_failures_num_by_severity}) -{ - print_category_specific_summary($category, $general_failures_by_category_severity->{$category}); - my $categoryline = ""; - for (@severities) - { - my $failuresbyseverity = 0; - $failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_}); - $categoryline .= ""; - } - $categoryline .= ""; - print AGGREGATED "$categoryline\n"; -} -print AGGREGATED "
category$_
$category$failuresbyseverity
\n"; -print AGGREGATED "
\n"; - -print AGGREGATED "
PACKGE-SPECIFIC FAILURES
\n"; -print AGGREGATED "\n"; -$tableheader = ""; -for (@severities) { $tableheader .= ""; } -$tableheader .= ""; -print AGGREGATED "$tableheader\n"; -for my $package (keys %{$recipe_failures_num_by_severity}) -{ - print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package}); - my $packagesummaryhtml = $package; - $packagesummaryhtml =~ s,/,_,; - $packagesummaryhtml .= ".html"; - my $packageline = ""; - for (@severities) - { - my $failuresbyseverity = 0; - $failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_}); - $packageline .= ""; - } - $packageline .= ""; - print AGGREGATED "$packageline\n"; -} -print AGGREGATED "
package$_
$package$failuresbyseverity
\n"; -close(AGGREGATED); - -translate_detail_files_to_html(); - - -sub print_category_specific_summary -{ - my ($category, $failures_by_severity) = @_; - - my $filenamebase = $category; - $filenamebase =~ s,/,_,; - - open(SPECIFIC, ">$outputdir/$filenamebase.html"); - print SPECIFIC "FAILURES FOR CATEGORY $category
\n"; - - for my $severity (@severities) - { - if (defined $failures_by_severity->{$severity}) - { - print SPECIFIC "
".uc($severity)."
\n"; - print SPECIFIC "\n"; - # $subcategory, $severity, $component, $phase, $recipe, $file, $line - my $tableheader = ""; - print SPECIFIC "$tableheader\n"; - - for my $failure (@{$failures_by_severity->{$severity}}) - { - my $failureline = ""; - $failureline .= ""; - $failureline .= ""; - $failureline .= ""; - print SPECIFIC "$failureline\n"; - } - - print SPECIFIC "
categoryconfigurationlog snippet
$failure->{subcategory}$failure->{config}item $failure->{linenum}
\n"; - print SPECIFIC "
\n"; - } - } - - close(SPECIFIC); -} - -sub print_package_specific_summary -{ - my ($package, $failures_by_severity) = @_; - - my $filenamebase = $package; - $filenamebase =~ s,/,_,; - - open(SPECIFIC, ">$outputdir/$filenamebase.html"); - print SPECIFIC "FAILURES FOR PACKAGE $package
\n"; - - for my $severity (@severities) - { - if (defined $failures_by_severity->{$severity}) - { - print SPECIFIC "
".uc($severity)."
\n"; - print SPECIFIC "\n"; - # $subcategory, $severity, $component, $phase, $recipe, $file, $line - my $tableheader = ""; - print SPECIFIC "$tableheader\n"; - - for my $failure (@{$failures_by_severity->{$severity}}) - { - my $failureline = ""; - $failureline .= ""; - $failureline .= ""; - $failureline .= ""; - $failureline .= ""; - $failureline .= ""; - $failureline .= ""; - print SPECIFIC "$failureline\n"; - } - - print SPECIFIC "
categoryconfigurationcomponentphaserecipelog snippet
$failure->{subcategory}$failure->{config}$failure->{component}$failure->{phase}$failure->{recipe}item $failure->{linenum}
\n"; - print SPECIFIC "
\n"; - } - } - - close(SPECIFIC); -} - -sub translate_detail_files_to_html -{ - opendir(DIR, $raptorbitsdir); - my @failurefiles = readdir(DIR); - closedir(DIR); - @failurefiles = grep(/\.txt$/, @failurefiles); - - for my $file (@failurefiles) - { - $file =~ /(.*)\.txt$/; - my $filenamebase = $1; - - my $filecontent = ''; - open(FILE, "$raptorbitsdir/$file"); - { - local $/=undef; - $filecontent = ; - } - close(FILE); - - $filecontent =~ s,---(failure_item_\d+)---,---$1---,g; - $filecontent = "
$filecontent
"; - - open(FILE, ">$outputdir/$filenamebase\_failures.html"); - print FILE $filecontent; - close(FILE); - } -} diff -r 5fdd5e70280d -r d00048f1b036 common/tools/raptor/uh.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/uh.pl Fri Nov 13 17:47:50 2009 +0000 @@ -0,0 +1,358 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Unite and HTML-ize Raptor log files + +use strict; +use FindBin; +use lib $FindBin::Bin; +use RaptorError; +use RaptorWarning; +use RaptorInfo; +use RaptorUnreciped; +use RaptorRecipe; + +use XML::SAX; +use RaptorSAXHandler; +use Getopt::Long; + +our $raptorbitsdir = 'raptorbits'; +our $basedir = ''; +my $outputdir = "html"; +our $raptor_config = 'dummy_config'; +our $current_log_file = ''; +my $help = 0; +GetOptions(( + 'basedir=s' => \$basedir, + 'help!' => \$help +)); +my @logfiles = @ARGV; + +$help = 1 if (!@logfiles); + +if ($help) +{ + print "Unite and HTML-ize Raptor log files.\n"; + print "Usage: perl uh.pl [OPTIONS] FILE1 FILE2 ...\n"; + print "where OPTIONS are:\n"; + print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n"; + exit(0); +} + +if ($basedir) +{ + $raptorbitsdir = "$basedir/raptorbits"; + $outputdir = "$basedir/html"; +} +mkdir($basedir) if (!-d$basedir); + +system("rmdir /S /Q $raptorbitsdir") if (-d $raptorbitsdir); +mkdir($raptorbitsdir); +#print "Created dir $raptorbitsdir.\n"; + +# create empty summary file anyway +open(SUMMARY, ">$raptorbitsdir/summary.csv"); +close(SUMMARY); + +my $saxhandler = RaptorSAXHandler->new(); +$saxhandler->add_observer('RaptorError', $RaptorError::reset_status); +$saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status); +$saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status); +$saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status); +$saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status); + +my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); +for (@logfiles) +{ + print "Reading file: $_\n"; + $current_log_file = $_; + $parser->parse_uri($_); +} + +print "Generating HTML...\n"; + +system("rd /S /Q $outputdir") if (-d $outputdir); +mkdir ($outputdir); + +my $raptor_errors = {}; +my $raptor_warnings = {}; +my $raptor_unreciped = {}; +my $general_failures_num_by_severity = {}; +my $general_failures_by_category_severity = {}; +my $recipe_failures_num_by_severity = {}; +my $recipe_failures_by_package_severity = {}; +#my $severities = {}; +my @severities = ('critical', 'major', 'minor', 'unknown'); + +# READ SUMMARY.CSV FILE +my $csv_file = "$raptorbitsdir/summary.csv"; +my $csv_linenum = 0; +open(CSV, $csv_file); +while() +{ + $csv_linenum ++; + my $line = $_; + + if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/) + { + my $failure = {}; + $failure->{category} = $1; + $failure->{subcategory} = $2; + $failure->{severity} = $3; + $failure->{config} = $4; + $failure->{component} = $5; + $failure->{mmp} = $6; + $failure->{phase} = $7; + $failure->{recipe} = $8; + $failure->{file} = $9; + $failure->{linenum} = $10; + + my $failure_package = ''; + + if (!$failure->{category}) + { + print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n"; + next; + } + + if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component}) + { + print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n"; + next; + } + if ($failure->{component}) + { + if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),) + { + $failure_package = $1; + } + else + { + print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n"; + next; + } + } + + $failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory}); + $failure->{severity} = 'unknown' if (!$failure->{severity}); + $failure->{mmp} = '-' if (!$failure->{mmp}); + + # populate severities dynamically. + #$severities->{$failure->{severity}} = 1; + + # put failure items into their category container + if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i) + { + $general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}}); + my $general_failure = $general_failures_num_by_severity->{$failure->{category}}; + + if (!defined $general_failure->{$failure->{severity}}) + { + $general_failure->{$failure->{severity}} = 1; + } + else + { + $general_failure->{$failure->{severity}} ++; + } + + $general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}}); + $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}); + push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure); + } + elsif ($failure->{category} =~ /^recipe_failure$/i) + { + $recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package}); + my $package_failure = $recipe_failures_num_by_severity->{$failure_package}; + + if (!defined $package_failure->{$failure->{severity}}) + { + $package_failure->{$failure->{severity}} = 1; + } + else + { + $package_failure->{$failure->{severity}} ++; + } + + $recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package}); + $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}); + push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure); + } + } + else + { + print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n"; + } +} +close(CSV); + +# PRINT HTML SUMMARY +my $aggregated_html = "$outputdir/index.html"; +open(AGGREGATED, ">$aggregated_html"); +print AGGREGATED "RAPTOR BUILD SUMMARY
\n"; + +print AGGREGATED "
GENERAL FAILURES
\n"; +print AGGREGATED "\n"; +my $tableheader = ""; +for (@severities) { $tableheader .= ""; } +$tableheader .= ""; +print AGGREGATED "$tableheader\n"; +for my $category (keys %{$general_failures_num_by_severity}) +{ + print_category_specific_summary($category, $general_failures_by_category_severity->{$category}); + my $categoryline = ""; + for (@severities) + { + my $failuresbyseverity = 0; + $failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_}); + $categoryline .= ""; + } + $categoryline .= ""; + print AGGREGATED "$categoryline\n"; +} +print AGGREGATED "
category$_
$category$failuresbyseverity
\n"; +print AGGREGATED "
\n"; + +print AGGREGATED "
PACKGE-SPECIFIC FAILURES
\n"; +print AGGREGATED "\n"; +$tableheader = ""; +for (@severities) { $tableheader .= ""; } +$tableheader .= ""; +print AGGREGATED "$tableheader\n"; +for my $package (keys %{$recipe_failures_num_by_severity}) +{ + print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package}); + my $packagesummaryhtml = $package; + $packagesummaryhtml =~ s,/,_,; + $packagesummaryhtml .= ".html"; + my $packageline = ""; + for (@severities) + { + my $failuresbyseverity = 0; + $failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_}); + $packageline .= ""; + } + $packageline .= ""; + print AGGREGATED "$packageline\n"; +} +print AGGREGATED "
package$_
$package$failuresbyseverity
\n"; +close(AGGREGATED); + +translate_detail_files_to_html(); + +print "OK, done. Please open $outputdir/index.html.\n"; + + +sub print_category_specific_summary +{ + my ($category, $failures_by_severity) = @_; + + my $filenamebase = $category; + $filenamebase =~ s,/,_,; + + open(SPECIFIC, ">$outputdir/$filenamebase.html"); + print SPECIFIC "FAILURES FOR CATEGORY $category
\n"; + + for my $severity (@severities) + { + if (defined $failures_by_severity->{$severity}) + { + print SPECIFIC "
".uc($severity)."
\n"; + print SPECIFIC "\n"; + # $subcategory, $severity, $mmp, $phase, $recipe, $file, $line + my $tableheader = ""; + print SPECIFIC "$tableheader\n"; + + for my $failure (@{$failures_by_severity->{$severity}}) + { + my $failureline = ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + print SPECIFIC "$failureline\n"; + } + + print SPECIFIC "
categorylog filelog snippet
$failure->{subcategory}$failure->{config}item $failure->{linenum}
\n"; + print SPECIFIC "
\n"; + } + } + + close(SPECIFIC); +} + +sub print_package_specific_summary +{ + my ($package, $failures_by_severity) = @_; + + my $filenamebase = $package; + $filenamebase =~ s,/,_,; + + open(SPECIFIC, ">$outputdir/$filenamebase.html"); + print SPECIFIC "FAILURES FOR PACKAGE $package
\n"; + + for my $severity (@severities) + { + if (defined $failures_by_severity->{$severity}) + { + print SPECIFIC "
".uc($severity)."
\n"; + print SPECIFIC "\n"; + # $subcategory, $severity, $mmp, $phase, $recipe, $file, $line + my $tableheader = ""; + print SPECIFIC "$tableheader\n"; + + for my $failure (@{$failures_by_severity->{$severity}}) + { + my $failureline = ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + print SPECIFIC "$failureline\n"; + } + + print SPECIFIC "
categoryconfigurationmmpphaserecipelog snippet
$failure->{subcategory}$failure->{config}$failure->{mmp}$failure->{phase}$failure->{recipe}item $failure->{linenum}
\n"; + print SPECIFIC "
\n"; + } + } + + close(SPECIFIC); +} + +sub translate_detail_files_to_html +{ + opendir(DIR, $raptorbitsdir); + my @failurefiles = readdir(DIR); + closedir(DIR); + @failurefiles = grep(/\.txt$/, @failurefiles); + + for my $file (@failurefiles) + { + $file =~ /(.*)\.txt$/; + my $filenamebase = $1; + + my $filecontent = ''; + open(FILE, "$raptorbitsdir/$file"); + { + local $/=undef; + $filecontent = ; + } + close(FILE); + + $filecontent =~ s,---(failure_item_\d+)---,---$1---,g; + $filecontent = "
$filecontent
"; + + open(FILE, ">$outputdir/$filenamebase\_failures.html"); + print FILE $filecontent; + close(FILE); + } +} \ No newline at end of file