# HG changeset patch # User arnaudl@UK-ARNAUDL # Date 1251296276 -3600 # Node ID 6734444dd1f2def4df107be120825b74a93efec0 # Parent 63fdf0583f67ed9c77afe26f32c036d9a9833dc0# Parent 52675b624b66719cb73396f2347cd9700e205ca0 Merge. diff -r 63fdf0583f67 -r 6734444dd1f2 common/build.postbuild.xml --- a/common/build.postbuild.xml Wed Aug 26 15:14:59 2009 +0100 +++ b/common/build.postbuild.xml Wed Aug 26 15:17:56 2009 +0100 @@ -301,8 +301,8 @@ - - + + @@ -318,6 +318,7 @@ + @@ -337,21 +338,34 @@ + + + + + - + + + + + + + + + - - + + @@ -456,7 +470,7 @@ ant: antProperties() - raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,component,phase,recipe,file,line]}) + raptor_summary: csv(${build.log.dir}/raptorbits/summary.csv,{separator:',',headers:[category,subcategory,severity,config,component,phase,recipe,file,line]}) files: csv(${build.drive}/output/logs/analysis/tmp_yarp_files.csv,{separator:',',headers:[name,path,localpath]}) diff -r 63fdf0583f67 -r 6734444dd1f2 common/build.xml --- a/common/build.xml Wed Aug 26 15:14:59 2009 +0100 +++ b/common/build.xml Wed Aug 26 15:17:56 2009 +0100 @@ -27,6 +27,12 @@ + + + + + + @@ -57,20 +63,32 @@ - [SF-PREP] + + + + + + + + + + +Insufficient space to run this build to completion. (Was looking for ${sf.drive.space.needed}.) To suppress this check, set the property sf.suppress.drive.space.check to any value. + + - - + + - - - - + + + + diff -r 63fdf0583f67 -r 6734444dd1f2 common/common_props.ant.xml --- a/common/common_props.ant.xml Wed Aug 26 15:14:59 2009 +0100 +++ b/common/common_props.ant.xml Wed Aug 26 15:17:56 2009 +0100 @@ -2,6 +2,8 @@ + + @@ -13,19 +15,20 @@ - + + - - diff -r 63fdf0583f67 -r 6734444dd1f2 common/templates/truclean.ant.xml.ftl --- a/common/templates/truclean.ant.xml.ftl Wed Aug 26 15:14:59 2009 +0100 +++ b/common/templates/truclean.ant.xml.ftl Wed Aug 26 15:17:56 2009 +0100 @@ -6,6 +6,8 @@ <#assign dollar="$"/> <#list data as pkg_detail> + <#if pkg_detail.sysdef != ""> + @@ -25,6 +27,8 @@ <#assign target_depends="${target_depends}"+","+"sf-truclean-${count}"/> <#assign count=count+1/> + + diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/populateziptemplate.pl --- a/common/tools/populateziptemplate.pl Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/populateziptemplate.pl Wed Aug 26 15:17:56 2009 +0100 @@ -68,7 +68,7 @@ my $keyAttr = { config => "name", name => "set"}; # Load template my $xml = XML::Simple->new(); -my $zipConfig = $xml->XMLin($template, KeyAttr => $keyAttr); +my $zipConfig = $xml->XMLin($template, keyattr => $keyAttr); my @allRndFiles; # For each package in CSV... @@ -148,7 +148,7 @@ my @excludes = map { {name => "exclude", value => "$_"} } @allRndFiles; push @{$zipConfig->{config}->{config}->{bin}->{config}->{set}}, @excludes; -$xml->XMLout($zipConfig, OutputFile => $ftl, XMLDecl => 1, RootName => 'build', KeyAttr => $keyAttr); +$xml->XMLout($zipConfig, OutputFile => $ftl, XMLDecl => 1, RootName => 'build', keyattr => $keyAttr); # Output all rnd files into exclude list for later open my $fh, ">", $rndExcludes or die "Cannot write exlude file!"; diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/RaptorCommon.pm --- a/common/tools/raptor/RaptorCommon.pm Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/RaptorCommon.pm Wed Aug 26 15:17:56 2009 +0100 @@ -14,22 +14,8 @@ package RaptorCommon; -our $CATEGORY_GENERAL = 'general'; - -our $CATEGORY_RAPTORERROR = 'raptor_error'; -our $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION = 'cannot_process_schema_version'; -our $CATEGORY_RAPTORERROR_NOBLDINFFOUND = 'no_bld_inf_found'; -our $CATEGORY_RAPTORERROR_CANTFINDMMPFILE = 'cant_find_mmp_file'; -our $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS = 'make_exited_with_errors'; -our $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION = 'tool_didnot_return_version'; - -our $CATEGORY_RAPTORWARNING = 'raptor_warning'; -our $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2 = 'missing_enable_abiv2_mode'; - -our $SEVERITY_UNKNOWN = 'unknown'; our $SEVERITY_CRITICAL = 'critical'; our $SEVERITY_MAJOR = 'major'; -our $SEVERITY_NORMAL = 'normal'; our $SEVERITY_MINOR = 'minor'; sub init @@ -48,7 +34,7 @@ my ($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line) = @_; open(SUMMARY, ">>$::basedir/summary.csv"); - print SUMMARY "$category,$subcategory,$severity,$component,$phase,$recipe,$file,$line\n"; + print SUMMARY "$category,$subcategory,$severity,$::raptor_config,$component,$phase,$recipe,$file,$line\n"; close(SUMMARY); } diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/RaptorError.pm --- a/common/tools/raptor/RaptorError.pm Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/RaptorError.pm Wed Aug 26 15:17:56 2009 +0100 @@ -36,51 +36,57 @@ $buildlog_error_status->{on_chars} = 'RaptorError::on_chars_buildlog_error'; my $filename = ''; +my $failure_item = 0; my $characters = ''; -my $category = $RaptorCommon::CATEGORY_RAPTORERROR; +my $CATEGORY_RAPTORERROR = 'raptor_error'; +my $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION = 'cannot_process_schema_version'; +my $CATEGORY_RAPTORERROR_NOBLDINFFOUND = 'no_bld_inf_found'; +my $CATEGORY_RAPTORERROR_CANTFINDMMPFILE = 'cant_find_mmp_file'; +my $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS = 'make_exited_with_errors'; +my $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION = 'tool_didnot_return_version'; sub process { my ($text, $component, $phase, $recipe, $file, $line) = @_; - my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + my $category = $CATEGORY_RAPTORERROR; + my $severity = ''; + my $subcategory = ''; if ($text =~ m,Cannot process schema version .* of file,) { $severity = $RaptorCommon::SEVERITY_CRITICAL; - my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION; + $subcategory = $CATEGORY_RAPTORERROR_CANNOTPROCESSSCHEMAVERSION; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } elsif ($text =~ m,No bld\.inf found at,) { $severity = $RaptorCommon::SEVERITY_MAJOR; - my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_NOBLDINFFOUND; + $subcategory = $CATEGORY_RAPTORERROR_NOBLDINFFOUND; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } elsif ($text =~ m,Can't find mmp file,) { - $severity = $RaptorCommon::SEVERITY_NORMAL; - my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_CANTFINDMMPFILE; + $severity = $RaptorCommon::SEVERITY_MINOR; + $subcategory = $CATEGORY_RAPTORERROR_CANTFINDMMPFILE; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } elsif ($text =~ m,The make-engine exited with errors,) { $severity = $RaptorCommon::SEVERITY_CRITICAL; - my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS; + $subcategory = $CATEGORY_RAPTORERROR_MAKEEXITEDWITHERRORS; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } elsif ($text =~ m,tool .* from config .* did not return version .* as required,) { $severity = $RaptorCommon::SEVERITY_CRITICAL; - my $subcategory = $RaptorCommon::CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION; + $subcategory = $CATEGORY_RAPTORERROR_TOOLDIDNOTRETURNVERSION; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } else # log everything by default { - $severity = $RaptorCommon::SEVERITY_NORMAL; - my $subcategory = ''; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } } @@ -89,7 +95,7 @@ { RaptorCommon::init(); - $filename = "$::basedir/errors.txt"; + $filename = "$::basedir/raptor_error.txt"; if (!-f$filename) { print "Writing errors file $filename\n"; @@ -100,7 +106,6 @@ sub on_start_buildlog_error { - open(FILE, ">>$filename"); } sub on_chars_buildlog_error @@ -118,17 +123,31 @@ { #print "on_end_buildlog_error\n"; - print FILE $characters if ($characters =~ m,[^\s^\r^\n],); - print FILE "\n" if ($characters !~ m,[\r\n]$, ); - close(FILE); + $characters =~ s,^[\r\n]*,,; + $characters =~ s,[\r\n]*$,,; - # get the line number - not really optimized - my $linecount = 0; - open(FILE, "$filename"); - for ($linecount = 0; ; $linecount++) { } - close(FILE); + if ($characters =~ m,[^\s^\r^\n],) + { + if ($failure_item == 0 and -f "$filename") + { + open(FILE, "$filename"); + { + local $/ = undef; + my $filecontent = ; + $failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s); + } + close(FILE); + } + + $failure_item++; - process($characters, '', '', '', "errors.txt", $linecount); + open(FILE, ">>$filename"); + print FILE "---failure_item_$failure_item\---\n"; + print FILE "$characters\n\n"; + close(FILE); + + process($characters, '', '', '', "raptor_error.txt", $failure_item); + } $characters = ''; } diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/RaptorInfo.pm --- a/common/tools/raptor/RaptorInfo.pm Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/RaptorInfo.pm Wed Aug 26 15:17:56 2009 +0100 @@ -42,7 +42,7 @@ { my ($text) = @_; - my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + my $severity = ''; if ($text =~ m,unmatchable,) { diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/RaptorRecipe.pm --- a/common/tools/raptor/RaptorRecipe.pm Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/RaptorRecipe.pm Wed Aug 26 15:17:56 2009 +0100 @@ -21,70 +21,92 @@ our $reset_status = {}; my $buildlog_status = {}; my $buildlog_recipe_status = {}; +my $buildlog_recipe_status_status = {}; $reset_status->{name} = 'reset_status'; $reset_status->{next_status} = {buildlog=>$buildlog_status}; $buildlog_status->{name} = 'buildlog_status'; $buildlog_status->{next_status} = {recipe=>$buildlog_recipe_status}; +$buildlog_status->{on_start} = 'RaptorRecipe::on_start_buildlog'; +$buildlog_status->{on_end} = 'RaptorRecipe::on_end_buildlog'; $buildlog_recipe_status->{name} = 'buildlog_recipe_status'; -$buildlog_recipe_status->{next_status} = {}; +$buildlog_recipe_status->{next_status} = {status=>$buildlog_recipe_status_status}; $buildlog_recipe_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe'; $buildlog_recipe_status->{on_end} = 'RaptorRecipe::on_end_buildlog_recipe'; $buildlog_recipe_status->{on_chars} = 'RaptorRecipe::on_chars_buildlog_recipe'; -my $characters = ''; +$buildlog_recipe_status_status->{name} = 'buildlog_recipe_status_status'; +$buildlog_recipe_status_status->{next_status} = {}; +$buildlog_recipe_status_status->{on_start} = 'RaptorRecipe::on_start_buildlog_recipe_status'; + + +my $filename = ''; +my $failure_item = 0; + my $recipe_info = {}; -my $category = $RaptorCommon::CATEGORY_RAPTORRECIPE; +my $characters = ''; + +my $CATEGORY_RECIPEFAILURE = 'recipe_failure'; +my $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE = 'armcc_cannot_open_source_input_file'; +my $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE = 'armlink_could_not_open_file'; +my $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE = 'elf2e32_could_not_open_file'; sub process { - my ($text) = @_; + my ($text, $component, $phase, $recipe, $file, $line) = @_; + + my $category = $CATEGORY_RECIPEFAILURE; + my $severity = ''; + my $subcategory = ''; - my $severity = $RaptorCommon::SEVERITY_UNKNOWN; - - if ($text =~ m,Cannot process schema version .* of file,) + if ($text =~ m,Error: #5: cannot open source input file .*: No such file or directory,) + { + $severity = $RaptorCommon::SEVERITY_MAJOR; + my $subcategory = $CATEGORY_RECIPEFAILURE_ARMCC_CANNOTOPENSOURCEINPUTFILE; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,Fatal error: L6002U: Could not open file .*: No such file or directory,) { - $severity = $RaptorCommon::SEVERITY_CRITICAL; - - #dump_recipe($category, $severity, $text); - print "$category, $severity, $text\n"; + $severity = $RaptorCommon::SEVERITY_MAJOR; + my $subcategory = $CATEGORY_RECIPEFAILURE_ARMLINK_COULDNOTOPENFILE; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,elf2e32 : Error: E1001: Could not open file : .*.,) + { + $severity = $RaptorCommon::SEVERITY_MAJOR; + my $subcategory = $CATEGORY_RECIPEFAILURE_ELF2E32_COULDNOTOPENFILE; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } + else # log everything by default + { + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } +} + +sub on_start_buildlog +{ + #print FILE "line,layer,component,name,armlicence,platform,phase,code,bldinf,mmp,target,source,\n"; + + RaptorCommon::init(); } sub on_start_buildlog_recipe { my ($el) = @_; - $recipe_info = {}; - $characters = ''; + #print "on_start_buildlog_recipe\n"; - my $attrstring = ''; - my $bldinf = ''; + $recipe_info = {}; my $attributes = $el->{Attributes}; for (keys %{$attributes}) { - if ($attributes->{$_}->{'LocalName'} eq 'bldinf') - { - $bldinf = $attributes->{$_}->{'Value'}; - } - - $attrstring .= "$attributes->{$_}->{'LocalName'}}='$attributes->{$_}->{'Value'}' "; - + $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'}; + #print "$_ -> $attributes->{$_}->{'Value'}\n"; } - - if ($bldinf eq '') - { - print "WARNING: recipe tag with no bldinf attribute. Associating to package unknown/unknown\n"; - $bldinf = "/sf/unknown/unknown/group/bld.inf"; - } - - $recipe_info->{bldinf} = $bldinf; - - $characters = "\n"; } sub on_chars_buildlog_recipe @@ -98,30 +120,88 @@ #print "characters is now -->$characters<--\n"; } +sub on_start_buildlog_recipe_status +{ + my ($el) = @_; + + my $attributes = $el->{Attributes}; + for (keys %{$attributes}) + { + if ($attributes->{$_}->{'LocalName'} eq 'code') + { + $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'}; + } + elsif ($attributes->{$_}->{'LocalName'} eq 'exit') + { + $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'}; + } + elsif ($attributes->{$_}->{'LocalName'} eq 'attempt') + { + $recipe_info->{$attributes->{$_}->{'LocalName'}} = $attributes->{$_}->{'Value'}; + } + } +} + sub on_end_buildlog_recipe { - #print "on_end_buildlog_recipe\n"; - - $characters .= "\n\n"; - - my $normalized = lc($recipe_info->{bldinf}); - $normalized =~ s,^[A-Za-z]:,,; - $normalized =~ s,[\\],/,g; - - $normalized =~ m,^/sf/([^/]+)/([^/]+)/,; - my $layer = $1; - my $package = $2; - - mkdir("$::basedir/recipes"); - mkdir("$::basedir/recipes/$layer"); - mkdir("$::basedir/recipes/$layer/$package"); - - my $filename = "$::basedir/recipes/$layer/$package/recipes.txt"; - - print "Writing recipes file $filename\n" if (!-f$filename); - open(FILE, ">>$filename"); - print FILE $characters; - close(FILE); + if ($recipe_info->{exit} =~ /failed/) + { + # normalize bldinf path + $recipe_info->{bldinf} = lc($recipe_info->{bldinf}); + $recipe_info->{bldinf} =~ s,^[A-Za-z]:,,; + $recipe_info->{bldinf} =~ s,[\\],/,g; + + my $package = ''; + if ($recipe_info->{bldinf} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),) + { + $package = $1; + $package =~ s,/,_,; + } + else + { + print "WARNING: can't understand bldinf attribute of recipe: $recipe_info->{bldinf}. Won't dump to failed recipes file.\n"; + } + + $characters =~ s,^[\r\n]*,,; + $characters =~ s,[\r\n]*$,,; + + if ($package) + { + $filename = "$::basedir/$package.txt"; + if (!-f$filename) + { + print "Writing recipe file $filename\n"; + open(FILE, ">$filename"); + close(FILE); + } + + if ($failure_item == 0 and -f "$filename") + { + open(FILE, "$filename"); + { + local $/ = undef; + my $filecontent = ; + $failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s); + } + close(FILE); + } + + $failure_item++; + + open(FILE, ">>$filename"); + print FILE "---failure_item_$failure_item\---\n"; + print FILE "$characters\n\n"; + close(FILE); + } + + process($characters, $recipe_info->{bldinf}, $recipe_info->{phase}, $recipe_info->{name}, "$package.txt", $failure_item); + } + + $characters = ''; +} + +sub on_end_buildlog +{ } diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/RaptorUnreciped.pm --- a/common/tools/raptor/RaptorUnreciped.pm Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/RaptorUnreciped.pm Wed Aug 26 15:17:56 2009 +0100 @@ -33,30 +33,154 @@ $buildlog_subtag_status->{name} = 'buildlog_subtag_status'; $buildlog_subtag_status->{next_status} = {}; +$buildlog_subtag_status->{on_start} = 'RaptorUnreciped::on_start_buildlog_subtag'; +$buildlog_subtag_status->{on_end} = 'RaptorUnreciped::on_end_buildlog_subtag'; + +my $filename = ''; +my $failure_item = 0; my $characters = ''; +my $store_chars = 1; + +my $CATEGORY_RAPTORUNRECIPED = 'raptor_unreciped'; +my $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET = 'no_rule_to_make_target'; +my $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS = 'target_not_remade_for_errors'; +my $CATEGORY_RAPTORUNRECIPED_NOTHINGTOBEDONEFOR = 'nothing_to_be_done_for'; +my $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET = 'ignoring_old_commands_for_target'; +my $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET = 'overriding_commands_for_target'; + +sub process +{ + my ($text, $component, $phase, $recipe, $file, $line) = @_; + + my $category = $CATEGORY_RAPTORUNRECIPED; + my $severity = ''; + my $subcategory = ''; + + if ($text =~ m,make\.exe: \*\*\* No rule to make target,) + { + $severity = $RaptorCommon::SEVERITY_MAJOR; + my $subcategory = $CATEGORY_RAPTORUNRECIPED_NORULETOMAKETARGET; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,make\.exe: Target .* not remade because of errors,) + { + $severity = $RaptorCommon::SEVERITY_MINOR; + my $subcategory = $CATEGORY_RAPTORUNRECIPED_TARGETNOTREMADEFORERRORS; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,: warning: ignoring old commands for target,) + { + $severity = $RaptorCommon::SEVERITY_MINOR; + my $subcategory = $CATEGORY_RAPTORUNRECIPED_IGNORINGOLDCOMMANDSFORTARGET; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,: warning: overriding commands for target,) + { + $severity = $RaptorCommon::SEVERITY_MINOR; + my $subcategory = $CATEGORY_RAPTORUNRECIPED_OVERRIDINGCOMMANDSFORTARGET; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,make\.exe: Nothing to be done for .*,) + { + $severity = $RaptorCommon::SEVERITY_MINOR; + my $subcategory = $CATEGORY_RAPTORUNRECIPED_NOTHINGTOBEDONEFOR; + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } + elsif ($text =~ m,^(true|false)$,) + { + # don't dump + } + else # log everything by default + { + RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); + } +} sub on_start_buildlog { - my $filename = "$::basedir/unreciped.txt"; - print "Writing unreciped file $filename\n" if (!-f$filename); - open(FILE, ">>$filename"); + RaptorCommon::init(); + + $filename = "$::basedir/raptor_unreciped.txt"; + if (!-f$filename) + { + print "Writing unreciped file $filename\n"; + open(FILE, ">$filename"); + close(FILE); + } } sub on_chars_buildlog { my ($ch) = @_; - my $characters = $ch->{Data}; + #print "on_chars_buildlog\n"; + + if ($store_chars) + { + $characters .= $ch->{Data}; + + #print "characters is now -->$characters<--\n"; + } +} + +sub on_end_buildlog_subtag +{ + $store_chars = 1; +} + +sub process_characters +{ + #print "process_characters\n"; + + $characters =~ s,^[\r\n]*,,; + $characters =~ s,[\r\n]*$,,; + + #print "characters is -->$characters<--\n"; - print FILE $characters if ($characters =~ m,[^\s^\r^\n],); + my @lines = split(/[\r\n]/, $characters); + for my $line (@lines) + { + if ($line =~ m,[^\s^\r^\n],) + { + #print "dumping chars\n"; + + if ($failure_item == 0 and -f "$filename") + { + open(FILE, "$filename"); + { + local $/ = undef; + my $filecontent = ; + $failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s); + } + close(FILE); + } + + $failure_item++; + + open(FILE, ">>$filename"); + print FILE "---failure_item_$failure_item\---\n"; + print FILE "$line\n\n"; + close(FILE); + + process($line, '', '', '', "raptor_unreciped.txt", $failure_item); + } + } - #print "characters is now -->$characters<--\n"; + $characters = ''; + $store_chars = 0; +} + +sub on_start_buildlog_subtag +{ + #print "on_start_buildlog_subtag\n"; + + process_characters(); } sub on_end_buildlog { - close(FILE); + process_characters(); } diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/RaptorWarning.pm --- a/common/tools/raptor/RaptorWarning.pm Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/RaptorWarning.pm Wed Aug 26 15:17:56 2009 +0100 @@ -36,27 +36,29 @@ $buildlog_warning_status->{on_chars} = 'RaptorWarning::on_chars_buildlog_warning'; my $filename = ''; +my $failure_item = 0; my $characters = ''; -my $category = $RaptorCommon::CATEGORY_RAPTORWARNING; +my $CATEGORY_RAPTORWARNING = 'raptor_warning'; +my $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2 = 'missing_enable_abiv2_mode'; sub process { my ($text, $component, $phase, $recipe, $file, $line) = @_; - my $severity = $RaptorCommon::SEVERITY_UNKNOWN; + my $category = $CATEGORY_RAPTORWARNING; + my $severity = ''; + my $subcategory = ''; if ($text =~ m,missing flag ENABLE_ABIV2_MODE,) { - $severity = $RaptorCommon::SEVERITY_NORMAL; - my $subcategory = $RaptorCommon::CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2; + $severity = $RaptorCommon::SEVERITY_MINOR; + my $subcategory = $CATEGORY_RAPTORWARNING_MISSINGFLAGABIV2; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } else # log everything by default { - $severity = $RaptorCommon::SEVERITY_NORMAL; - my $subcategory = ''; RaptorCommon::dump_fault($category, $subcategory, $severity, $component, $phase, $recipe, $file, $line); } } @@ -65,7 +67,7 @@ { RaptorCommon::init(); - $filename = "$::basedir/warnings.txt"; + $filename = "$::basedir/raptor_warning.txt"; if (!-f$filename) { print "Writing warnings file $filename\n"; @@ -93,17 +95,31 @@ { #print "on_end_buildlog_warning\n"; - print FILE $characters if ($characters =~ m,[^\s^\r^\n],); - print FILE "\n" if ($characters !~ m,[\r\n]$, ); - close(FILE); + $characters =~ s,^[\r\n]*,,; + $characters =~ s,[\r\n]*$,,; - # get the line number - not really optimized - my $linecount = 0; - open(FILE, "$filename"); - for ($linecount = 0; ; $linecount++) { } - close(FILE); + if ($characters =~ m,[^\s^\r^\n],) + { + if ($failure_item == 0 and -f "$filename") + { + open(FILE, "$filename"); + { + local $/ = undef; + my $filecontent = ; + $failure_item = $1 if ($filecontent =~ m/.*---failure_item_(\d+)/s); + } + close(FILE); + } + + $failure_item++; - process($characters, '', '', '', "warnings.txt", $linecount); + open(FILE, ">>$filename"); + print FILE "---failure_item_$failure_item\---\n"; + print FILE "$characters\n\n"; + close(FILE); + + process($characters, '', '', '', "raptor_warning.txt", $failure_item); + } $characters = ''; } diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/parse.pl --- a/common/tools/raptor/parse.pl Wed Aug 26 15:14:59 2009 +0100 +++ b/common/tools/raptor/parse.pl Wed Aug 26 15:17:56 2009 +0100 @@ -13,6 +13,8 @@ # Run the raptor parsers use strict; +use FindBin; +use lib $FindBin::Bin; use RaptorReleaseable; use RaptorError; use RaptorWarning; @@ -32,6 +34,7 @@ my $unreciped_module = 0; my $recipe_module = 0; our $basedir = ''; +our $raptor_config = ''; my $append = 0; my $help = 0; GetOptions(( @@ -42,7 +45,8 @@ 'info!' => \$info_module, 'unreciped!' => \$unreciped_module, 'recipe!' => \$recipe_module, - 'basedir:s' => \$basedir, + 'basedir=s' => \$basedir, + 'config=s' => \$raptor_config, 'append!' => \$append, 'help!' => \$help )); diff -r 63fdf0583f67 -r 6734444dd1f2 common/tools/raptor/summarize.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common/tools/raptor/summarize.pl Wed Aug 26 15:17:56 2009 +0100 @@ -0,0 +1,311 @@ +# Copyright (c) 2009 Symbian Foundation Ltd +# This component and the accompanying materials are made available +# under the terms of the License "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Symbian Foundation Ltd - initial contribution. +# +# Contributors: +# +# Description: +# Generate an HTML summary of the Raptor build from the output of the raptor parser + +use strict; +use FindBin; +use lib $FindBin::Bin; +use Getopt::Long; + +my $raptorbitsdir = 0; +my $outputdir = '.'; +my $help = 0; +GetOptions(( + 'raptorbitsdir=s' => \$raptorbitsdir, + 'outputdir=s' => \$outputdir, + 'help!' => \$help +)); + +$help = 1 if (!$raptorbitsdir); + +if ($help) +{ + print "Generate an HTML summary of the Raptor build from a summary.csv file\n"; + print "Usage: perl summarize.pl --raptorbitsdir=DIR [--outputdir=DIR]\n"; + exit(0); +} + +$outputdir = "$outputdir/html"; + +system("rd /S /Q $outputdir") if (-d $outputdir); +mkdir ($outputdir); + +my $raptor_errors = {}; +my $raptor_warnings = {}; +my $raptor_unreciped = {}; +my $general_failures_num_by_severity = {}; +my $general_failures_by_category_severity = {}; +my $recipe_failures_num_by_severity = {}; +my $recipe_failures_by_package_severity = {}; +#my $severities = {}; +my @severities = ('critical', 'major', 'minor', 'unknown'); + +# READ SUMMARY.CSV FILE +my $csv_file = "$raptorbitsdir/summary.csv"; +my $csv_linenum = 0; +open(CSV, $csv_file); +while() +{ + $csv_linenum ++; + my $line = $_; + + if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/) + { + my $failure = {}; + $failure->{category} = $1; + $failure->{subcategory} = $2; + $failure->{severity} = $3; + $failure->{config} = $4; + $failure->{component} = $5; + $failure->{phase} = $6; + $failure->{recipe} = $7; + $failure->{file} = $8; + $failure->{linenum} = $9; + + my $failure_package = ''; + + if (!$failure->{category}) + { + print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n"; + next; + } + + if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component}) + { + print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n"; + next; + } + if ($failure->{component}) + { + if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),) + { + $failure_package = $1; + } + else + { + print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n"; + next; + } + } + + $failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory}); + $failure->{severity} = 'unknown' if (!$failure->{severity}); + + # populate severities dynamically. + #$severities->{$failure->{severity}} = 1; + + # put failure items into their category container + if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i) + { + $general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}}); + my $general_failure = $general_failures_num_by_severity->{$failure->{category}}; + + if (!defined $general_failure->{$failure->{severity}}) + { + $general_failure->{$failure->{severity}} = 1; + } + else + { + $general_failure->{$failure->{severity}} ++; + } + + $general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}}); + $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}); + push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure); + } + elsif ($failure->{category} =~ /^recipe_failure$/i) + { + $recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package}); + my $package_failure = $recipe_failures_num_by_severity->{$failure_package}; + + if (!defined $package_failure->{$failure->{severity}}) + { + $package_failure->{$failure->{severity}} = 1; + } + else + { + $package_failure->{$failure->{severity}} ++; + } + + $recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package}); + $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}); + push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure); + } + } + else + { + print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n"; + } +} +close(CSV); + +# PRINT HTML SUMMARY +my $aggregated_html = "$outputdir/index.html"; +open(AGGREGATED, ">$aggregated_html"); +print AGGREGATED "RAPTOR BUILD SUMMARY
\n"; + +print AGGREGATED "
GENERAL FAILURES
\n"; +print AGGREGATED "\n"; +my $tableheader = ""; +for (@severities) { $tableheader .= ""; } +$tableheader .= ""; +print AGGREGATED "$tableheader\n"; +for my $category (keys %{$general_failures_num_by_severity}) +{ + print_category_specific_summary($category, $general_failures_by_category_severity->{$category}); + my $categoryline = ""; + for (@severities) + { + my $failuresbyseverity = 0; + $failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_}); + $categoryline .= ""; + } + $categoryline .= ""; + print AGGREGATED "$categoryline\n"; +} +print AGGREGATED "
category$_
$category$failuresbyseverity
\n"; +print AGGREGATED "
\n"; + +print AGGREGATED "
PACKGE-SPECIFIC FAILURES
\n"; +print AGGREGATED "\n"; +$tableheader = ""; +for (@severities) { $tableheader .= ""; } +$tableheader .= ""; +print AGGREGATED "$tableheader\n"; +for my $package (keys %{$recipe_failures_num_by_severity}) +{ + print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package}); + my $packagesummaryhtml = $package; + $packagesummaryhtml =~ s,/,_,; + $packagesummaryhtml .= ".html"; + my $packageline = ""; + for (@severities) + { + my $failuresbyseverity = 0; + $failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_}); + $packageline .= ""; + } + $packageline .= ""; + print AGGREGATED "$packageline\n"; +} +print AGGREGATED "
package$_
$package$failuresbyseverity
\n"; +close(AGGREGATED); + +translate_detail_files_to_html(); + + +sub print_category_specific_summary +{ + my ($category, $failures_by_severity) = @_; + + my $filenamebase = $category; + $filenamebase =~ s,/,_,; + + open(SPECIFIC, ">$outputdir/$filenamebase.html"); + print SPECIFIC "FAILURES FOR CATEGORY $category
\n"; + + for my $severity (@severities) + { + if (defined $failures_by_severity->{$severity}) + { + print SPECIFIC "
".uc($severity)."
\n"; + print SPECIFIC "\n"; + # $subcategory, $severity, $component, $phase, $recipe, $file, $line + my $tableheader = ""; + print SPECIFIC "$tableheader\n"; + + for my $failure (@{$failures_by_severity->{$severity}}) + { + my $failureline = ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + print SPECIFIC "$failureline\n"; + } + + print SPECIFIC "
categoryconfigurationlog snippet
$failure->{subcategory}$failure->{config}item $failure->{linenum}
\n"; + print SPECIFIC "
\n"; + } + } + + close(SPECIFIC); +} + +sub print_package_specific_summary +{ + my ($package, $failures_by_severity) = @_; + + my $filenamebase = $package; + $filenamebase =~ s,/,_,; + + open(SPECIFIC, ">$outputdir/$filenamebase.html"); + print SPECIFIC "FAILURES FOR PACKAGE $package
\n"; + + for my $severity (@severities) + { + if (defined $failures_by_severity->{$severity}) + { + print SPECIFIC "
".uc($severity)."
\n"; + print SPECIFIC "\n"; + # $subcategory, $severity, $component, $phase, $recipe, $file, $line + my $tableheader = ""; + print SPECIFIC "$tableheader\n"; + + for my $failure (@{$failures_by_severity->{$severity}}) + { + my $failureline = ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + $failureline .= ""; + print SPECIFIC "$failureline\n"; + } + + print SPECIFIC "
categoryconfigurationcomponentphaserecipelog snippet
$failure->{subcategory}$failure->{config}$failure->{component}$failure->{phase}$failure->{recipe}item $failure->{linenum}
\n"; + print SPECIFIC "
\n"; + } + } + + close(SPECIFIC); +} + +sub translate_detail_files_to_html +{ + opendir(DIR, $raptorbitsdir); + my @failurefiles = readdir(DIR); + closedir(DIR); + @failurefiles = grep(/\.txt$/, @failurefiles); + + for my $file (@failurefiles) + { + $file =~ /(.*)\.txt$/; + my $filenamebase = $1; + + my $filecontent = ''; + open(FILE, "$raptorbitsdir/$file"); + { + local $/=undef; + $filecontent = ; + } + close(FILE); + + $filecontent =~ s,---(failure_item_\d+)---,---$1---,g; + $filecontent = "
$filecontent
"; + + open(FILE, ">$outputdir/$filenamebase\_failures.html"); + print FILE $filecontent; + close(FILE); + } +} diff -r 63fdf0583f67 -r 6734444dd1f2 sf-package/package_props.ant.xml --- a/sf-package/package_props.ant.xml Wed Aug 26 15:14:59 2009 +0100 +++ b/sf-package/package_props.ant.xml Wed Aug 26 15:17:56 2009 +0100 @@ -2,6 +2,8 @@ + + @@ -21,6 +23,7 @@ + diff -r 63fdf0583f67 -r 6734444dd1f2 sf-platform/build.xml --- a/sf-platform/build.xml Wed Aug 26 15:14:59 2009 +0100 +++ b/sf-platform/build.xml Wed Aug 26 15:17:56 2009 +0100 @@ -3,6 +3,15 @@ + + + + + + + + + @@ -448,8 +457,5 @@
- - -
diff -r 63fdf0583f67 -r 6734444dd1f2 sf-platform/platform_props.ant.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sf-platform/platform_props.ant.xml Wed Aug 26 15:17:56 2009 +0100 @@ -0,0 +1,7 @@ + + + + + + + diff -r 63fdf0583f67 -r 6734444dd1f2 sf-platform/platform_refs.ant.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sf-platform/platform_refs.ant.xml Wed Aug 26 15:17:56 2009 +0100 @@ -0,0 +1,5 @@ + + + + +