uh_parser/uh.pl
changeset 176 6d3c3db11e72
child 185 b76adfbc6648
equal deleted inserted replaced
175:266a7e9b9237 176:6d3c3db11e72
       
     1 # Copyright (c) 2009 Symbian Foundation Ltd
       
     2 # This component and the accompanying materials are made available
       
     3 # under the terms of the License "Eclipse Public License v1.0"
       
     4 # which accompanies this distribution, and is available
       
     5 # at the URL "http://www.eclipse.org/legal/epl-v10.html".
       
     6 #
       
     7 # Initial Contributors:
       
     8 # Symbian Foundation Ltd - initial contribution.
       
     9 #
       
    10 # Contributors:
       
    11 #
       
    12 # Description:
       
    13 # Unite and HTML-ize Raptor log files
       
    14 
       
    15 use strict;
       
    16 use FindBin;
       
    17 use lib $FindBin::Bin;
       
    18 use RaptorError;
       
    19 use RaptorWarning;
       
    20 use RaptorInfo;
       
    21 use RaptorUnreciped;
       
    22 use RaptorRecipe;
       
    23 
       
    24 use XML::SAX;
       
    25 use RaptorSAXHandler;
       
    26 use Getopt::Long;
       
    27 
       
    28 our $raptorbitsdir = 'raptorbits';
       
    29 our $basedir = '';
       
    30 my $outputdir = "html";
       
    31 our $raptor_config = 'dummy_config';
       
    32 our $current_log_file = '';
       
    33 my $help = 0;
       
    34 GetOptions((
       
    35 	'basedir=s' => \$basedir,
       
    36 	'help!' => \$help
       
    37 ));
       
    38 my @logfiles = @ARGV;
       
    39 
       
    40 $help = 1 if (!@logfiles);
       
    41 
       
    42 if ($help)
       
    43 {
       
    44 	print "Unite and HTML-ize Raptor log files.\n";
       
    45 	print "Usage: perl uh.pl [OPTIONS] FILE1 FILE2 ...\n";
       
    46 	print "where OPTIONS are:\n";
       
    47 	print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n";
       
    48 	exit(0);
       
    49 }
       
    50 
       
    51 if ($basedir)
       
    52 {
       
    53 	$raptorbitsdir = "$basedir/raptorbits";
       
    54 	$outputdir = "$basedir/html";
       
    55 }
       
    56 mkdir($basedir) if (!-d$basedir);
       
    57 
       
    58 $raptorbitsdir =~ s,/,\\,g; # this is because rmdir doens't cope correctly with the forward slashes
       
    59 
       
    60 system("rmdir /S /Q $raptorbitsdir") if (-d $raptorbitsdir);
       
    61 mkdir($raptorbitsdir);
       
    62 #print "Created dir $raptorbitsdir.\n";
       
    63 
       
    64 # create empty summary file anyway
       
    65 open(SUMMARY, ">$raptorbitsdir/summary.csv");
       
    66 close(SUMMARY);
       
    67 
       
    68 my $saxhandler = RaptorSAXHandler->new();
       
    69 $saxhandler->add_observer('RaptorError', $RaptorError::reset_status);
       
    70 $saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status);
       
    71 $saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status);
       
    72 $saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status);
       
    73 $saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status);
       
    74 
       
    75 our $allbldinfs = {};
       
    76 
       
    77 my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);
       
    78 for (@logfiles)
       
    79 {
       
    80 	print "Reading file: $_\n";
       
    81 	$current_log_file = $_;
       
    82 	$parser->parse_uri($_);
       
    83 }
       
    84 
       
    85 my @allpackages = distinct_packages($allbldinfs);
       
    86 
       
    87 print "Generating HTML...\n";
       
    88 
       
    89 system("rd /S /Q $outputdir") if (-d $outputdir);
       
    90 mkdir ($outputdir);
       
    91 
       
    92 my $raptor_errors = {};
       
    93 my $raptor_warnings = {};
       
    94 my $raptor_unreciped = {};
       
    95 my $general_failures_num_by_severity = {};
       
    96 my $general_failures_by_category_severity = {};
       
    97 my $recipe_failures_num_by_severity = {};
       
    98 my $recipe_failures_by_package_severity = {};
       
    99 #my $severities = {};
       
   100 my @severities = ('critical', 'major', 'minor', 'unknown');
       
   101 
       
   102 # READ SUMMARY.CSV FILE
       
   103 my $csv_file = "$raptorbitsdir/summary.csv";
       
   104 my $csv_linenum = 0;
       
   105 open(CSV, $csv_file);
       
   106 while(<CSV>)
       
   107 {
       
   108 	$csv_linenum ++;
       
   109 	my $line = $_;
       
   110 	
       
   111 	if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/)
       
   112 	{
       
   113 		my $failure = {};
       
   114 		$failure->{category} = $1;
       
   115 		$failure->{subcategory} = $2;
       
   116 		$failure->{severity} = $3;
       
   117 		$failure->{config} = $4;
       
   118 		$failure->{component} = $5;
       
   119 		$failure->{mmp} = $6;
       
   120 		$failure->{phase} = $7;
       
   121 		$failure->{recipe} = $8;
       
   122 		$failure->{file} = $9;
       
   123 		$failure->{linenum} = $10;
       
   124 		
       
   125 		my $failure_package = '';
       
   126 		
       
   127 		if (!$failure->{category})
       
   128 		{
       
   129 			print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n";
       
   130 			next;
       
   131 		}
       
   132 		
       
   133 		if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component})
       
   134 		{
       
   135 			print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n";
       
   136 			next;
       
   137 		}
       
   138 		if ($failure->{component})
       
   139 		{
       
   140 			if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
       
   141 			{
       
   142 				$failure_package = $1;
       
   143 			}
       
   144 			else
       
   145 			{
       
   146 				print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n";
       
   147 				next;
       
   148 			}
       
   149 		}
       
   150 		
       
   151 		$failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory});
       
   152 		$failure->{severity} = 'unknown' if (!$failure->{severity});
       
   153 		$failure->{mmp} = '-' if (!$failure->{mmp});
       
   154 		
       
   155 		# populate severities dynamically.
       
   156 		#$severities->{$failure->{severity}} = 1;
       
   157 		
       
   158 		# put failure items into their category container
       
   159 		if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i)
       
   160 		{
       
   161 			$general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}});
       
   162 			my $general_failure = $general_failures_num_by_severity->{$failure->{category}};
       
   163 			
       
   164 			if (!defined $general_failure->{$failure->{severity}})
       
   165 			{
       
   166 				$general_failure->{$failure->{severity}} = 1;
       
   167 			}
       
   168 			else
       
   169 			{
       
   170 				$general_failure->{$failure->{severity}} ++;
       
   171 			}
       
   172 			
       
   173 			$general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}});
       
   174 			$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}});
       
   175 			push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure);
       
   176 		}
       
   177 		elsif ($failure->{category} =~ /^recipe_failure$/i)
       
   178 		{
       
   179 			$recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package});
       
   180 			my $package_failure = $recipe_failures_num_by_severity->{$failure_package};
       
   181 			
       
   182 			if (!defined $package_failure->{$failure->{severity}})
       
   183 			{
       
   184 				$package_failure->{$failure->{severity}} = 1;
       
   185 			}
       
   186 			else
       
   187 			{
       
   188 				$package_failure->{$failure->{severity}} ++;
       
   189 			}
       
   190 			
       
   191 			$recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package});
       
   192 			$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}});
       
   193 			push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure);
       
   194 		}
       
   195 	}
       
   196 	else
       
   197 	{
       
   198 		print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n";
       
   199 	}
       
   200 }
       
   201 close(CSV);
       
   202 
       
   203 # PRINT HTML SUMMARY
       
   204 my $aggregated_html = "$outputdir/index.html";
       
   205 open(AGGREGATED, ">$aggregated_html");
       
   206 print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n";
       
   207 
       
   208 print AGGREGATED "<br/>GENERAL FAILURES<br/>\n";
       
   209 print AGGREGATED "<table border='1'>\n";
       
   210 my $tableheader = "<tr><th>category</th>";
       
   211 for (@severities) { $tableheader .= "<th>$_</th>"; }
       
   212 $tableheader .= "</tr>";
       
   213 print AGGREGATED "$tableheader\n";
       
   214 for my $category (keys %{$general_failures_num_by_severity})
       
   215 {
       
   216 	print_category_specific_summary($category, $general_failures_by_category_severity->{$category});
       
   217 	my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>";
       
   218 	for (@severities)
       
   219 	{
       
   220 		my $failuresbyseverity = 0;
       
   221 		$failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_});
       
   222 		$categoryline .= "<td>$failuresbyseverity</td>";
       
   223 	}
       
   224 	$categoryline .= "</tr>";
       
   225 	print AGGREGATED "$categoryline\n";
       
   226 }
       
   227 print AGGREGATED "</table>\n";
       
   228 print AGGREGATED "<br/>\n";
       
   229 
       
   230 print AGGREGATED "<br/>PACKAGE-SPECIFIC FAILURES<br/>\n";
       
   231 print AGGREGATED "<table border='1'>\n";
       
   232 $tableheader = "<tr><th>package</th>";
       
   233 for (@severities) { $tableheader .= "<th>$_</th>"; }
       
   234 $tableheader .= "</tr>";
       
   235 print AGGREGATED "$tableheader\n";
       
   236 for my $package (@allpackages)
       
   237 {
       
   238 	if (defined $recipe_failures_num_by_severity->{$package})
       
   239 	{
       
   240 		print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package});
       
   241 		my $packagesummaryhtml = $package;
       
   242 		$packagesummaryhtml =~ s,/,_,;
       
   243 		$packagesummaryhtml .= ".html";
       
   244 		my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>";
       
   245 		for (@severities)
       
   246 		{
       
   247 			my $failuresbyseverity = 0;
       
   248 			$failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_});
       
   249 			$packageline .= "<td>$failuresbyseverity</td>";
       
   250 		}
       
   251 		$packageline .= "</tr>";
       
   252 		print AGGREGATED "$packageline\n";
       
   253 	}
       
   254 	else
       
   255 	{
       
   256 		my $packageline = "<tr><td>$package</td>";
       
   257 		for (@severities) { $packageline .= "<td>0</td>"; }
       
   258 		$packageline .= "</tr>";
       
   259 		print AGGREGATED "$packageline\n";
       
   260 	}
       
   261 }
       
   262 print AGGREGATED "</table>\n";
       
   263 close(AGGREGATED);
       
   264 
       
   265 translate_detail_files_to_html();
       
   266 
       
   267 print "OK, done. Please open $outputdir/index.html.\n";
       
   268 
       
   269 
       
   270 sub print_category_specific_summary
       
   271 {
       
   272 	my ($category, $failures_by_severity) = @_;
       
   273 	
       
   274 	my $filenamebase = $category;
       
   275 	$filenamebase =~ s,/,_,;
       
   276 	
       
   277 	open(SPECIFIC, ">$outputdir/$filenamebase.html");
       
   278 	print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n";
       
   279 		
       
   280 	for my $severity (@severities)
       
   281 	{
       
   282 		if (defined $failures_by_severity->{$severity})
       
   283 		{
       
   284 			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
       
   285 			print SPECIFIC "<table border='1'>\n";
       
   286 			# $subcategory, $severity, $mmp, $phase, $recipe, $file, $line
       
   287 			my $tableheader = "<tr><th>category</th><th>log file</th><th>log snippet</th></tr>";
       
   288 			print SPECIFIC "$tableheader\n";
       
   289 			
       
   290 			for my $failure (@{$failures_by_severity->{$severity}})
       
   291 			{
       
   292 				my $failureline = "<tr><td>$failure->{subcategory}</td>";
       
   293 				$failureline .= "<td>$failure->{config}</td>";
       
   294 				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
       
   295 				$failureline .= "</tr>";
       
   296 				print SPECIFIC "$failureline\n";
       
   297 			}
       
   298 			
       
   299 			print SPECIFIC "</table>\n";
       
   300 			print SPECIFIC "<br/>\n";
       
   301 		}
       
   302 	}
       
   303 	
       
   304 	close(SPECIFIC);
       
   305 }
       
   306 
       
   307 sub print_package_specific_summary
       
   308 {
       
   309 	my ($package, $failures_by_severity) = @_;
       
   310 	
       
   311 	my $filenamebase = $package;
       
   312 	$filenamebase =~ s,/,_,;
       
   313 	
       
   314 	open(SPECIFIC, ">$outputdir/$filenamebase.html");
       
   315 	print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n";
       
   316 		
       
   317 	for my $severity (@severities)
       
   318 	{
       
   319 		if (defined $failures_by_severity->{$severity})
       
   320 		{
       
   321 			print SPECIFIC "<br/>".uc($severity)."<br/>\n";
       
   322 			print SPECIFIC "<table border='1'>\n";
       
   323 			# $subcategory, $severity, $mmp, $phase, $recipe, $file, $line
       
   324 			my $tableheader = "<tr><th>category</th><th>configuration</th><th>mmp</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>";
       
   325 			print SPECIFIC "$tableheader\n";
       
   326 			
       
   327 			for my $failure (@{$failures_by_severity->{$severity}})
       
   328 			{
       
   329 				my $failureline = "<tr><td>$failure->{subcategory}</td>";
       
   330 				$failureline .= "<td>$failure->{config}</td>";
       
   331 				$failureline .= "<td>$failure->{mmp}</td>";
       
   332 				$failureline .= "<td>$failure->{phase}</td>";
       
   333 				$failureline .= "<td>$failure->{recipe}</td>";
       
   334 				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";
       
   335 				$failureline .= "</tr>";
       
   336 				print SPECIFIC "$failureline\n";
       
   337 			}
       
   338 			
       
   339 			print SPECIFIC "</table>\n";
       
   340 			print SPECIFIC "<br/>\n";
       
   341 		}
       
   342 	}
       
   343 	
       
   344 	close(SPECIFIC);
       
   345 }
       
   346 
       
   347 sub translate_detail_files_to_html
       
   348 {
       
   349 	opendir(DIR, $raptorbitsdir);
       
   350 	my @failurefiles = readdir(DIR);
       
   351 	closedir(DIR);	
       
   352 	@failurefiles = grep(/\.txt$/, @failurefiles);
       
   353 	
       
   354 	for my $file (@failurefiles)
       
   355 	{
       
   356 		$file =~ /(.*)\.txt$/;
       
   357 		my $filenamebase = $1;
       
   358 		
       
   359 		my $filecontent = '';
       
   360 		open(FILE, "$raptorbitsdir/$file");
       
   361 		{
       
   362 			local $/=undef;
       
   363 			$filecontent = <FILE>;
       
   364 		}
       
   365 		close(FILE);
       
   366 		
       
   367 		$filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g;
       
   368 		$filecontent = "<pre>$filecontent</pre>";
       
   369 		
       
   370 		open(FILE, ">$outputdir/$filenamebase\_failures.html");
       
   371 		print FILE $filecontent;
       
   372 		close(FILE);
       
   373 	}
       
   374 }
       
   375 
       
   376 sub distinct_packages
       
   377 {
       
   378 	my ($allbldinfs) = @_;
       
   379 	
       
   380 	my $allpackages = {};
       
   381 	
       
   382 	for my $bldinf (keys %{$allbldinfs})
       
   383 	{
       
   384 		# normalize bldinf path
       
   385 		$bldinf = lc($bldinf);
       
   386 		$bldinf =~ s,^[A-Za-z]:,,;
       
   387 		$bldinf =~ s,[\\],/,g;
       
   388 		
       
   389 		my $package = '';
       
   390 		if ($bldinf =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)
       
   391 		{
       
   392 			$package = $1;
       
   393 		}
       
   394 		else
       
   395 		{
       
   396 			print "WARNING: can't understand bldinf attribute of recipe: $bldinf. Won't dump to failed recipes file.\n";
       
   397 		}
       
   398 		
       
   399 		$allpackages->{$package} = 1;
       
   400 	}
       
   401 	
       
   402 	return sort {$a cmp $b} keys %{$allpackages};
       
   403 }