1 # Copyright (c) 2009 Symbian Foundation Ltd

     2 # This component and the accompanying materials are made available

     3 # under the terms of the License "Eclipse Public License v1.0"

     4 # which accompanies this distribution, and is available

     5 # at the URL "http://www.eclipse.org/legal/epl-v10.html".

     6 #

     7 # Initial Contributors:

     8 # Symbian Foundation Ltd - initial contribution.

     9 #

    10 # Contributors:

    11 #

    12 # Description:

    13 # Unite and HTML-ize Raptor log files

    14 

    15 use strict;

    16 use FindBin;

    17 use lib $FindBin::Bin;

    18 use RaptorError;

    19 use RaptorWarning;

    20 use RaptorInfo;

    21 use RaptorUnreciped;

    22 use RaptorRecipe;

    23 

    24 use XML::SAX;

    25 use RaptorSAXHandler;

    26 use Getopt::Long;

    27 

    28 our $raptorbitsdir = 'raptorbits';

    29 our $basedir = '';

    30 my $outputdir = "html";

    31 our $raptor_config = 'dummy_config';

    32 our $current_log_file = '';

    33 my $help = 0;

    34 GetOptions((

    35 	'basedir=s' => \$basedir,

    36 	'help!' => \$help

    37 ));

    38 my @logfiles = @ARGV;

    39 

    40 $help = 1 if (!@logfiles);

    41 

    42 if ($help)

    43 {

    44 	print "Unite and HTML-ize Raptor log files.\n";

    45 	print "Usage: perl uh.pl [OPTIONS] FILE1 FILE2 ...\n";

    46 	print "where OPTIONS are:\n";

    47 	print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n";

    48 	exit(0);

    49 }

    50 

    51 if ($basedir)

    52 {

    53 	$raptorbitsdir = "$basedir/raptorbits";

    54 	$outputdir = "$basedir/html";

    55 }

    56 mkdir($basedir) if (!-d$basedir);

    57 

    58 $raptorbitsdir =~ s,/,\\,g; # this is because rmdir doens't cope correctly with the forward slashes

    59 

    60 system("rmdir /S /Q $raptorbitsdir") if (-d $raptorbitsdir);

    61 mkdir($raptorbitsdir);

    62 #print "Created dir $raptorbitsdir.\n";

    63 

    64 # create empty summary file anyway

    65 open(SUMMARY, ">$raptorbitsdir/summary.csv");

    66 close(SUMMARY);

    67 

    68 my $saxhandler = RaptorSAXHandler->new();

    69 $saxhandler->add_observer('RaptorError', $RaptorError::reset_status);

    70 $saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status);

    71 $saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status);

    72 $saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status);

    73 $saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status);

    74 

    75 our $allbldinfs = {};

    76 

    77 my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler);

    78 for (@logfiles)

    79 {

    80 	print "Reading file: $_\n";

    81 	$current_log_file = $_;

    82 	$parser->parse_uri($_);

    83 }

    84 

    85 my @allpackages = distinct_packages($allbldinfs);

    86 

    87 print "Generating HTML...\n";

    88 

    89 system("rd /S /Q $outputdir") if (-d $outputdir);

    90 mkdir ($outputdir);

    91 

    92 my $raptor_errors = {};

    93 my $raptor_warnings = {};

    94 my $raptor_unreciped = {};

    95 my $general_failures_num_by_severity = {};

    96 my $general_failures_by_category_severity = {};

    97 my $recipe_failures_num_by_severity = {};

    98 my $recipe_failures_by_package_severity = {};

    99 #my $severities = {};

   100 my @severities = ('critical', 'major', 'minor', 'unknown');

   101 

   102 # READ SUMMARY.CSV FILE

   103 my $csv_file = "$raptorbitsdir/summary.csv";

   104 my $csv_linenum = 0;

   105 open(CSV, $csv_file);

   106 while(<CSV>)

   107 {

   108 	$csv_linenum ++;

   109 	my $line = $_;

   110 	

   111 	if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/)

   112 	{

   113 		my $failure = {};

   114 		$failure->{category} = $1;

   115 		$failure->{subcategory} = $2;

   116 		$failure->{severity} = $3;

   117 		$failure->{config} = $4;

   118 		$failure->{component} = $5;

   119 		$failure->{mmp} = $6;

   120 		$failure->{phase} = $7;

   121 		$failure->{recipe} = $8;

   122 		$failure->{file} = $9;

   123 		$failure->{linenum} = $10;

   124 		

   125 		my $failure_package = '';

   126 		

   127 		if (!$failure->{category})

   128 		{

   129 			print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n";

   130 			next;

   131 		}

   132 		

   133 		if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component})

   134 		{

   135 			print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n";

   136 			next;

   137 		}

   138 		if ($failure->{component})

   139 		{

   140 			if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)

   141 			{

   142 				$failure_package = $1;

   143 			}

   144 			else

   145 			{

   146 				print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n";

   147 				next;

   148 			}

   149 		}

   150 		

   151 		$failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory});

   152 		$failure->{severity} = 'unknown' if (!$failure->{severity});

   153 		$failure->{mmp} = '-' if (!$failure->{mmp});

   154 		

   155 		# populate severities dynamically.

   156 		#$severities->{$failure->{severity}} = 1;

   157 		

   158 		# put failure items into their category container

   159 		if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i)

   160 		{

   161 			$general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}});

   162 			my $general_failure = $general_failures_num_by_severity->{$failure->{category}};

   163 			

   164 			if (!defined $general_failure->{$failure->{severity}})

   165 			{

   166 				$general_failure->{$failure->{severity}} = 1;

   167 			}

   168 			else

   169 			{

   170 				$general_failure->{$failure->{severity}} ++;

   171 			}

   172 			

   173 			$general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}});

   174 			$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}});

   175 			push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure);

   176 		}

   177 		elsif ($failure->{category} =~ /^recipe_failure$/i)

   178 		{

   179 			$recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package});

   180 			my $package_failure = $recipe_failures_num_by_severity->{$failure_package};

   181 			

   182 			if (!defined $package_failure->{$failure->{severity}})

   183 			{

   184 				$package_failure->{$failure->{severity}} = 1;

   185 			}

   186 			else

   187 			{

   188 				$package_failure->{$failure->{severity}} ++;

   189 			}

   190 			

   191 			$recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package});

   192 			$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}});

   193 			push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure);

   194 		}

   195 	}

   196 	else

   197 	{

   198 		print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n";

   199 	}

   200 }

   201 close(CSV);

   202 

   203 # PRINT HTML SUMMARY

   204 my $aggregated_html = "$outputdir/index.html";

   205 open(AGGREGATED, ">$aggregated_html");

   206 print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n";

   207 

   208 print AGGREGATED "<br/>GENERAL FAILURES<br/>\n";

   209 print AGGREGATED "<table border='1'>\n";

   210 my $tableheader = "<tr><th>category</th>";

   211 for (@severities) { $tableheader .= "<th>$_</th>"; }

   212 $tableheader .= "</tr>";

   213 print AGGREGATED "$tableheader\n";

   214 for my $category (keys %{$general_failures_num_by_severity})

   215 {

   216 	print_category_specific_summary($category, $general_failures_by_category_severity->{$category});

   217 	my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>";

   218 	for (@severities)

   219 	{

   220 		my $failuresbyseverity = 0;

   221 		$failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_});

   222 		$categoryline .= "<td>$failuresbyseverity</td>";

   223 	}

   224 	$categoryline .= "</tr>";

   225 	print AGGREGATED "$categoryline\n";

   226 }

   227 print AGGREGATED "</table>\n";

   228 print AGGREGATED "<br/>\n";

   229 

   230 print AGGREGATED "<br/>PACKAGE-SPECIFIC FAILURES<br/>\n";

   231 print AGGREGATED "<table border='1'>\n";

   232 $tableheader = "<tr><th>package</th>";

   233 for (@severities) { $tableheader .= "<th>$_</th>"; }

   234 $tableheader .= "</tr>";

   235 print AGGREGATED "$tableheader\n";

   236 for my $package (@allpackages)

   237 {

   238 	if (defined $recipe_failures_num_by_severity->{$package})

   239 	{

   240 		print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package});

   241 		my $packagesummaryhtml = $package;

   242 		$packagesummaryhtml =~ s,/,_,;

   243 		$packagesummaryhtml .= ".html";

   244 		my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>";

   245 		for (@severities)

   246 		{

   247 			my $failuresbyseverity = 0;

   248 			$failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_});

   249 			$packageline .= "<td>$failuresbyseverity</td>";

   250 		}

   251 		$packageline .= "</tr>";

   252 		print AGGREGATED "$packageline\n";

   253 	}

   254 	else

   255 	{

   256 		my $packageline = "<tr><td>$package</td>";

   257 		for (@severities) { $packageline .= "<td>0</td>"; }

   258 		$packageline .= "</tr>";

   259 		print AGGREGATED "$packageline\n";

   260 	}

   261 }

   262 print AGGREGATED "</table>\n";

   263 close(AGGREGATED);

   264 

   265 translate_detail_files_to_html();

   266 

   267 print "OK, done. Please open $outputdir/index.html.\n";

   268 

   269 

   270 sub print_category_specific_summary

   271 {

   272 	my ($category, $failures_by_severity) = @_;

   273 	

   274 	my $filenamebase = $category;

   275 	$filenamebase =~ s,/,_,;

   276 	

   277 	open(SPECIFIC, ">$outputdir/$filenamebase.html");

   278 	print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n";

   279 		

   280 	for my $severity (@severities)

   281 	{

   282 		if (defined $failures_by_severity->{$severity})

   283 		{

   284 			print SPECIFIC "<br/>".uc($severity)."<br/>\n";

   285 			print SPECIFIC "<table border='1'>\n";

   286 			# $subcategory, $severity, $mmp, $phase, $recipe, $file, $line

   287 			my $tableheader = "<tr><th>category</th><th>log file</th><th>log snippet</th></tr>";

   288 			print SPECIFIC "$tableheader\n";

   289 			

   290 			for my $failure (@{$failures_by_severity->{$severity}})

   291 			{

   292 				my $failureline = "<tr><td>$failure->{subcategory}</td>";

   293 				$failureline .= "<td>$failure->{config}</td>";

   294 				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";

   295 				$failureline .= "</tr>";

   296 				print SPECIFIC "$failureline\n";

   297 			}

   298 			

   299 			print SPECIFIC "</table>\n";

   300 			print SPECIFIC "<br/>\n";

   301 		}

   302 	}

   303 	

   304 	close(SPECIFIC);

   305 }

   306 

   307 sub print_package_specific_summary

   308 {

   309 	my ($package, $failures_by_severity) = @_;

   310 	

   311 	my $filenamebase = $package;

   312 	$filenamebase =~ s,/,_,;

   313 	

   314 	open(SPECIFIC, ">$outputdir/$filenamebase.html");

   315 	print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n";

   316 		

   317 	for my $severity (@severities)

   318 	{

   319 		if (defined $failures_by_severity->{$severity})

   320 		{

   321 			print SPECIFIC "<br/>".uc($severity)."<br/>\n";

   322 			print SPECIFIC "<table border='1'>\n";

   323 			# $subcategory, $severity, $mmp, $phase, $recipe, $file, $line

   324 			my $tableheader = "<tr><th>category</th><th>configuration</th><th>mmp</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>";

   325 			print SPECIFIC "$tableheader\n";

   326 			

   327 			for my $failure (@{$failures_by_severity->{$severity}})

   328 			{

   329 				my $failureline = "<tr><td>$failure->{subcategory}</td>";

   330 				$failureline .= "<td>$failure->{config}</td>";

   331 				$failureline .= "<td>$failure->{mmp}</td>";

   332 				$failureline .= "<td>$failure->{phase}</td>";

   333 				$failureline .= "<td>$failure->{recipe}</td>";

   334 				$failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>";

   335 				$failureline .= "</tr>";

   336 				print SPECIFIC "$failureline\n";

   337 			}

   338 			

   339 			print SPECIFIC "</table>\n";

   340 			print SPECIFIC "<br/>\n";

   341 		}

   342 	}

   343 	

   344 	close(SPECIFIC);

   345 }

   346 

   347 sub translate_detail_files_to_html

   348 {

   349 	opendir(DIR, $raptorbitsdir);

   350 	my @failurefiles = readdir(DIR);

   351 	closedir(DIR);	

   352 	@failurefiles = grep(/\.txt$/, @failurefiles);

   353 	

   354 	for my $file (@failurefiles)

   355 	{

   356 		$file =~ /(.*)\.txt$/;

   357 		my $filenamebase = $1;

   358 		

   359 		my $filecontent = '';

   360 		open(FILE, "$raptorbitsdir/$file");

   361 		{

   362 			local $/=undef;

   363 			$filecontent = <FILE>;

   364 		}

   365 		close(FILE);

   366 		

   367 		$filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g;

   368 		$filecontent = "<pre>$filecontent</pre>";

   369 		

   370 		open(FILE, ">$outputdir/$filenamebase\_failures.html");

   371 		print FILE $filecontent;

   372 		close(FILE);

   373 	}

   374 }

   375 

   376 sub distinct_packages

   377 {

   378 	my ($allbldinfs) = @_;

   379 	

   380 	my $allpackages = {};

   381 	

   382 	for my $bldinf (keys %{$allbldinfs})

   383 	{

   384 		# normalize bldinf path

   385 		$bldinf = lc($bldinf);

   386 		$bldinf =~ s,^[A-Za-z]:,,;

   387 		$bldinf =~ s,[\\],/,g;

   388 		

   389 		my $package = '';

   390 		if ($bldinf =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),)

   391 		{

   392 			$package = $1;

   393 		}

   394 		else

   395 		{

   396 			print "WARNING: can't understand bldinf attribute of recipe: $bldinf. Won't dump to failed recipes file.\n";

   397 		}

   398 		

   399 		$allpackages->{$package} = 1;

   400 	}

   401 	

   402 	return sort {$a cmp $b} keys %{$allpackages};

   403 }