|
1 # Copyright (c) 2009 Symbian Foundation Ltd |
|
2 # This component and the accompanying materials are made available |
|
3 # under the terms of the License "Eclipse Public License v1.0" |
|
4 # which accompanies this distribution, and is available |
|
5 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
6 # |
|
7 # Initial Contributors: |
|
8 # Symbian Foundation Ltd - initial contribution. |
|
9 # |
|
10 # Contributors: |
|
11 # |
|
12 # Description: |
|
13 # Unite and HTML-ize Raptor log files |
|
14 |
|
15 use strict; |
|
16 use FindBin; |
|
17 use lib $FindBin::Bin; |
|
18 use RaptorError; |
|
19 use RaptorWarning; |
|
20 use RaptorInfo; |
|
21 use RaptorUnreciped; |
|
22 use RaptorRecipe; |
|
23 |
|
24 use XML::SAX; |
|
25 use RaptorSAXHandler; |
|
26 use Getopt::Long; |
|
27 |
|
28 our $raptorbitsdir = 'raptorbits'; |
|
29 our $basedir = ''; |
|
30 my $outputdir = "html"; |
|
31 our $raptor_config = 'dummy_config'; |
|
32 our $current_log_file = ''; |
|
33 my $help = 0; |
|
34 GetOptions(( |
|
35 'basedir=s' => \$basedir, |
|
36 'help!' => \$help |
|
37 )); |
|
38 my @logfiles = @ARGV; |
|
39 |
|
40 $help = 1 if (!@logfiles); |
|
41 |
|
42 if ($help) |
|
43 { |
|
44 print "Unite and HTML-ize Raptor log files.\n"; |
|
45 print "Usage: perl uh.pl [OPTIONS] FILE1 FILE2 ...\n"; |
|
46 print "where OPTIONS are:\n"; |
|
47 print "\t--basedir=DIR Generate output under DIR (defaults to current dir)\n"; |
|
48 exit(0); |
|
49 } |
|
50 |
|
51 if ($basedir) |
|
52 { |
|
53 $raptorbitsdir = "$basedir/raptorbits"; |
|
54 $outputdir = "$basedir/html"; |
|
55 } |
|
56 mkdir($basedir) if (!-d$basedir); |
|
57 |
|
58 $raptorbitsdir =~ s,/,\\,g; # this is because rmdir doens't cope correctly with the forward slashes |
|
59 |
|
60 system("rmdir /S /Q $raptorbitsdir") if (-d $raptorbitsdir); |
|
61 mkdir($raptorbitsdir); |
|
62 #print "Created dir $raptorbitsdir.\n"; |
|
63 |
|
64 # create empty summary file anyway |
|
65 open(SUMMARY, ">$raptorbitsdir/summary.csv"); |
|
66 close(SUMMARY); |
|
67 |
|
68 my $saxhandler = RaptorSAXHandler->new(); |
|
69 $saxhandler->add_observer('RaptorError', $RaptorError::reset_status); |
|
70 $saxhandler->add_observer('RaptorWarning', $RaptorWarning::reset_status); |
|
71 $saxhandler->add_observer('RaptorInfo', $RaptorInfo::reset_status); |
|
72 $saxhandler->add_observer('RaptorUnreciped', $RaptorUnreciped::reset_status); |
|
73 $saxhandler->add_observer('RaptorRecipe', $RaptorRecipe::reset_status); |
|
74 |
|
75 our $allbldinfs = {}; |
|
76 |
|
77 my $parser = XML::SAX::ParserFactory->parser(Handler=>$saxhandler); |
|
78 for (@logfiles) |
|
79 { |
|
80 print "Reading file: $_\n"; |
|
81 $current_log_file = $_; |
|
82 $parser->parse_uri($_); |
|
83 } |
|
84 |
|
85 my @allpackages = distinct_packages($allbldinfs); |
|
86 |
|
87 print "Generating HTML...\n"; |
|
88 |
|
89 system("rd /S /Q $outputdir") if (-d $outputdir); |
|
90 mkdir ($outputdir); |
|
91 |
|
92 my $raptor_errors = {}; |
|
93 my $raptor_warnings = {}; |
|
94 my $raptor_unreciped = {}; |
|
95 my $general_failures_num_by_severity = {}; |
|
96 my $general_failures_by_category_severity = {}; |
|
97 my $recipe_failures_num_by_severity = {}; |
|
98 my $recipe_failures_by_package_severity = {}; |
|
99 #my $severities = {}; |
|
100 my @severities = ('critical', 'major', 'minor', 'unknown'); |
|
101 |
|
102 # READ SUMMARY.CSV FILE |
|
103 my $csv_file = "$raptorbitsdir/summary.csv"; |
|
104 my $csv_linenum = 0; |
|
105 open(CSV, $csv_file); |
|
106 while(<CSV>) |
|
107 { |
|
108 $csv_linenum ++; |
|
109 my $line = $_; |
|
110 |
|
111 if ($line =~ /([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)/) |
|
112 { |
|
113 my $failure = {}; |
|
114 $failure->{category} = $1; |
|
115 $failure->{subcategory} = $2; |
|
116 $failure->{severity} = $3; |
|
117 $failure->{config} = $4; |
|
118 $failure->{component} = $5; |
|
119 $failure->{mmp} = $6; |
|
120 $failure->{phase} = $7; |
|
121 $failure->{recipe} = $8; |
|
122 $failure->{file} = $9; |
|
123 $failure->{linenum} = $10; |
|
124 |
|
125 my $failure_package = ''; |
|
126 |
|
127 if (!$failure->{category}) |
|
128 { |
|
129 print "WARNING: summary line without a category at $csv_file line $csv_linenum. Skipping\n"; |
|
130 next; |
|
131 } |
|
132 |
|
133 if ($failure->{category} =~ m,^recipe_failure$,i and !$failure->{component}) |
|
134 { |
|
135 print "WARNING: recipe_failure with component field empty at $csv_file line $csv_linenum. Skipping\n"; |
|
136 next; |
|
137 } |
|
138 if ($failure->{component}) |
|
139 { |
|
140 if ($failure->{component} =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),) |
|
141 { |
|
142 $failure_package = $1; |
|
143 } |
|
144 else |
|
145 { |
|
146 print "WARNING: summary line with wrong component path at $csv_file line $csv_linenum. Skipping\n"; |
|
147 next; |
|
148 } |
|
149 } |
|
150 |
|
151 $failure->{subcategory} = 'uncategorized' if (!$failure->{subcategory}); |
|
152 $failure->{severity} = 'unknown' if (!$failure->{severity}); |
|
153 $failure->{mmp} = '-' if (!$failure->{mmp}); |
|
154 |
|
155 # populate severities dynamically. |
|
156 #$severities->{$failure->{severity}} = 1; |
|
157 |
|
158 # put failure items into their category container |
|
159 if ($failure->{category} =~ /^raptor_(error|warning|unreciped)$/i) |
|
160 { |
|
161 $general_failures_num_by_severity->{$failure->{category}} = {} if (!defined $general_failures_num_by_severity->{$failure->{category}}); |
|
162 my $general_failure = $general_failures_num_by_severity->{$failure->{category}}; |
|
163 |
|
164 if (!defined $general_failure->{$failure->{severity}}) |
|
165 { |
|
166 $general_failure->{$failure->{severity}} = 1; |
|
167 } |
|
168 else |
|
169 { |
|
170 $general_failure->{$failure->{severity}} ++; |
|
171 } |
|
172 |
|
173 $general_failures_by_category_severity->{$failure->{category}} = {} if (!defined $general_failures_by_category_severity->{$failure->{category}}); |
|
174 $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}} = [] if (!defined $general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}); |
|
175 push(@{$general_failures_by_category_severity->{$failure->{category}}->{$failure->{severity}}}, $failure); |
|
176 } |
|
177 elsif ($failure->{category} =~ /^recipe_failure$/i) |
|
178 { |
|
179 $recipe_failures_num_by_severity->{$failure_package} = {} if (!defined $recipe_failures_num_by_severity->{$failure_package}); |
|
180 my $package_failure = $recipe_failures_num_by_severity->{$failure_package}; |
|
181 |
|
182 if (!defined $package_failure->{$failure->{severity}}) |
|
183 { |
|
184 $package_failure->{$failure->{severity}} = 1; |
|
185 } |
|
186 else |
|
187 { |
|
188 $package_failure->{$failure->{severity}} ++; |
|
189 } |
|
190 |
|
191 $recipe_failures_by_package_severity->{$failure_package} = {} if (!defined $recipe_failures_by_package_severity->{$failure_package}); |
|
192 $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}} = [] if (!defined $recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}); |
|
193 push(@{$recipe_failures_by_package_severity->{$failure_package}->{$failure->{severity}}}, $failure); |
|
194 } |
|
195 } |
|
196 else |
|
197 { |
|
198 print "WARNING: line does not match expected format at $csv_file line $csv_linenum. Skipping\n"; |
|
199 } |
|
200 } |
|
201 close(CSV); |
|
202 |
|
203 # PRINT HTML SUMMARY |
|
204 my $aggregated_html = "$outputdir/index.html"; |
|
205 open(AGGREGATED, ">$aggregated_html"); |
|
206 print AGGREGATED "RAPTOR BUILD SUMMARY<br/>\n"; |
|
207 |
|
208 print AGGREGATED "<br/>GENERAL FAILURES<br/>\n"; |
|
209 print AGGREGATED "<table border='1'>\n"; |
|
210 my $tableheader = "<tr><th>category</th>"; |
|
211 for (@severities) { $tableheader .= "<th>$_</th>"; } |
|
212 $tableheader .= "</tr>"; |
|
213 print AGGREGATED "$tableheader\n"; |
|
214 for my $category (keys %{$general_failures_num_by_severity}) |
|
215 { |
|
216 print_category_specific_summary($category, $general_failures_by_category_severity->{$category}); |
|
217 my $categoryline = "<tr><td><a href='$category.html'>$category</a></td>"; |
|
218 for (@severities) |
|
219 { |
|
220 my $failuresbyseverity = 0; |
|
221 $failuresbyseverity = $general_failures_num_by_severity->{$category}->{$_} if (defined $general_failures_num_by_severity->{$category}->{$_}); |
|
222 $categoryline .= "<td>$failuresbyseverity</td>"; |
|
223 } |
|
224 $categoryline .= "</tr>"; |
|
225 print AGGREGATED "$categoryline\n"; |
|
226 } |
|
227 print AGGREGATED "</table>\n"; |
|
228 print AGGREGATED "<br/>\n"; |
|
229 |
|
230 print AGGREGATED "<br/>PACKAGE-SPECIFIC FAILURES<br/>\n"; |
|
231 print AGGREGATED "<table border='1'>\n"; |
|
232 $tableheader = "<tr><th>package</th>"; |
|
233 for (@severities) { $tableheader .= "<th>$_</th>"; } |
|
234 $tableheader .= "</tr>"; |
|
235 print AGGREGATED "$tableheader\n"; |
|
236 for my $package (@allpackages) |
|
237 { |
|
238 if (defined $recipe_failures_num_by_severity->{$package}) |
|
239 { |
|
240 print_package_specific_summary($package, $recipe_failures_by_package_severity->{$package}); |
|
241 my $packagesummaryhtml = $package; |
|
242 $packagesummaryhtml =~ s,/,_,; |
|
243 $packagesummaryhtml .= ".html"; |
|
244 my $packageline = "<tr><td><a href='$packagesummaryhtml'>$package</a></td>"; |
|
245 for (@severities) |
|
246 { |
|
247 my $failuresbyseverity = 0; |
|
248 $failuresbyseverity = $recipe_failures_num_by_severity->{$package}->{$_} if (defined $recipe_failures_num_by_severity->{$package}->{$_}); |
|
249 $packageline .= "<td>$failuresbyseverity</td>"; |
|
250 } |
|
251 $packageline .= "</tr>"; |
|
252 print AGGREGATED "$packageline\n"; |
|
253 } |
|
254 else |
|
255 { |
|
256 my $packageline = "<tr><td>$package</td>"; |
|
257 for (@severities) { $packageline .= "<td>0</td>"; } |
|
258 $packageline .= "</tr>"; |
|
259 print AGGREGATED "$packageline\n"; |
|
260 } |
|
261 } |
|
262 print AGGREGATED "</table>\n"; |
|
263 close(AGGREGATED); |
|
264 |
|
265 translate_detail_files_to_html(); |
|
266 |
|
267 print "OK, done. Please open $outputdir/index.html.\n"; |
|
268 |
|
269 |
|
270 sub print_category_specific_summary |
|
271 { |
|
272 my ($category, $failures_by_severity) = @_; |
|
273 |
|
274 my $filenamebase = $category; |
|
275 $filenamebase =~ s,/,_,; |
|
276 |
|
277 open(SPECIFIC, ">$outputdir/$filenamebase.html"); |
|
278 print SPECIFIC "FAILURES FOR CATEGORY $category<br/>\n"; |
|
279 |
|
280 for my $severity (@severities) |
|
281 { |
|
282 if (defined $failures_by_severity->{$severity}) |
|
283 { |
|
284 print SPECIFIC "<br/>".uc($severity)."<br/>\n"; |
|
285 print SPECIFIC "<table border='1'>\n"; |
|
286 # $subcategory, $severity, $mmp, $phase, $recipe, $file, $line |
|
287 my $tableheader = "<tr><th>category</th><th>log file</th><th>log snippet</th></tr>"; |
|
288 print SPECIFIC "$tableheader\n"; |
|
289 |
|
290 for my $failure (@{$failures_by_severity->{$severity}}) |
|
291 { |
|
292 my $failureline = "<tr><td>$failure->{subcategory}</td>"; |
|
293 $failureline .= "<td>$failure->{config}</td>"; |
|
294 $failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>"; |
|
295 $failureline .= "</tr>"; |
|
296 print SPECIFIC "$failureline\n"; |
|
297 } |
|
298 |
|
299 print SPECIFIC "</table>\n"; |
|
300 print SPECIFIC "<br/>\n"; |
|
301 } |
|
302 } |
|
303 |
|
304 close(SPECIFIC); |
|
305 } |
|
306 |
|
307 sub print_package_specific_summary |
|
308 { |
|
309 my ($package, $failures_by_severity) = @_; |
|
310 |
|
311 my $filenamebase = $package; |
|
312 $filenamebase =~ s,/,_,; |
|
313 |
|
314 open(SPECIFIC, ">$outputdir/$filenamebase.html"); |
|
315 print SPECIFIC "FAILURES FOR PACKAGE $package<br/>\n"; |
|
316 |
|
317 for my $severity (@severities) |
|
318 { |
|
319 if (defined $failures_by_severity->{$severity}) |
|
320 { |
|
321 print SPECIFIC "<br/>".uc($severity)."<br/>\n"; |
|
322 print SPECIFIC "<table border='1'>\n"; |
|
323 # $subcategory, $severity, $mmp, $phase, $recipe, $file, $line |
|
324 my $tableheader = "<tr><th>category</th><th>configuration</th><th>mmp</th><th>phase</th><th>recipe</th><th>log snippet</th></tr>"; |
|
325 print SPECIFIC "$tableheader\n"; |
|
326 |
|
327 for my $failure (@{$failures_by_severity->{$severity}}) |
|
328 { |
|
329 my $failureline = "<tr><td>$failure->{subcategory}</td>"; |
|
330 $failureline .= "<td>$failure->{config}</td>"; |
|
331 $failureline .= "<td>$failure->{mmp}</td>"; |
|
332 $failureline .= "<td>$failure->{phase}</td>"; |
|
333 $failureline .= "<td>$failure->{recipe}</td>"; |
|
334 $failureline .= "<td><a href='$filenamebase\_failures.html#failure_item_$failure->{linenum}'>item $failure->{linenum}</a></td>"; |
|
335 $failureline .= "</tr>"; |
|
336 print SPECIFIC "$failureline\n"; |
|
337 } |
|
338 |
|
339 print SPECIFIC "</table>\n"; |
|
340 print SPECIFIC "<br/>\n"; |
|
341 } |
|
342 } |
|
343 |
|
344 close(SPECIFIC); |
|
345 } |
|
346 |
|
347 sub translate_detail_files_to_html |
|
348 { |
|
349 opendir(DIR, $raptorbitsdir); |
|
350 my @failurefiles = readdir(DIR); |
|
351 closedir(DIR); |
|
352 @failurefiles = grep(/\.txt$/, @failurefiles); |
|
353 |
|
354 for my $file (@failurefiles) |
|
355 { |
|
356 $file =~ /(.*)\.txt$/; |
|
357 my $filenamebase = $1; |
|
358 |
|
359 my $filecontent = ''; |
|
360 open(FILE, "$raptorbitsdir/$file"); |
|
361 { |
|
362 local $/=undef; |
|
363 $filecontent = <FILE>; |
|
364 } |
|
365 close(FILE); |
|
366 |
|
367 $filecontent =~ s,---(failure_item_\d+)---,<a name="$1">---$1---</a>,g; |
|
368 $filecontent = "<pre>$filecontent</pre>"; |
|
369 |
|
370 open(FILE, ">$outputdir/$filenamebase\_failures.html"); |
|
371 print FILE $filecontent; |
|
372 close(FILE); |
|
373 } |
|
374 } |
|
375 |
|
376 sub distinct_packages |
|
377 { |
|
378 my ($allbldinfs) = @_; |
|
379 |
|
380 my $allpackages = {}; |
|
381 |
|
382 for my $bldinf (keys %{$allbldinfs}) |
|
383 { |
|
384 # normalize bldinf path |
|
385 $bldinf = lc($bldinf); |
|
386 $bldinf =~ s,^[A-Za-z]:,,; |
|
387 $bldinf =~ s,[\\],/,g; |
|
388 |
|
389 my $package = ''; |
|
390 if ($bldinf =~ m,/((os|mw|app|tools|ostools|adaptation)/[^/]*),) |
|
391 { |
|
392 $package = $1; |
|
393 } |
|
394 else |
|
395 { |
|
396 print "WARNING: can't understand bldinf attribute of recipe: $bldinf. Won't dump to failed recipes file.\n"; |
|
397 } |
|
398 |
|
399 $allpackages->{$package} = 1; |
|
400 } |
|
401 |
|
402 return sort {$a cmp $b} keys %{$allpackages}; |
|
403 } |