|
1 # Copyright (c) 2010 Symbian Foundation Ltd |
|
2 # This component and the accompanying materials are made available |
|
3 # under the terms of the License "Eclipse Public License v1.0" |
|
4 # which accompanies this distribution, and is available |
|
5 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
6 # |
|
7 # Script name: tests_results_statistics_extraction.pl |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Symbian Foundation Ltd - initial contribution. |
|
11 # |
|
12 # Contributors: |
|
13 # Arnaud Lenoir <arnaudl@symbian.org> |
|
14 # |
|
15 # Created 18/01/2010 |
|
16 # |
|
17 # Description: |
|
18 # Allow the extraction of the statistics (total, passed, failed, etc) for Syborg (QEMU) tests |
|
19 # |
|
20 # |
|
21 # Parameters passed: |
|
22 # Param1: Log file containing all the information collected during the Syborg tests |
|
23 # Param2: File containing the values to look for. DO WE NEED TO DO IT THAT WAY? COULD BE NICE TO MAKE THAT TOOLS GENERIC |
|
24 # Param3: NA for the moment |
|
25 |
|
26 # Modules necessary to run this script |
|
27 use Getopt::Long; |
|
28 use strict; |
|
29 use Data::Dumper; |
|
30 |
|
31 # Variables for parameters |
|
32 my $help; |
|
33 my $file_to_analyse; |
|
34 |
|
35 # variables |
|
36 my $system_cmd; # Used to execute DOS commands |
|
37 my $results_destination_file; |
|
38 my $full_list_destination_file; |
|
39 my $total_nb_of_tests = 0; # Count the number total of tests |
|
40 my $total_nb_of_passed_tests = 0; # Count the number of tests that have passed |
|
41 my $total_nb_of_failed_tests = 0; # Count the number of tests that have failed |
|
42 my $total_nb_of_error_tests = 0; # Count the number of tests that have failed due to an error |
|
43 my $total_nb_of_timeout_tests = 0; # Count the number of tests that have failed due to a timeout |
|
44 my $total_nb_of_unknown_tests_results = 0; # If ever one of the string we are looking for is not reported |
|
45 my @list_unknown_tests_results; # If ever one of the string we are looking for is not reported, then save it and display it later on in the file we are saving |
|
46 # Magic formula: $total_nb_of_tests = $total_nb_of_passed_tests + $total_nb_of_failed_tests + $total_nb_of_error_tests + $total_nb_of_timeout_tests + $total_nb_of_unknown_tests_results; |
|
47 my %list_all_tests_results; # List all the tests that are done and their final status. Use test name as key. |
|
48 my %list_all_errors; # List all the tests that errors of any sort. Use test name as key. |
|
49 |
|
50 # Constantes declaration |
|
51 my $results_destination_extension_file = "_statistics\.csv"; |
|
52 my $full_list_destination_extension_file = "_full_list\.csv"; |
|
53 # Strings to look for in the file to get statistics |
|
54 my $running_test_string = "RUNTESTS: Test "; |
|
55 my $passed_test_string = "OK"; |
|
56 my $failed_test_string = "FAIL"; |
|
57 my $error_test_string = " ERROR"; |
|
58 my $timeout_test_string = "TIMEOUT"; |
|
59 # Strings used to extract data for each tests (name, status, error) |
|
60 my $search4test_name_string = $running_test_string."([\\w._-]*[\.]+[\\w]*)\\s"; |
|
61 #print "search4test_name_string = <$search4test_name_string>\n"; |
|
62 my $search4failling_error_string = $failed_test_string."\\s[-]+\\s([\\w\\s,]*)"; |
|
63 #print "search4failling_error_string = <$search4failling_error_string>\n"; |
|
64 my $search4error_string = $error_test_string."\\s([\\w\\s,-]*)"; |
|
65 #print "search4error_string = <$search4error_string>\n"; |
|
66 |
|
67 |
|
68 my $total_nb_tests_save = "Total nb of tests"; |
|
69 my $total_nb_tests_passed_save = "Total nb of tests passed"; |
|
70 my $total_nb_tests_failed_save = "Total nb of tests failed"; |
|
71 my $total_nb_tests_error_save = "Total nb of tests failed due to an error"; |
|
72 my $total_nb_tests_timeout_save = "Total nb of tests failed on timeout"; |
|
73 my $total_nb_tests_unknown_save = "Total nb of tests unknown"; |
|
74 my $check_result_ok = "Check that total nb of tests Vs results is CORRECT?"; |
|
75 my $display_analyse_and_update_script = "Analyse the following data and uppdate script accordingly!!!!!!"; |
|
76 my $display_total_nb_of_unknown_tests_results = "Here is the number of unknown tests results:"; |
|
77 my $display_unknown_tests_results = "Here is the list of unknown tests results lines that need to be analysed:"; |
|
78 my $test_name_column_title = "Test Name"; |
|
79 my $test_status_column_title = "Test Status"; |
|
80 my $test_error_column_title = "What Error?"; |
|
81 |
|
82 |
|
83 # Check if some arguments have been passed to the script |
|
84 my $nb_arg_passed = scalar(@ARGV); |
|
85 if(!$nb_arg_passed) |
|
86 { |
|
87 print "No arguments/parameters passed then print help!\n"; |
|
88 helpme(); |
|
89 } |
|
90 |
|
91 # Retrieve the parameters passed to the script |
|
92 my %optmap = ( 'file_to_analyse' => \$file_to_analyse, |
|
93 'help' => \$help); |
|
94 |
|
95 GetOptions(\%optmap, |
|
96 'file_to_analyse=s', |
|
97 'help'); |
|
98 |
|
99 print "The value for help is: <$help>\n"; |
|
100 print "file_to_analyse: <$file_to_analyse>\n"; |
|
101 |
|
102 if ($help) |
|
103 { |
|
104 print "The user required the display of help\n"; |
|
105 helpme(); |
|
106 } |
|
107 |
|
108 if (!defined($file_to_analyse)) |
|
109 { |
|
110 print "No files has been passed as parameter!!!\n"; |
|
111 helpme(); |
|
112 } |
|
113 |
|
114 # Check if we need the file to examine exists or not |
|
115 if (-f $file_to_analyse) |
|
116 { |
|
117 print "<$file_to_analyse> exists!!!\n"; |
|
118 |
|
119 # Define the name of the ouptut file that will contain the results of our analysis |
|
120 $results_destination_file = "$file_to_analyse" . "$results_destination_extension_file"; |
|
121 print "The path and file where the statistics of this script will be saved is: <$results_destination_file>\n"; |
|
122 $full_list_destination_file = "$file_to_analyse" . "$full_list_destination_extension_file"; |
|
123 print "The path and file where all the tests results of this script will be saved is: <$full_list_destination_file>\n"; |
|
124 } |
|
125 else |
|
126 { |
|
127 print "File <$file_to_analyse> doesn't exist!!!\n"; |
|
128 # Exit program because no files to analyse. |
|
129 exit(0); |
|
130 } |
|
131 |
|
132 # Analysis of the file |
|
133 # 1st open file and check line by line. |
|
134 # 2nd record statistics analysis |
|
135 |
|
136 # Example of line we are looking for to analyse: |
|
137 # Name of the test |
|
138 # RUNTESTS: Started test t_lat2.EXE |
|
139 # Result: |
|
140 # RUNTESTS: Test t_lat2.EXE OK |
|
141 # RUNTESTS: Test t_hcr.EXE FAIL - Exit code 2,84,USER |
|
142 # RUNTESTS: Test t_dcnone.EXE ERROR Could not load file, error -1 |
|
143 # RUNTESTS: Test t_scan.EXE TIMEOUT |
|
144 |
|
145 open (INPUTFILE, "$file_to_analyse"); |
|
146 |
|
147 foreach my $local_line (<INPUTFILE>) |
|
148 { |
|
149 chomp($local_line); # Remove trailig \n |
|
150 #print "Line read from the file is: <$local_line>\n"; |
|
151 |
|
152 if ( $local_line =~ /$running_test_string/) |
|
153 { |
|
154 #print "this line indicates that a test has been run\n"; |
|
155 $total_nb_of_tests++; |
|
156 |
|
157 # Extract the name of the test |
|
158 $local_line =~ /$search4test_name_string/; |
|
159 my $test_name = $1; |
|
160 #print "test_name = <$test_name>\n"; |
|
161 |
|
162 if ($local_line =~ /$passed_test_string/) |
|
163 { |
|
164 #print "this line indicates that a test has passed\n"; |
|
165 $total_nb_of_passed_tests++; |
|
166 |
|
167 $list_all_tests_results {"$test_name"} = $passed_test_string; |
|
168 #print "list_all_tests_results {$test_name} = <".$list_all_tests_results {"$test_name"}.">\n"; |
|
169 } |
|
170 elsif ($local_line =~ /$failed_test_string/) |
|
171 { |
|
172 #print "this line indicates that a test has failed\n"; |
|
173 $total_nb_of_failed_tests++; |
|
174 |
|
175 $list_all_tests_results {"$test_name"} = $failed_test_string; |
|
176 #print "list_all_tests_results {$test_name} = <".$list_all_tests_results {"$test_name"}.">\n"; |
|
177 |
|
178 # Look for the failling error to save it. |
|
179 $local_line =~ /$search4failling_error_string/; |
|
180 my $test_failling_error = $1; |
|
181 $test_failling_error =~ s/,/\;/g; # To remove the "," otherwise this is not good for the writting of the data in the csv file |
|
182 #print "test_failling_error = <$test_failling_error>\n"; |
|
183 $list_all_errors {"$test_name"} = $test_failling_error; |
|
184 #print "list_all_errors {$test_name} = <".$list_all_errors {"$test_name"}.">\n"; |
|
185 } |
|
186 elsif ($local_line =~ /$error_test_string/) |
|
187 { |
|
188 #print "this line indicates that a test has failed due to an error\n"; |
|
189 $total_nb_of_error_tests++; |
|
190 |
|
191 $list_all_tests_results {"$test_name"} = $error_test_string; |
|
192 #print "list_all_tests_results {$test_name} = <".$list_all_tests_results {"$test_name"}.">\n"; |
|
193 |
|
194 # Look for the error, which make the test to fail |
|
195 $local_line =~ /$search4error_string/; |
|
196 my $test_error = $1; |
|
197 $test_error =~ s/,/\;/g; # To remove the "," otherwise this is not good for the writting of the data in the csv file |
|
198 #print "test_error = <$test_error>\n"; |
|
199 $list_all_errors {"$test_name"} = $test_error; |
|
200 #print "list_all_errors {$test_name} = <".$list_all_errors {"$test_name"}.">\n"; |
|
201 } |
|
202 elsif ($local_line =~ /$timeout_test_string/) |
|
203 { |
|
204 #print "this line indicates that a test has failed on timeout\n"; |
|
205 $total_nb_of_timeout_tests++; |
|
206 |
|
207 $list_all_tests_results {"$test_name"} = $timeout_test_string; |
|
208 #print "list_all_tests_results {$test_name} = <".$list_all_tests_results {"$test_name"}.">\n"; |
|
209 } |
|
210 else |
|
211 { |
|
212 print "\n"; |
|
213 chomp($local_line); |
|
214 print "This line contains a result that is unknown to us\n"; |
|
215 print "Please analyse this line and add the appropriate test to solve the problem: <$local_line>\n"; |
|
216 push(@list_unknown_tests_results, $local_line); |
|
217 } |
|
218 } |
|
219 else |
|
220 { |
|
221 #print "This line is of no interest to us!\n"; |
|
222 } |
|
223 } |
|
224 |
|
225 print "\n"; |
|
226 |
|
227 close (INPUTFILE); |
|
228 |
|
229 $total_nb_of_unknown_tests_results = scalar(@list_unknown_tests_results); |
|
230 |
|
231 # Result section |
|
232 print "Here are the values obtained:\n"; |
|
233 print "total_nb_of_tests = <$total_nb_of_tests>\n"; |
|
234 print "total_nb_of_passed_tests = <$total_nb_of_passed_tests>\n"; |
|
235 print "total_nb_of_failed_tests = <$total_nb_of_failed_tests>\n"; |
|
236 print "total_nb_of_error_tests = <$total_nb_of_error_tests>\n"; |
|
237 print "total_nb_of_timeout_tests = <$total_nb_of_timeout_tests>\n"; |
|
238 print "total_nb_of_unknown_tests_results = <$total_nb_of_unknown_tests_results>\n"; |
|
239 if ($total_nb_of_unknown_tests_results) |
|
240 { |
|
241 print "Here are the lines containing unknown tests results:\n<\n@list_unknown_tests_results>\n"; |
|
242 } |
|
243 |
|
244 print "\n\n"; |
|
245 print "Display tables to populate in the csv file to list all tests\n"; |
|
246 if(%list_all_tests_results) |
|
247 { |
|
248 print Dumper(\%list_all_tests_results); |
|
249 print "\n"; |
|
250 } |
|
251 if(%list_all_errors) |
|
252 { |
|
253 print Dumper(\%list_all_errors); |
|
254 print "\n"; |
|
255 } |
|
256 |
|
257 |
|
258 # List all the tests with their results & errors if some |
|
259 open(OUTPUTFILE,">$full_list_destination_file"); |
|
260 |
|
261 print OUTPUTFILE $test_name_column_title.",".$test_status_column_title.",".$test_error_column_title."\n"; # Working, just easier to read |
|
262 #print OUTPUTFILE "$test_name_column_title,$test_status_column_title,$test_error_column_title\n"; # Working |
|
263 |
|
264 foreach (keys %list_all_tests_results) |
|
265 { |
|
266 my $val = $_; |
|
267 #print "Test Name: <$val>\n"; |
|
268 print OUTPUTFILE "$val,".$list_all_tests_results {"$val"}.",".$list_all_errors {"$val"}."\n"; # Do we need to check if list_all_errors is empty or not? |
|
269 } |
|
270 |
|
271 close (OUTPUTFILE); |
|
272 |
|
273 # Store statistics in the file |
|
274 open(OUTPUTFILE,">$results_destination_file"); |
|
275 |
|
276 print OUTPUTFILE $total_nb_tests_save.",$total_nb_of_tests\n"; |
|
277 print OUTPUTFILE $total_nb_tests_passed_save.",$total_nb_of_passed_tests\n"; |
|
278 print OUTPUTFILE $total_nb_tests_failed_save.",$total_nb_of_failed_tests\n"; |
|
279 print OUTPUTFILE $total_nb_tests_error_save.",$total_nb_of_error_tests\n"; |
|
280 print OUTPUTFILE $total_nb_tests_timeout_save.",$total_nb_of_timeout_tests\n"; |
|
281 print OUTPUTFILE $total_nb_tests_unknown_save.",$total_nb_of_unknown_tests_results\n"; |
|
282 |
|
283 print OUTPUTFILE "\n"; |
|
284 |
|
285 if ($total_nb_of_tests == ($total_nb_of_passed_tests + $total_nb_of_failed_tests + $total_nb_of_error_tests + $total_nb_of_timeout_tests + $total_nb_of_unknown_tests_results) ) |
|
286 { |
|
287 print OUTPUTFILE $check_result_ok.",YES"."\n"; |
|
288 } |
|
289 else |
|
290 { |
|
291 print OUTPUTFILE $check_result_ok.",NO"."\n"; |
|
292 } |
|
293 |
|
294 if($total_nb_of_unknown_tests_results !=0) |
|
295 { |
|
296 #print "\nThere are tests that have not been counted because not in the list of things we are looking for, please analyse file: <$results_destination_file>!\n"; |
|
297 |
|
298 print OUTPUTFILE "\n\n"; |
|
299 print OUTPUTFILE $display_analyse_and_update_script; |
|
300 print OUTPUTFILE "\n"; |
|
301 print OUTPUTFILE $display_total_nb_of_unknown_tests_results.",".$total_nb_of_unknown_tests_results."\n"; |
|
302 print OUTPUTFILE "\n"; |
|
303 print OUTPUTFILE $display_unknown_tests_results."\n"; |
|
304 foreach (@list_unknown_tests_results) |
|
305 { |
|
306 print OUTPUTFILE $_."\n"; |
|
307 } |
|
308 } |
|
309 close (OUTPUTFILE); |
|
310 |
|
311 print "\n\nHere is your output file: <$results_destination_file> !!!!!\n\n"; |
|
312 |
|
313 # End of the program!!! |
|
314 exit(0); |
|
315 |
|
316 |
|
317 # Functions section |
|
318 |
|
319 # Help |
|
320 sub helpme |
|
321 { |
|
322 print "\nfct: helpme\n"; |
|
323 print "tests_results_statistics_extraction.pl\n"; |
|
324 print "Usage:\n"; |
|
325 print "tests_results_statistics_extraction.pl --help\n"; |
|
326 print "tests_results_statistics_extraction.pl --file_to_analyse=c:\\temp\\f32tests_log.txt\n"; |
|
327 print "Options:\n"; |
|
328 print "--help:\t\t\t\t display this help and exit\n"; |
|
329 print "--file_to_analyse:\t\t is the path to the directory containing the package_definition.xml file\n"; |
|
330 |
|
331 exit(0); |
|
332 } |
|
333 |
|
334 |