1 # Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

     2 # All rights reserved.

     3 # This component and the accompanying materials are made available

     4 # under the terms of "Eclipse Public License v1.0"

     5 # which accompanies this distribution, and is available

     6 # at the URL "http://www.eclipse.org/legal/epl-v10.html".

     7 #

     8 # Initial Contributors:

     9 # Nokia Corporation - initial contribution.

    10 #

    11 # Contributors:

    12 #

    13 # Description:

    14 #

    15 

    16 use strict;

    17 use LWP::Simple;

    18 use TestScriptResults;

    19 use Getopt::Std;

    20 use File::Find;

    21 

    22 # Mistral tags the result files from test scripts in a particular format, depending on the test harness used to execute them. 

    23 # We use these names to identify which files are results files.

    24 my $KMistralCoreConfFileName = "-coreconf-";

    25 my $KMistralTEFFileName = "-testexecute-";

    26 my $KMistralTestFrameworkFileName = "-testframework"; # No trailing hyphen due to different security variants

    27 

    28 #  It appears that Mistral uses a completely different file format for results from HW runs

    29 my $KMistrealTEFFileNameHW = "testexecute";

    30 my $KMistrealTFFileNameHW1 = "testframework";

    31 my $KMistrealTFFileNameHW2 = "testframeworkmmddcap";

    32 my $KMistrealTFFileNameHW3 = "testframeworkuecap";

    33 my $KMistrealTFFileNameHW4 = "testframeworknone";

    34 

    35 # Work in progress script outputs.

    36 my $iKnownFails;

    37 my $iUnknownFailsButKnownInRef;

    38 my $iUnknownFailsButKnownInOtherCodelines;

    39 my $iUnknownFails;

    40 my $iMissingTests; 	

    41 

    42 my $iVerbose;

    43 

    44 ###############################################

    45 ##	Source test domain specific functionality

    46 ###############################################

    47 # Scans the test results summary page from the Mistral build and compiles a collection of the 

    48 # URL paths for the log files.

    49 # param1 - the URL to the Test Results HTML Summary page of the Mistral build

    50 # returns - the array of logs' URLS

    51 sub GetResultFilesFromMistral

    52 	{

    53 	my $aMistralTestResultsSummaryUrl = shift;

    54 

    55 	my @fileNames;

    56 	my $nextFileNamesIndex = 0;

    57 

    58 	my $summaryContents = get "$aMistralTestResultsSummaryUrl"; # Returns undef if failure

    59 	unless ($summaryContents)

    60 		{

    61 		print "\nERROR: Unable to retrieve source summary file from $aMistralTestResultsSummaryUrl\n";

    62 		exit;

    63 		}

    64 	

    65 	#Could add an optimisation to this to search for the <td class="error"> tag BUT doesn't take into account the crashed tests and would need to check here the pass/fail count

    66 	#In addition this might lead to migration problems with Sirroco and certainly isn't appliable to developer's personal builds where we must scan each file

    67 	#So it's simpler to do the brute force approach and keep all implementations alike, it doesn't take long anyway

    68 	while ($summaryContents =~ s/href="(.*?($KMistralCoreConfFileName|$KMistralTEFFileName|$KMistralTestFrameworkFileName).*?\.html?)"//i)

    69 		{

    70 		$fileNames[$nextFileNamesIndex] = $1;

    71 		$nextFileNamesIndex++;

    72 		}

    73 

    74 	# Scan this second so we don't accidentally cut off the filenames if they are matched against Winscw runs

    75 	while ($summaryContents =~ s/href="(.*?($KMistrealTEFFileNameHW|$KMistrealTFFileNameHW1|$KMistrealTFFileNameHW2|$KMistrealTFFileNameHW3|$KMistrealTFFileNameHW4).*?\.html?)"//i)

    76 		{

    77 		$fileNames[$nextFileNamesIndex] = $1;

    78 		$nextFileNamesIndex++;

    79 		}

    80 	

    81 	unless ($fileNames[0])

    82 		{

    83 		print "\nERROR: No test result log files found, please ensure the -s parameter points to Mistral's TestResults.html summary page\n";	

    84 		exit;

    85 		}

    86 		

    87 	return @fileNames;

    88 	}

    89 

    90 # Parse the Mistral test results summary page and generate an array of TestScriptResults

    91 # objects which contain all the required information such as script names, test counts

    92 # and failures.

    93 # param1 - the URL of the Test Results HTML Summary page for the Mistral build 

    94 # returns - the array of parsed log data

    95 sub PopulateResultsArrayFromMistral

    96 	{

    97 	my $aTestResultsSummaryPageUrl = shift;

    98 	my @aResultsArray = ();

    99 	my $nextFreeIndexInResultsArray = 0;

   100 	

   101 	my @fileNames = GetResultFilesFromMistral($aTestResultsSummaryPageUrl);

   102 	foreach my $fileName (@fileNames)

   103 		{

   104 		my $testFileContents = get "$fileName";

   105 		unless ($testFileContents)

   106 			{

   107 			print "\nERROR: Unable to open logfile $fileName\n";

   108 			next;

   109 			}

   110 			

   111 		my $scriptResults = TestScriptResults->TestScriptResults();

   112 		$scriptResults->SetFilePath($fileName);		

   113 		if (($fileName =~ m/.*?$KMistralTEFFileName(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTEFFileNameHW(.*?)\.html?/i))

   114 			{

   115 			$scriptResults->SetScriptName($1);

   116 			if (!($testFileContents =~ m/TEST CASE SUMMARY:/i) && ($testFileContents =~ m/SUMMARY:/i))

   117 				{

   118 				# One of the old MM TEF tests which didn't use testcases

   119 				$scriptResults->TEFNoTestcasesTest();

   120 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 0);

   121 				}

   122 			else

   123 				{

   124 				$scriptResults->TEFTest();

   125 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 1);

   126 				}	

   127 			}

   128 		elsif ($fileName =~ m/.*?$KMistralCoreConfFileName(.*?)\.txt\.html?/i)

   129 			{

   130 			$scriptResults->SetScriptName($1);

   131 			$scriptResults->CoreConfTest();

   132 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanCoreConfLogFileForFailures($scriptResults, \$testFileContents);

   133 			}

   134 		elsif (($fileName =~ m/.*?$KMistralTestFrameworkFileName[^\-]*?\-(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW2(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW3(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW4(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW1(.*?)\.html?/i))

   135 			{

   136 			$scriptResults->SetScriptName($1);

   137 			$scriptResults->TestFrameworkTest();

   138 			

   139 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanTestFrameworkLogFileForFailures($scriptResults, \$testFileContents);

   140 			}

   141 		else

   142 			{

   143 			print "\nWARNING: Results file has unrecognised format - $fileName.\n";			

   144 			}

   145 		$nextFreeIndexInResultsArray++;	

   146 		}

   147 	

   148 	return \@aResultsArray;

   149 	}

   150 	

   151 

   152 # Walk the file path provided by the developer for his local machine, parse test logs

   153 #  and generate an array of TestScriptResults objects which contain all the required 

   154 # information such as script names, test counts and failures.

   155 # param1 - the pathname of a folder containing the test results

   156 # returns - the array of parsed log data

   157 sub PopulateResultsArrayFromLocalMachine

   158 	{

   159 	my $aTestResultsPath = shift;

   160 	my @aResultsArray = ();

   161 	my $nextFreeIndexInResultsArray = 0;

   162 

   163 	my @directories = ("$aTestResultsPath");

   164 	my @fileNames;

   165 	find( sub { push @fileNames, $File::Find::name if /\.html?$/ }, @directories );

   166 

   167 	foreach my $fileName (@fileNames)

   168 		{

   169 		my $testFileContents = do { local $/; open(I,"$fileName"); <I> };

   170 		my $scriptResults = TestScriptResults->TestScriptResults();

   171 		$scriptResults->SetFilePath($fileName);

   172 		$fileName =~ m/([^\\\/\.]*)\.[^\\\/]*\Z/;

   173 		$scriptResults->SetScriptName($1);

   174 		

   175 		if ($testFileContents =~ m/Core Loader Conformance Suite/)

   176 			{

   177 			$scriptResults->CoreConfTest();

   178 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanCoreConfLogFileForFailures($scriptResults, \$testFileContents);

   179 			}

   180 		elsif ($testFileContents =~ m/TestFrameworkMain.cpp/)

   181 			{

   182 			$scriptResults->TestFrameworkTest();

   183 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanTestFrameworkLogFileForFailures($scriptResults, \$testFileContents);

   184 			}

   185 		elsif ($testFileContents =~ m/TEF Version/)

   186 			{

   187 			if ($testFileContents =~ m/TEST CASE SUMMARY:/i)

   188 				{

   189 				$scriptResults->TEFTest();

   190 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 1);

   191 				}

   192 			else

   193 				{

   194 				# One of the old MM TEF tests which didn't use testcases

   195 				$scriptResults->TEFNoTestcasesTest();

   196 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 0);

   197 				}			

   198 			}

   199 		else

   200 			{

   201 			print "\nWARNING: Results file has unrecognised format - $fileName.\n";	

   202 			}

   203 					

   204 		$nextFreeIndexInResultsArray++;	

   205 		}

   206 	

   207 	return \@aResultsArray;

   208 	}

   209 	

   210 ###############################################

   211 ##	Test harness specific functionality

   212 ###############################################	

   213 

   214 # Parses a TF log file for failures, inconclusives and crashes

   215 # param1 - the TestScriptResults object to populate

   216 # param2 - reference to the contents of the log file

   217 # returns - the TestScriptResults object containing the parsed data

   218 sub ScanTestFrameworkLogFileForFailures

   219 	{

   220 	my ($aScriptResults, $aLogFileContents) = @_;

   221 	

   222 	my $testFileContents = $$aLogFileContents;

   223 	my $numberOfTests = 0;

   224 	# Search for test case results, which take the following form:

   225 	#04/09/2002 15:51:39:772    V   Log.cpp 736 Test Result for TSI_ICL_BTRANS_01:MM_ICL_BTRANS_I_0202_CP is PASS

   226 	while ($testFileContents =~ s/Test[\s]+Result[\s]+for[\s]+[\S]+:([\_|\-|\S]+)[\s]+is[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   227 		{

   228 		my $testId  = $1;

   229 	    my $result = $2;

   230 	    $numberOfTests++;

   231 		if ($result =~ /PASS/)

   232 			{

   233 			# Do nothing

   234 			}

   235 		elsif ($result =~ /INCONCLUSIVE/)

   236 			{

   237 			$aScriptResults->AddInconclusiveResult($testId);

   238 			}

   239 		else

   240 			{

   241 			# Treat all other results as failure

   242 			$aScriptResults->AddFailureResult($testId);

   243 			}

   244 		}

   245 	$aScriptResults->SetTotalTestCount($numberOfTests);

   246 	

   247 	unless ($testFileContents =~ m/Test Results Summary/)

   248 		{

   249 		# Test file summary not in the log file - the test has crashed

   250 		$aScriptResults->TestCrashed();

   251 		}

   252 	

   253 	return $aScriptResults;

   254 	}

   255 

   256 # Parses a TEF log file for failures, inconclusives and crashes

   257 # param1 - the TestScriptResults object to populate

   258 # param2 - reference to the contents of the log file

   259 # param3 - boolean whether the test is using testcases (older MM tests didn't, in which case we use the test steps)

   260 # returns - the TestScriptResults object containing the parsed data

   261 sub ScanTEFLogFileForFailures

   262 	{

   263 	my ($aScriptResults, $aLogFileContents, $aTestcasesInUse) = @_;

   264 	

   265 	my $testFileContents = $$aLogFileContents;

   266 	my $numberOfTests = 0;

   267 

   268 	# Search for test case results, which take the following form:

   269 	# 02:56:42:145 c:\mm\tsu_3gp_compose_api_te.script Line = 58 Command = END_TESTCASE MM-3GP-COMP-U-0003-CP ***TestCaseResult = PASS 

   270 	# Or if testcases not in use in this script file test step results, which take the following form:

   271 	# 14:20:51:459 c:\mm\tsu_mmf_aclnt_securedrmtestdata.script Line = 36 Command = RUN_TEST_STEP 1000 RTAUtils ImportDrmArchive c:\mm\tsu_mmf_aclnt_securedrmtestdata.ini ImportSmallWavDrmArchiveAudio ***Result = UNEXECUTED 

   272 	while (($testFileContents =~ s/Command[\s]+=[\s]+END\_TESTCASE[\s]+([\_|\-|\S]+)[\s]+\*\*\*TestCaseResult[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   273 			|| (!$aTestcasesInUse && ($testFileContents =~ s/Command[\s]+=[\s]+RUN\_TEST\_STEP[\s]+\d*[\s]+.*?[\s]+([\_|\-|\S]+)[\s]+.*?\*\*\*Result[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)))

   274 		{

   275 		my $testId  = $1;

   276 	    my $result = $2;

   277 	    $numberOfTests++;

   278 		if ($result =~ /PASS/)

   279 			{

   280 			# Do nothing

   281 			}

   282 		elsif ($result =~ /INCONCLUSIVE/)

   283 			{

   284 			$aScriptResults->AddInconclusiveResult($testId);

   285 			}

   286 		else

   287 			{

   288 			# Treat all other results as failure

   289 			$aScriptResults->AddFailureResult($testId);

   290 			}

   291 		}

   292 	$aScriptResults->SetTotalTestCount($numberOfTests);

   293 	

   294 	# Testcase scripts use TEST CASE SUMMARY, non-testcase scripts just use SUMMARY

   295 	unless ($testFileContents =~ m/SUMMARY:/)

   296 		{

   297 		# Test file summary not in the log file - the test has crashed

   298 		$aScriptResults->TestCrashed();

   299 		}

   300 	

   301 	return $aScriptResults;	

   302 	}

   303 

   304 # Parses a CoreConf log file for failures, inconclusives and crashes

   305 # param1 - the TestScriptResults object to populate

   306 # param2 - reference to the contents of the log file

   307 # returns - the TestScriptResults object containing the parsed data	

   308 sub ScanCoreConfLogFileForFailures

   309 	{

   310 	my ($aScriptResults, $aLogFileContents) = @_;

   311 	my $testFileContents = $$aLogFileContents;

   312 	

   313 	if ($testFileContents =~ s/\*\*\*\s*?Summary of tests executed(.*\n)*//)

   314 		{

   315 		my $numberOfTests = 0;

   316 

   317 		# Parse the summary listings

   318 		while($testFileContents =~ s/\*\*\*\s*?Passed tests:((.*\n)*?)\*\*\*\s*?Failed tests:.*\n((\*\*\*.*\n)*)//)

   319 			{

   320 			my $passingTests = $1;

   321 			my $failingTests = $3;

   322 			

   323 			# Passing tests

   324 			while ($passingTests =~ s/\*\*\*[\s]*?\S+[\s]*\n//)

   325 				{

   326 				$numberOfTests++;

   327 				}

   328 			

   329 			# Failing tests

   330 			while ($failingTests =~ s/\*\*\*[\s]*?(\S+)[\s]*\n//)

   331 				{

   332 				$aScriptResults->AddFailureResult($1);

   333 				$numberOfTests++;

   334 				}			

   335 			}

   336 

   337 		$aScriptResults->SetTotalTestCount($numberOfTests);		

   338 		}

   339 	else

   340 		{

   341 		# Test file summary not in the log file - the test has crashed

   342 		$aScriptResults->TestCrashed();		

   343 		}

   344 	

   345 	return $aScriptResults;		

   346 	}

   347 

   348 # Gets the test case count from a Test Execute Framework log file

   349 # param1 - a reference to the contents of the TEF log file

   350 #returns - The number of test cases in the script or -1 if the summary could not be found

   351 sub GetTEFTestCount

   352 	{

   353 	my $aReferenceLogContents = shift;

   354 	my $refContents = $$aReferenceLogContents;

   355 

   356 	if ($refContents =~ m/TEST CASE SUMMARY:(.*\n)*?.*?PASS =\s*(\d*)(.*\n)*?.*?FAIL =\s*(\d*)(.*\n)*?.*?INCONCLUSIVE =\s*(\d*)/)

   357 		{

   358 		my $result = $2 + $4 + $6;

   359 		return $result;

   360 		}

   361 	

   362 	if ($refContents =~ m/SUMMARY:(.*\n)*?.*?PASS =\s*(\d*)(.*\n)*?.*?FAIL =\s*(\d*)(.*\n)*?.*?ABORT =\s*(\d*)(.*\n)*?.*?PANIC =\s*(\d*)(.*\n)*?.*?INCONCLUSIVE =\s*(\d*)(.*\n)*?.*?UNKNOWN =\s*(\d*)(.*\n)*?.*?UNEXECUTED =\s*(\d*)/)	

   363 		{

   364 		# One of the MM tests that doesn't use testcases

   365 		my $result = $2 + $4 + $6 + $8 + $10 + $12 + $14;

   366 		return $result;

   367 		}									

   368 	

   369 	# Summary not found, we crashed

   370 	return -1;

   371 	}

   372 

   373 # Gets the test count from a Test Framework log file

   374 # param1 - a reference to the contents of the Test Framework log file

   375 #returns - The number of test cases in the script or -1 if the summary could not be found	

   376 sub GetTestFrameworkTestCount

   377 	{

   378 	my $aReferenceLogContents = shift;

   379 	my $refContents = $$aReferenceLogContents;

   380 	

   381 	unless ($refContents =~ m/Test Results Summary(.*\n)*?.*?Total\s*:(\d*)\s*\n/)

   382 		{

   383 		# Summary not found, we crashed

   384 		return -1;

   385 		}

   386 	return $2;

   387 	}

   388 

   389 # Gets the test count from an OpenMAX IL Core Conformance log file

   390 # param1 - a reference to the contents of the Core Conformance log file

   391 #returns - The number of test cases in the script or -1 if the summary could not be found	

   392 sub GetCoreConfTestCount

   393 	{

   394 	my $aReferenceLogContents = shift;

   395 	my $refContents = $$aReferenceLogContents;

   396 	

   397 	unless ($refContents =~ m/\*\*\*\s*Summary of tests executed(.*\n)*?\*\*\*\s*Total :\s*(\d*)\s*\n/)

   398 		{

   399 		# Summary not found, we crashed

   400 		return -1;

   401 		}

   402 	return $2;

   403 	}

   404 

   405 # Scans a TEF log looking for the test specified and checks if it returned inconclusive

   406 # param1 - the test name

   407 # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref

   408 # param3 - whether the TEF script uses testcases

   409 # returns - 1 if it returned inconclusive, 0 otherwise	

   410 sub MatchingTEFInconclusiveResult	

   411 	{

   412 	my $aInconclusive = shift;

   413 	my $aReferenceLogContents = shift;

   414 	my $aTestcasesInUse = shift;

   415 	my $refContents = $$aReferenceLogContents;

   416 	if ($aTestcasesInUse)

   417 		{

   418 		if ($refContents =~ s/Command[\s]+=[\s]+END\_TESTCASE[\s]+$aInconclusive[\s]+\*\*\*TestCaseResult[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   419 			{

   420 			if ($1 eq "INCONCLUSIVE")

   421 				{

   422 				return 1;

   423 				}

   424 			}

   425 		}

   426 	else

   427 		{

   428 		if ($refContents =~ s/Command[\s]+=[\s]+RUN\_TEST\_STEP[\s]+\d*[\s]+.*?[\s]+$aInconclusive[\s]+.*?\*\*\*Result[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   429 			{

   430 			if ($1 eq "INCONCLUSIVE")

   431 				{

   432 				return 1;

   433 				}

   434 			}

   435 		}

   436 	return 0;

   437 	}

   438 

   439 # Scans a TestFramework log looking for the test specified and checks if it returned inconclusive

   440 # param1 - the test name

   441 # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref

   442 # returns - 1 if it returned inconclusive, 0 otherwise		

   443 sub MatchingTFInconclusiveResult 

   444 	{

   445 	my $aInconclusive = shift;

   446 	my $aReferenceLogContents = shift;

   447 	my $refContents = $$aReferenceLogContents;

   448 	if ($refContents =~ s/Test[\s]+Result[\s]+for[\s]+[\S]+:$aInconclusive[\s]+is[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   449 		{

   450 		if ($1 eq "INCONCLUSIVE")

   451 			{

   452 			return 1;

   453 			}

   454 		}

   455 	return 0;

   456 	}

   457 

   458 # Scans a TEF log looking for the test specified and checks if it returned an error

   459 # param1 - the test name

   460 # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref

   461 # param3 - whether the TEF script uses testcases

   462 # returns - 1 if it returned an error, 0 otherwise		

   463 sub MatchingTEFErrorResult 

   464 	{

   465 	my $aFail = shift;

   466 	my $aReferenceLogContents = shift;

   467 	my $aTestcasesInUse = shift;

   468 	my $refContents = $$aReferenceLogContents;

   469 	

   470 	if ($aTestcasesInUse)

   471 		{

   472 		if (not $refContents =~ s/Command[\s]+=[\s]+END\_TESTCASE[\s]+$aFail[\s]+\*\*\*TestCaseResult[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   473 			{

   474 			return 0;

   475 			}

   476 	

   477 		if (($1 eq "PASS") || ($1 eq "INCONCLUSIVE"))

   478 			{

   479 			return 0;

   480 			}

   481 		}

   482 	else

   483 		{

   484 		if (not $refContents =~ s/Command[\s]+=[\s]+RUN\_TEST\_STEP[\s]+\d*[\s]+.*?[\s]+$aFail[\s]+.*?\*\*\*Result[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   485 			{

   486 			return 0;

   487 			}

   488 	

   489 		if (($1 eq "PASS") || ($1 eq "INCONCLUSIVE"))

   490 			{

   491 			return 0;

   492 			}

   493 		}

   494 		

   495 	return 1;

   496 	}	

   497 	

   498 # Scans a TestFramework log looking for the test specified and checks if it returned an error

   499 # param1 - the test name

   500 # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref

   501 # returns - 1 if it returned an error, 0 otherwise		

   502 sub MatchingTFErrorResult 

   503 	{

   504 	my $aFail = shift;

   505 	my $aReferenceLogContents = shift;

   506 	my $refContents = $$aReferenceLogContents;

   507 	if (not $refContents =~ s/Test[\s]+Result[\s]+for[\s]+[\S]+:$aFail[\s]+is[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)

   508 		{

   509 		return 0;

   510 		}

   511 	

   512 	if (($1 eq "PASS") || ($1 eq "INCONCLUSIVE"))

   513 		{

   514 		return 0;

   515 		}

   516 		

   517 	return 1;

   518 	}	

   519 	

   520 # N.B. VERY IMPORTANT that the log file is passed by reference.  Core conf scripts can include numerous

   521 # different tests that use the same test name so to avoid false positives from this function the 

   522 # substitution done in the comparison MUST affect the caller's copy of the log file.

   523 sub MatchingCoreConfErrorResult

   524 	{	

   525 	my $aFail = shift;

   526 	my $aReferenceLogContents = shift;

   527 	my $refContents = $$aReferenceLogContents;

   528 	

   529 	if ($refContents =~ s/\*\*\*[\s]*$aFail[\s]*FAILED//)

   530 		{

   531 		return 1;

   532 		}

   533 

   534 	return 0;	

   535 	}

   536 	

   537 ###############################################		

   538 ##	Failure comparison functions

   539 ###############################################		

   540 

   541 # The analysis takes an array of TestScriptResults, each storing the results of a test script's log file.

   542 # It processes this list in stages looking for any scripts with errors and trying to match them against

   543 # entries in a Known Failure sheet and/or comparison reference build.  Whenever a match is found

   544 # the error is removed from the TestScriptResults object, these essentially act as an object containing

   545 # unresolved errors.  When errors are matched their details are added to the corresponding text 

   546 # variables at the top of this file (e.g. iKnownFails).

   547 sub AnalyseFailures

   548 	{

   549 	my ($aKFSheetLocation, $aReferenceMistralURL, $aIgnoreFailsNotInRef, $aResultsRef, $aKfPlatforms) = @_;

   550 

   551 	my @results = @$aResultsRef;

   552 	

   553 	my $refContents = undef;

   554 	if ($aReferenceMistralURL)

   555 		{

   556 		$refContents = get "$aReferenceMistralURL"; # Returns undef if failure

   557 		unless ($refContents)

   558 			{

   559 			print "\nERROR: Unable to retrieve reference summary file from $aReferenceMistralURL\n";

   560 			exit;

   561 			}		

   562 		}

   563 	

   564 	my $kfSheetContents = undef;

   565 	if ($aKFSheetLocation)

   566 		{

   567 		unless ($aKFSheetLocation =~ m/\.xml\Z/i)

   568 			{

   569 			print "\nERROR: KF sheet ($aKFSheetLocation) does not appear to be in XML format, you should save the Excel worksheet as XML Spreadsheet 2003 format for input to this script.\n";	

   570 			exit;

   571 			}		

   572 		$kfSheetContents = do { local $/; open(I,"$aKFSheetLocation"); <I> };

   573 		unless ($kfSheetContents)

   574 			{

   575 			print "\nERROR: Unable to open KF sheet at $aKFSheetLocation\n";	

   576 			exit;			

   577 			}

   578 		}

   579 		

   580 	foreach my $scriptResults (@results)

   581 		{

   582 		my $scriptName = $scriptResults->ScriptName();

   583 		

   584 		my $referenceResults = undef;

   585 		

   586 		# If a reference build was provided, find the corresponding log file

   587 		if ($refContents)

   588 			{

   589 			# We use substitution here because some of the coreconf test scripts

   590 			# have the same script name.  This ensures we don't keep referencing

   591 			# against the same copy.  We cannot do the comparison on the script 

   592 			# path because this isn't resilient to adding new tests or cross domain 

   593 			# comparisons.

   594 			# N.B. This assumes the test list ordering remains the same.

   595 			# N.B. The optional .txt is for Core Conf tests

   596 			$refContents =~ s/href="(.*?$scriptName(\.txt)?(\.script)?\.html?)"//;

   597 			my $file = $1;

   598 			if ($file eq undef)

   599 				{

   600 				if ($aIgnoreFailsNotInRef)

   601 					{

   602 					next;

   603 					}

   604 				print "\nWARNING: Unable to find $scriptName in Reference Build\n";	

   605 				}

   606 			else

   607 				{

   608 				$referenceResults = get "$file";	# returns undef on failure

   609 				unless ($referenceResults)

   610 					{

   611 					print "\nWARNING: Unable to open $scriptName ($file) in Reference Build\n";			

   612 					}

   613 				}

   614 			}

   615 		

   616 		my $refTestCrashed;

   617 		# Check the test count hasn't decreased

   618 		if ($referenceResults)

   619 			{

   620 			$refTestCrashed = CheckTestCount($scriptResults, \$referenceResults);

   621 			}

   622 			

   623 		if (not $scriptResults->AnyFailures())

   624 			{

   625 			# No errors so move onto the next

   626 			next;

   627 			}

   628 		

   629 		if ($kfSheetContents)

   630 			{

   631 			ResolveFailuresAgainstKFSheet($scriptResults, \$kfSheetContents, $aKfPlatforms); 

   632 		

   633 			if (not $scriptResults->AnyFailures())

   634 				{

   635 				# All errors resolved against KF sheet so move onto the next

   636 				next;

   637 				}

   638 			}	

   639 		

   640 		if ($referenceResults)

   641 			{

   642 			ResolveFailuresAgainstReferenceRun($scriptResults, $refTestCrashed, \$referenceResults);

   643 			

   644 			if (not $scriptResults->AnyFailures())

   645 				{

   646 				# All errors resolved against reference run so move onto the next

   647 				next;

   648 				}			

   649 			}

   650 		

   651 		# Unknown failures

   652 		$iUnknownFails = $iUnknownFails . "\n$scriptName\n";

   653 		

   654 		my $inconText;

   655 		my @inconclusives = $scriptResults->Inconclusives();

   656 		foreach my $inconclusive (@inconclusives)

   657 			{

   658 			$inconText = $inconText . "$inconclusive\n";

   659 			}

   660 		my $failText;

   661 		my @failures = $scriptResults->Failures();

   662 		foreach my $failure (@failures)

   663 			{

   664 			$failText = $failText . "$failure\n";

   665 			}	

   666 			

   667 		if ($inconText)

   668 			{

   669 			$iUnknownFails = $iUnknownFails . "INCONCLUSIVES:\n$inconText";

   670 			}

   671 		if ($failText)

   672 			{

   673 			$iUnknownFails = $iUnknownFails . "FAILS:\n$failText";

   674 			}

   675 		if ($scriptResults->DidItCrash())	

   676 			{

   677 			$iUnknownFails = $iUnknownFails . "CRASHED\n";

   678 			}

   679 		}	

   680 	

   681 	}

   682 

   683 # Will compare the test count of the test in question against the equivalent result from a reference build's log

   684 # to detect if there has been a reduction in the total number of tests run as part of the script, unless that script 

   685 # has crashed.  As a side effect the return value indicates whether the test in question crashed or not (saves

   686 # having to scan it twice).

   687 # param1 - the log results to compare against the reference build's run

   688 # param2 - the contents of the reference build's corresponding test log

   689 # returns = 1 if the reference script crashed and we were unable to compare the counts, 0 otherwise

   690 sub CheckTestCount

   691 	{

   692 	my ($aTestResultsObject, $aReferenceLogContents) = @_;

   693 	

   694 	my $testHarness = $aTestResultsObject->TestHarness();

   695 	my $refTestCount;

   696 	if (($testHarness eq $TestScriptResults::KTEFTest) || ($testHarness eq $TestScriptResults::KTEFNoTestcasesTest))

   697 		{

   698 		$refTestCount = GetTEFTestCount($aReferenceLogContents);

   699 		}

   700 	elsif ($testHarness eq $TestScriptResults::KTestFrameworkTest)

   701 		{

   702 		$refTestCount = GetTestFrameworkTestCount($aReferenceLogContents);

   703 		}

   704 	else

   705 		{

   706 		$refTestCount = GetCoreConfTestCount($aReferenceLogContents);

   707 		}

   708 	

   709 	if ($refTestCount < 0)

   710 		{

   711 		# Reference test crashed

   712 		unless ($aTestResultsObject->DidItCrash())

   713 			{

   714 			my $scriptName = $aTestResultsObject->ScriptName();

   715 			print "\nWARNING: $scriptName crashed in the reference build, unable to compare test counts\n";

   716 			}

   717 		return 1;

   718 		}

   719 	

   720 	my $testCount = $aTestResultsObject->TotalTestCount;

   721 	if ($testCount < $refTestCount)

   722 		{

   723 		unless ($aTestResultsObject->DidItCrash())

   724 			{

   725 			my $testName = $aTestResultsObject->ScriptName();

   726 			$iMissingTests = $iMissingTests . "$testName  Previous = $refTestCount, Current = $testCount\n"; 	

   727 			}

   728 		}

   729 	return 0;	

   730 	}	

   731 

   732 # Will scan the Known Failure sheet for any entries indicating if the failures,inconclusives and/or crash

   733 # witnessed in the script in question are already known. If so these will be appended to the known failures list

   734 # and removed from the TestScriptResults object in question. If matching entries were found but existed for

   735 # other codelines then this information is stored but the failures are not removed from the TestScriptResults 

   736 # object.

   737 # param1 - TestScriptResults object for the script with failures

   738 # param2 - the contents of the known failure sheet

   739 # param3 - the array of platform entries in the known failure sheet that are applicable to this run

   740 sub ResolveFailuresAgainstKFSheet

   741 	{

   742 	my ($aScriptWithProblems, $aKfSheetContents, $aKfPlatforms) = @_;

   743 	my $kfSheetContents = $$aKfSheetContents;

   744 	my $scriptName = $aScriptWithProblems->ScriptName();

   745 	

   746 	my @kFApplicablePlatforms;

   747 	if ($aKfPlatforms)

   748 		{

   749 		# Will error if undef

   750 		@kFApplicablePlatforms = @$aKfPlatforms;

   751 		}

   752 

   753 	# Modified version of the KF Sheet contents, at the end of this function it will contain the Rows from the KF Sheet with any that correspond to the current test script stripped out.

   754 	# This is a by product of the fact that we inspect each row one at a time looking for any that are relevant, trying to do a global search results in potentially error prone and difficult to maintain

   755 	# regexs.  And in reality underneath Perl shouldn't be doing that much in the way of optimisation versus this approach.

   756 	my $wipKfSheet;

   757 	my $candidatesForOtherPlatforms;

   758 	my $foundSamePlatFails;

   759 	my $foundSamePlatInconcs;

   760 	my $foundSamePlatCrash;

   761 	while ($kfSheetContents =~ s/<Row((.|\n)*?>)((.|\n)*?)<\/Row>//i)

   762 		{

   763 		my $row = $3;

   764 		# Optimisation

   765 		unless ($row =~ m/$scriptName/i)

   766 			{

   767 			$wipKfSheet = $wipKfSheet . "<Row>$row<\/Row>";

   768 			next;

   769 			}

   770 		

   771 		# None of the cells prior to the 'Scriptname' named cell are of any interest, now verify this row is actually for 

   772 		# this test script and we didn't pick the hit up off the entries for one of the other fields, e.g. Remarks

   773 		my $applicable;

   774 		while ($row =~ s/(<Cell(.|\n)*?<\/Cell>)//i)

   775 			{

   776 			my $cell = $1;

   777 			if ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Scriptname\d*"\//i)

   778 				{

   779 				# Named field we're looking for

   780 				if ($cell =~ m/$scriptName/i)

   781 					{

   782 					$applicable = 1;

   783 					last;

   784 					}

   785 				else

   786 					{

   787 					$wipKfSheet = $wipKfSheet . "<Row>$cell\n$row<\/Row>";	# This entry isn't applicable to us, reinsert it into the KF sheet

   788 					last;

   789 					}

   790 				}

   791 			}

   792 		unless ($applicable)

   793 			{

   794 			next;

   795 			}

   796 		

   797 		# We are now dealing with a Row corresponding to the script in question

   798 		# So pull out the relevant named cells contents

   799 		my $crash;

   800 		my $fails;

   801 		my $inconclusives;

   802 		my $targets;

   803 		while ($row =~ s/(<Cell(.|\n)*?<\/Cell>)//i)

   804 			{

   805 			my $cell = $1;

   806 			if ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="CrashedStatus\d*"\//i)

   807 				{

   808 				$crash = $cell;

   809 				}

   810 			elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Fails\d*"\//i)

   811 				{

   812 				$fails = $cell;			

   813 				}

   814 			elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Inconclusives\d*"\//i)

   815 				{

   816 				$inconclusives = $cell;			

   817 				}

   818 			elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Platforms\d*"\//i)

   819 				{

   820 				$targets = $cell;			

   821 				}				

   822 			}

   823 			

   824 		my $platformMatched;

   825 		foreach my $platform (@kFApplicablePlatforms)

   826 			{

   827 			unless (($targets =~ m/$platform&#10;/i) || ($targets =~ m/$platform<\/Data>/i))	# To ensure for example that All isn't picked up by an All Winscw entry

   828 				{

   829 				next;	# Platform not found in the entry

   830 				}		

   831 			$platformMatched = 1;

   832 

   833 			if (($aScriptWithProblems->DidItCrash()) && ($crash =~ m/Crashed/i))

   834 				{

   835 				$foundSamePlatCrash = 1;

   836 				$aScriptWithProblems->ResetCrashed();

   837 				}

   838 				

   839 			if ($fails)

   840 				{

   841 				my @failures = $aScriptWithProblems->Failures();

   842 				$aScriptWithProblems->ResetFailures();

   843 				foreach my $failure (@failures)

   844 					{

   845 					if (($fails =~ m/$failure/i) || ($fails =~ m/>ALL<\/Data>/))

   846 						{

   847 						$foundSamePlatFails = $foundSamePlatFails . "$failure\n";

   848 						}

   849 					else

   850 						{

   851 						

   852 						$aScriptWithProblems->AddFailureResult($failure);

   853 						}

   854 					}

   855 				}

   856 

   857 			if ($inconclusives)

   858 				{

   859 				my @incons = $aScriptWithProblems->Inconclusives();

   860 				$aScriptWithProblems->ResetInconclusives();

   861 				foreach my $incon (@incons)

   862 					{

   863 					if (($inconclusives =~ m/$incon/i) || ($inconclusives =~ m/>ALL<\/Data>/))

   864 						{

   865 						$foundSamePlatInconcs = $foundSamePlatInconcs . "$incon\n";

   866 						}

   867 					else

   868 						{

   869 						$aScriptWithProblems->AddInconclusiveResult($incon);

   870 						}

   871 					}

   872 				}

   873 			}	# End of platform matching loop

   874 		

   875 		unless ($platformMatched)

   876 			{

   877 			# The row entry did not match any of the applicable platforms.  We need to keep checking for more appropriate rows.

   878 			# However, if after the KF Sheet has been parsed we still have unknown errors, we need to compare against these rows as well

   879 			# in case the issue has been seen on other platforms.

   880 	

   881 			$candidatesForOtherPlatforms = $candidatesForOtherPlatforms . "<Row>$crash$fails$inconclusives<\/Row>";

   882 			}

   883 			

   884 		}	# End of row scanning

   885 	

   886 

   887 	my $foundOtherPlatFails;

   888 	my $foundOtherPlatInconcs;

   889 	my $foundOtherPlatCrash;	

   890 	if ($aScriptWithProblems->AnyFailures())	

   891 		{

   892 		# Failures remain, potentially matched against the rows for other platforms

   893 		

   894 		while ($candidatesForOtherPlatforms =~ s/<Row>((.|\n)*?)<\/Row>//i)

   895 			{

   896 			my $row = $1;

   897 			my $crash;

   898 			my $fails;

   899 			my $inconclusives;

   900 			while ($row =~ s/(<Cell(.|\n)*?<\/Cell>)//i)

   901 				{

   902 				my $cell = $1;

   903 				if ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="CrashedStatus\d*"\//i)

   904 					{

   905 					$crash = $cell;

   906 					}

   907 				elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Fails\d*"\//i)

   908 					{

   909 					$fails = $cell;

   910 					}

   911 				elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Inconclusives\d*"\//i)

   912 					{

   913 					$inconclusives = $cell;

   914 					}					

   915 				}	# End of stripping out content details

   916 			

   917 			if (($aScriptWithProblems->DidItCrash()) && ($crash =~ m/Crashed<\/Data>/i))

   918 				{

   919 				$foundOtherPlatCrash = 1;

   920 				}

   921 				

   922 			if ($fails)

   923 				{

   924 				my @failures = $aScriptWithProblems->Failures();

   925 				foreach my $failure (@failures)

   926 					{

   927 					if (($fails =~ m/$failure/i) || ($fails =~ m/>ALL<\/Data>/))

   928 						{

   929 						$foundOtherPlatFails = $foundOtherPlatFails . "$failure\n";

   930 						}

   931 					}

   932 				}

   933 

   934 			if ($inconclusives)

   935 				{

   936 				my @incons = $aScriptWithProblems->Inconclusives();

   937 				foreach my $incon (@incons)

   938 					{

   939 					if (($inconclusives =~ m/$incon/i) || ($inconclusives =~ m/>ALL<\/Data>/))

   940 						{

   941 						$foundOtherPlatInconcs = $foundOtherPlatInconcs . "$incon\n";

   942 						}

   943 					}

   944 				}

   945 				

   946 			}	# End of cross-platform candidate matchine

   947 		}	# End of dealing with potential cross-platform matches

   948 

   949 	# Output known failures

   950 	if ($foundSamePlatFails || $foundSamePlatInconcs || $foundSamePlatCrash)

   951 		{

   952 		$iKnownFails = $iKnownFails . "\n$scriptName\n";

   953 		}

   954 	if ($foundSamePlatInconcs)

   955 		{

   956 		$iKnownFails = $iKnownFails . "INCONCLUSIVES:\n$foundSamePlatInconcs";

   957 		}

   958 	if ($foundSamePlatFails)

   959 		{

   960 		$iKnownFails = $iKnownFails . "FAILS:\n$foundSamePlatFails";

   961 		}

   962 	if ($foundSamePlatCrash)

   963 		{

   964 		$iKnownFails = $iKnownFails . "CRASHED\n";

   965 		}		

   966 	

   967 	# Output matches found only in other test platforms

   968 	if ($foundOtherPlatFails || $foundOtherPlatInconcs || $foundOtherPlatCrash)

   969 		{

   970 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines .  "\n$scriptName\n";

   971 		}

   972 	if ($foundOtherPlatInconcs)

   973 		{

   974 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines . "INCONCLUSIVES:\n$foundOtherPlatInconcs";

   975 		}

   976 	if ($foundOtherPlatFails)

   977 		{

   978 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines . "FAILS:\n$foundOtherPlatFails";

   979 		}

   980 	if ($foundOtherPlatCrash)

   981 		{

   982 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines . "CRASHED\n";

   983 		}

   984 		

   985 	$aKfSheetContents = $wipKfSheet;

   986 	}

   987 

   988 

   989 # Takes the corresponding log file from the reference build and test for a failing script and

   990 # compare it to see if the failures match.  If so this information is stored and the matching

   991 # failures removed from the TestScriptResults object.

   992 # param1 - TestScriptResults object for the script with failures

   993 # param2 - boolean whether the test crashed in the reference build and test

   994 # param3 - the contents of the corresponding log file from the reference build

   995 sub ResolveFailuresAgainstReferenceRun

   996 	{

   997 	my $aReferenceTestResults = shift;

   998 	my $aRefTestCrashed = shift;

   999 	my $aReferenceLogContents = shift;

  1000 	my $refContents = $$aReferenceLogContents;	

  1001 

  1002 	my $scriptName = $aReferenceTestResults->ScriptName();

  1003 	my $testHarness = $aReferenceTestResults->TestHarness();

  1004 	

  1005 	my $inconText;

  1006 	my @inconclusives = $aReferenceTestResults->Inconclusives();

  1007 	$aReferenceTestResults->ResetInconclusives();

  1008 	foreach my $inconclusive (@inconclusives)

  1009 		{

  1010 		if (($testHarness eq $TestScriptResults::KTEFTest) || ($testHarness eq $TestScriptResults::KTEFNoTestcasesTest))

  1011 			{

  1012 			my $testcasesInUse = 0;

  1013 			if ($testHarness eq $TestScriptResults::KTEFTest)

  1014 				{

  1015 				$testcasesInUse = 1;

  1016 				}

  1017 			if (MatchingTEFInconclusiveResult($inconclusive, \$refContents, $testcasesInUse))

  1018 				{

  1019 				$inconText = $inconText . "$inconclusive\n";

  1020 				}

  1021 			else

  1022 				{

  1023 				$aReferenceTestResults->AddInconclusiveResult($inconclusive);

  1024 				}

  1025 			}

  1026 		elsif ($testHarness eq $TestScriptResults::KTestFrameworkTest)

  1027 			{

  1028 			if (MatchingTFInconclusiveResult($inconclusive, \$refContents))

  1029 				{

  1030 				$inconText = $inconText . "$inconclusive\n";

  1031 				}

  1032 			else

  1033 				{

  1034 				$aReferenceTestResults->AddInconclusiveResult($inconclusive);			

  1035 				}			

  1036 			}

  1037 		# Core Conf tests have no comprehension of inconclusive

  1038 		}

  1039 			

  1040 	my $failText;

  1041 	my @failures = $aReferenceTestResults->Failures();

  1042 	$aReferenceTestResults->ResetFailures();

  1043 	foreach my $failure (@failures)

  1044 		{

  1045 		if (($testHarness eq $TestScriptResults::KTEFTest) || ($testHarness eq $TestScriptResults::KTEFNoTestcasesTest))

  1046 			{

  1047 			my $testcasesInUse = 0;

  1048 			if ($testHarness eq $TestScriptResults::KTEFTest)

  1049 				{

  1050 				$testcasesInUse = 1;

  1051 				}

  1052 			if (MatchingTEFErrorResult($failure, \$refContents, $testcasesInUse))

  1053 				{

  1054 				$failText = $failText . "$failure\n";

  1055 				}

  1056 			else

  1057 				{

  1058 				$aReferenceTestResults->AddFailureResult($failure);

  1059 				}

  1060 			}

  1061 		elsif ($testHarness eq $TestScriptResults::KTestFrameworkTest)

  1062 			{

  1063 			if (MatchingTFErrorResult($failure, \$refContents))

  1064 				{

  1065 				$failText = $failText . "$failure\n";

  1066 				}

  1067 			else

  1068 				{

  1069 				$aReferenceTestResults->AddFailureResult($failure);

  1070 				}			

  1071 			}

  1072 		else

  1073 			{

  1074 			# Core Conf 

  1075 			if (MatchingCoreConfErrorResult($failure, \$refContents))

  1076 				{

  1077 				$failText = $failText . "$failure\n";

  1078 				}

  1079 			else

  1080 				{

  1081 				$aReferenceTestResults->AddFailureResult($failure);

  1082 				}			

  1083 			}

  1084 		}

  1085 

  1086 	my $bothCrashed;

  1087 	if (($aReferenceTestResults->DidItCrash()) && $aRefTestCrashed)

  1088 		{

  1089 		$bothCrashed = 1;

  1090 		$aReferenceTestResults->ResetCrashed();

  1091 		}

  1092 	

  1093 	if ($inconText || $failText || $bothCrashed)

  1094 		{

  1095 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "\n$scriptName\n";

  1096 		}

  1097 	

  1098 	if 	($inconText)

  1099 		{

  1100 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "INCONCLUSIVES:\n$inconText";

  1101 		}

  1102 	if ($failText)

  1103 		{

  1104 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "FAILS:\n$failText";

  1105 		}

  1106 	if ($bothCrashed)

  1107 		{

  1108 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "CRASHED\n";

  1109 		}

  1110 	}

  1111 	

  1112 ###############################################

  1113 ##	Utility functions

  1114 ###############################################	

  1115 

  1116 sub PrintHelp

  1117 	{

  1118 	print "\n\nResultsComparison.pl -m Mode -s Results_source [-k Known_failure_sheet_path -c Codeline -p Platform] [-r Mistral_comparison_job_url] [-i] [-v]\n\n";

  1119 	print "  -m = Mode of operation, specify the source of the results that you are\n";

  1120 	print "    comparing. Currently supported modes: Mistral, Local\n\n";

  1121 	print "  -s = Source of the results, mode dependant:\n";

  1122 	print "      (i) Mistral = The URL of the test results HTML summary page\n";

  1123 	print "      (ii) Local = The pathname of a folder containing the test log files\n\n";

  1124 	print "  -k = Pathname of the KF sheet, this should be exported as an XML spreadsheet\n";

  1125 	print "    from Excel\n\n";

  1126 	print "  -r = Reference Mistral build, the URL of the test results HTML summary page.\n";

  1127 	print "    This can be used in addition or in place of the Known Failure sheet\n\n";

  1128 	print "  -i = Ignore test failures in scripts not found in the Reference Mistral build\n\n";

  1129 	print "  -c = An index indicating the codeline used in the test run, only used in\n";

  1130 	print "    conjunction with a KF sheet. Possible values: TB92, TB101, TB102\n\n";	

  1131 	print "  -p = An index indicating the platform the test run is from, only used in\n";

  1132 	print "    conjunction with a KF sheet. Possible values: Winscw, H6, H4 WDP,\n";

  1133 	print "    NaviEngine\n\n";

  1134 	print "  -v = Verbose output\n\n";

  1135 	}