sbsv2/raptor/test/common/run_tests.py
author timothy.murphy@nokia.com
Thu, 25 Mar 2010 13:43:28 +0000
branchfix
changeset 408 a819f9223567
parent 135 0092642f198e
permissions -rw-r--r--
fix: stop using "magic" numbers in string operations for the copyannofile2log feature fix: When using the copylogfromannofile workaround, extract the build ID and build duration and add to the log as these are useful for analysis. The log should now be identical to the stdout file. fix: Remove extra blank lines from output in copylogfromannofile mode.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     1
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     2
# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     3
# All rights reserved.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     4
# This component and the accompanying materials are made available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     5
# under the terms of the License "Eclipse Public License v1.0"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     6
# which accompanies this distribution, and is available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     7
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     8
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     9
# Initial Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    10
# Nokia Corporation - initial contribution.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    11
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    12
# Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    13
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    14
# Description: 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    15
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    16
# Runs the specified suite of raptor tests
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    17
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    18
import os
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    19
import sys
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    20
import re
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    21
import imp
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    22
import datetime
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    23
import traceback
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    24
raptor_tests = imp.load_source("raptor_tests", "common/raptor_tests.py")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    25
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    26
# Command line options ########################################################
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    27
from optparse import OptionParser
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    28
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    29
parser = OptionParser(
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    30
		prog = "run",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    31
		usage = "%prog [Options]")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    32
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    33
parser.add_option("-u", "--upload", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    34
		dest = "upload", default = None,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    35
		help = "Path for uploading results (Can be UNC path)")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    36
parser.add_option("-b", "--branch", action = "store", type = "choice",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    37
		dest = "branch", choices = ["master", "m", "fix", "f", "wip", "w"],
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    38
		help = "string indicating which branch is being tested:\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    39
		"master, fix or wip. Default is 'fix'")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    40
parser.add_option("-s", "--suite", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    41
		dest = "suite", help = "regex to use for selecting test suites")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    42
parser.add_option("-t", "--tests", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    43
		dest = "tests", help = "regex to use for selecting tests")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    44
parser.add_option("-d", "--debug", action = "store_true", dest = "debug_mode",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    45
		default = False, help = "Turns on debug-mode")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    46
parser.add_option("--test-home", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    47
		dest = "test_home",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    48
		help = "Location of custom .sbs_init.xml (name of directory in " +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    49
		"'custom_options'): test/custom_options/<test_home>/.sbs_init.xml")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    50
parser.add_option("--what-failed", action = "store_true", dest = "what_failed",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    51
		help = "Re-run all the tests that failed in the previous test run")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    52
parser.add_option("--clean", action = "store_true", dest = "clean",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    53
		help = "Clean EPOCROOT after each test is run")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    54
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    55
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    56
(options, args) = parser.parse_args()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    57
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    58
# Check for --what-failed and override '-s' and '-t' (including flagless regex)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    59
if options.what_failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    60
	try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    61
		what_failed_file = open("what_failed", "r")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    62
		what_failed = what_failed_file.readline()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    63
		what_failed_file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    64
		print "Running: run " + what_failed
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    65
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    66
		first = what_failed.find('"')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    67
		second = what_failed.find('"', (first + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    68
		options.suite = what_failed[(first + 1):second]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    69
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    70
		first = what_failed.find('"', (second + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    71
		second = what_failed.find('"', (first + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    72
		options.tests = what_failed[(first + 1):second]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    73
	except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    74
		# If no file exists, nothing failed, so run as usual
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    75
		pass
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    76
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    77
# Allow flagless test regex
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    78
if (options.tests == None) and (len(args) > 0):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    79
	options.tests = args[len(args) - 1]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    80
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    81
if options.upload != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    82
	if options.branch != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    83
		if options.branch == "m":
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    84
			branch = "master"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    85
		elif options.branch == "f":
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    86
			branch = "fix"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    87
		elif options.branch == "w":
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    88
			branch = "wip"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    89
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    90
			branch = options.branch
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    91
	else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    92
		print "Warning: Test branch not set - Use " + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    93
				"'-b [master|fix|wip]'\n Using default of 'Fix'..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    94
		branch = "fix"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    95
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    96
if options.debug_mode:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    97
	raptor_tests.activate_debug()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    98
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    99
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   100
# Set $HOME environment variable for finding a custom .sbs_init.xml 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   101
if options.test_home != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   102
	home_dir = options.test_home
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   103
	if home_dir in os.listdir("./custom_options"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   104
		os.environ["HOME"] = os.environ["SBS_HOME"] + "/test/custom_options/" \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   105
				+ home_dir + "/"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   106
	else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   107
		print "Warning: Path to custom .sbs_init.xml file not found (" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   108
				home_dir + ")\nUsing defaults..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   109
		options.test_home = None
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   110
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   111
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   112
def format_milliseconds(microseconds):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   113
	""" format a microsecond time in milliseconds """
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   114
	milliseconds = (microseconds / 1000)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   115
	if milliseconds == 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   116
		return "000"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   117
	elif milliseconds < 10:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   118
		return "00" + str(milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   119
	elif milliseconds < 100:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   120
		return "0" + str(milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   121
	return milliseconds
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   122
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   123
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   124
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   125
class TestRun(object):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   126
	"""Represents any series of tests"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   127
	def __init__(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   128
		self.test_set = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   129
		self.failed_tests = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   130
		self.error_tests = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   131
		self.pass_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   132
		self.fail_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   133
		self.skip_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   134
		self.exception_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   135
		self.test_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   136
		# For --what-failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   137
		self.suites_failed = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   138
		self.tests_failed = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   139
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   140
	def aggregate(self, atestrun):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   141
		""" Aggregate other test results into this one """
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   142
		self.test_set.append(atestrun)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   143
		self.test_total += len(atestrun.test_set)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   144
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   145
	def show(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   146
		for test_set in self.test_set:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   147
			print "\n\n" + str(test_set.suite_dir) + ":\n"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   148
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   149
			# If a suite has failed/erroneous tests, add it to what_failed
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   150
			if (test_set.fail_total + test_set.exception_total) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   151
				self.suites_failed.append(test_set.suite_dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   152
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   153
			if len(test_set.test_set) < 1:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   154
				print "No tests run"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   155
			else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   156
				print "PASSED: " + str(test_set.pass_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   157
				print "FAILED: " + str(test_set.fail_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   158
				if test_set.skip_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   159
					print "SKIPPED: " + str(test_set.skip_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   160
				if test_set.exception_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   161
					print "EXCEPTIONS: " + str(test_set.exception_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   162
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   163
				if test_set.fail_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   164
					print "\nFAILED TESTS:"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   165
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   166
					# Add each failed test to what_failed and print it
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   167
					for test in test_set.failed_tests:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   168
						self.tests_failed.append("^" + test + ".py")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   169
						print "\t", test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   170
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   171
				if test_set.exception_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   172
					print "\nERRONEOUS TESTS:"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   173
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   174
					# Add each erroneous test to what_failed and print it
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   175
					for test in test_set.error_tests:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   176
						first = test.find("'")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   177
						second = test.find("'", (first + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   178
						self.tests_failed.append("^" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   179
								test[(first + 1):second] + ".py")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   180
						print "\t", test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   181
						
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   182
	def what_failed(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   183
		"Create the file for --what-failed if there were failing tests"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   184
		if len(self.suites_failed) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   185
			self.what_failed = open("what_failed", "w")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   186
			# Add the suites and tests to the file as command-line options
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   187
			self.what_failed.write('-s "')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   188
			loop_number = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   189
			for suite in self.suites_failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   190
				loop_number += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   191
				self.what_failed.write(suite)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   192
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   193
				# If this is not the last suite, prepare to add another
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   194
				if loop_number < len(self.suites_failed):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   195
					self.what_failed.write("|")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   196
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   197
			self.what_failed.write('" -t "')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   198
			loop_number = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   199
			for test in self.tests_failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   200
				loop_number += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   201
				self.what_failed.write(test)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   202
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   203
				# If this is not the last test, prepare to add another
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   204
				if loop_number < len(self.tests_failed):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   205
					self.what_failed.write("|")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   206
			self.what_failed.write('"')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   207
			self.what_failed.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   208
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   209
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   210
			# If there were no failing tests this time, remove any previous file
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   211
			try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   212
				os.remove("what_failed")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   213
			except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   214
				try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   215
					os.chmod("what_failed", stat.S_IRWXU)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   216
					os.remove("what_failed")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   217
				except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   218
					pass
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   219
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   220
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   221
class Suite(TestRun):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   222
	"""A test suite"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   223
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   224
	python_file_regex = re.compile("(.*)\.py$", re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   225
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   226
	def __init__(self, dir, parent):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   227
		TestRun.__init__(self)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   228
		self.suite_dir = dir
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   229
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   230
		# Upload directory (if set)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   231
		self.upload_location = parent.upload_location
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   232
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   233
		# Regex for searching for tests
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   234
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   235
		self.test_file_regex = parent.test_file_regex
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   236
		self.test_pattern = parent.testpattern
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   237
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   238
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   239
	def run(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   240
		"""run the suite"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   241
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   242
		self.time_stamp = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   243
		self.results = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   244
		self.start_times = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   245
		self.end_times = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   246
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   247
		print "\n\nRunning " + str(self.suite_dir) + "..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   248
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   249
		# Iterate through all files in specified directory
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   250
		for test in os.listdir(self.suite_dir):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   251
			# Only check '*.py' files
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   252
			name_match = self.python_file_regex.match(test)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   253
			if name_match is not None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   254
				if self.test_file_regex is not None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   255
					# Each file that matches -t input is imported if any
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   256
					name_match = self.test_file_regex.match(test)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   257
				else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   258
					name_match = 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   259
				if name_match is not None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   260
					import_name = test[:-3]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   261
					try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   262
						self.test_set.append(imp.load_source(import_name,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   263
								(raptor_tests.ReplaceEnvs(self.suite_dir
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   264
								+ "/" + test))))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   265
					except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   266
						print "\n", (sys.exc_type.__name__ + ":"), \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   267
								sys.exc_value, "\n", \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   268
								traceback.print_tb(sys.exc_traceback)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   269
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   270
		test_number = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   271
		test_total = len(self.test_set)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   272
		if test_total < 1:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   273
			print "No tests in suite "+self.suite_dir+" matched by specification '"+self.test_pattern+"' (regex: /.*"+self.test_pattern+".*/)\n";
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   274
		# Run each test, capturing all its details and its results
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   275
		for test in self.test_set:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   276
			test_number += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   277
			# Save start/end times and save in dictionary for TMS
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   278
			start_time = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   279
			try:
9
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   280
				test_number_text = "\n\nTEST " + str(test_number) + "/" + \
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   281
						str(test_total) + ":"
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   282
				
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   283
				if self.fail_total > 0:
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   284
					test_number_text += "    So far " + str(self.fail_total) + \
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   285
							" FAILED"
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   286
				if self.exception_total > 0:
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   287
					test_number_text += "    So far " + str(self.exception_total) + \
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   288
							" ERRONEOUS"
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   289
				
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   290
				print test_number_text
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   291
				
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   292
				test_object = test.run()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   293
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   294
				end_time = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   295
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   296
				# Add leading 0s
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   297
				test_object.id = raptor_tests.fix_id(test_object.id)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   298
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   299
				# No millisecond function, so need to use microseconds/1000
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   300
				start_milliseconds = start_time.microsecond
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   301
				end_milliseconds = end_time.microsecond
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   302
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   303
				# Add trailing 0's if required
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   304
				start_milliseconds = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   305
						format_milliseconds(start_milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   306
				end_milliseconds = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   307
						format_milliseconds(end_milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   308
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   309
				self.start_times[test_object.id] = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   310
						start_time.strftime("%H:%M:%S:" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   311
						str(start_milliseconds))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   312
				self.end_times[test_object.id] = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   313
						end_time.strftime("%H:%M:%S:" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   314
						str(end_milliseconds))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   315
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   316
				run_time = (end_time - start_time)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   317
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   318
				run_time_seconds = (str(run_time.seconds) + "." + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   319
						str(format_milliseconds(run_time.microseconds)))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   320
				print ("RunTime: " + run_time_seconds + "s")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   321
				# Add to pass/fail count and save result to dictionary
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   322
				if test_object.result == raptor_tests.SmokeTest.PASS:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   323
					self.pass_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   324
					self.results[test_object.id] = "Passed"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   325
				elif test_object.result == raptor_tests.SmokeTest.FAIL:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   326
					self.fail_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   327
					self.results[test_object.id] = "Failed"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   328
					self.failed_tests.append(test_object.name)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   329
				elif test_object.result == raptor_tests.SmokeTest.SKIP:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   330
					self.skip_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   331
				# Clean epocroot after running each test if --clean option is specified
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   332
				if options.clean:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   333
					print "\nCLEANING TEST RESULTS..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   334
					raptor_tests.clean_epocroot()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   335
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   336
			except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   337
				print "\nTEST ERROR:"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   338
				print (sys.exc_type.__name__ + ":"), \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   339
						sys.exc_value, "\n", \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   340
						traceback.print_tb(sys.exc_traceback)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   341
				self.exception_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   342
				self.error_tests.append(str(self.test_set[test_number - 1]))
9
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   343
								
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   344
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   345
		if self.upload_location != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   346
			self.create_csv()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   347
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   348
		end_time_stamp = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   349
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   350
		runtime = end_time_stamp - self.time_stamp
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   351
		seconds = (str(runtime.seconds) + "." + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   352
				str(format_milliseconds(runtime.microseconds)))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   353
		if options.upload:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   354
			self.create_tri(seconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   355
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   356
		print ("\n" + str(self.suite_dir) + " RunTime: " + seconds + "s")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   357
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   358
	def create_csv(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   359
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   360
		This method will create a CSV file with the smoke test's output
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   361
				in order to successfully upload results to TMS QC
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   362
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   363
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   364
		# This sorts the dictionaries by their key values (Test IDs)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   365
		id_list = run_tests.sort_dict(self.results)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   366
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   367
		self.test_file_name = (self.suite_dir + "_" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   368
				self.time_stamp.strftime("%Y-%m-%d_%H-%M-%S") + "_" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   369
				branch + "_results.csv")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   370
		# This is the path for file-creation on the server. Includes
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   371
		self.test_path = (self.upload_location + "/csv/" + self.suite_dir + "/"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   372
				+ self.test_file_name)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   373
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   374
		try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   375
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   376
			if not os.path.isdir(self.upload_location + "/csv/" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   377
					self.suite_dir):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   378
				os.makedirs(self.upload_location + "/csv/" + self.suite_dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   379
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   380
			csv_file = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   381
					open(raptor_tests.ReplaceEnvs(os.path.normpath(self.test_path)),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   382
					"w")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   383
			csv_file.write("TestCaseID,StartTime,EndTime,Result\n")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   384
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   385
			for test_id in id_list:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   386
				csv_file.write("PCT-SBSV2-" + self.suite_dir + "-" + test_id + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   387
						"," + str(self.start_times[test_id]) + "," + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   388
						str(self.end_times[test_id]) + "," + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   389
						self.results[test_id] + "\n")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   390
			csv_file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   391
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   392
		except OSError, e:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   393
			print "SBS_TESTS: Error:", e
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   394
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   395
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   396
	def create_tri(self, overall_seconds):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   397
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   398
		This method will create a TRI (xml) file containing the location of the
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   399
				CSV file in order to successfully upload results to TMS QC
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   400
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   401
		# Path for the tri file
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   402
		tri_path = (self.upload_location + "/new/" + self.suite_dir + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   403
				"_" + self.time_stamp.strftime("%Y-%m-%d_%H-%M-%S") + ".xml")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   404
		run_name_timestamp = self.time_stamp.strftime(self.suite_dir + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   405
				"%Y-%m-%d_%H-%M-%S")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   406
		date_time_timestamp = self.time_stamp.strftime("%d.%m.%Y %H:%M:%S")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   407
		test_set_name = "Root\\Product Creation Tools\\Regression\\" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   408
				"SBS v2 (Raptor)\\" + self.suite_dir + "_"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   409
		if sys.platform.startswith("win"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   410
			test_set_name += ("WinXP_" + branch)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   411
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   412
			test_set_name += ("Linux_" + branch)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   413
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   414
		# /mnt/ -> // Fixes the difference in paths for lon-rhdev mounts vs. win
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   415
		if not sys.platform.startswith("win"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   416
			if self.test_path.startswith("/mnt/"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   417
				self.test_path = self.test_path.replace("mnt", "", 1)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   418
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   419
		try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   420
			tri_file = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   421
					open(raptor_tests.ReplaceEnvs(os.path.normpath(tri_path)), \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   422
					"w")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   423
			tri_file.write(
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   424
					"<TestRunInfo>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   425
						"\t<RunName>\n\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   426
							run_name_timestamp + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   427
						"\n\t</RunName>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   428
						"\t<TestGroup>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   429
							"\t\tSBSv2 (Non-SITK)\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   430
						"\t</TestGroup>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   431
						"\t<DateTime>\n\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   432
							date_time_timestamp + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   433
						"\n\t</DateTime>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   434
						"\t<RunDuration>\n\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   435
							overall_seconds + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   436
						"\n\t</RunDuration>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   437
						'\t<TestSet name="' + test_set_name + '">\n' + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   438
							"\t\t<TestResults>\n\t\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   439
								self.test_path + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   440
							"\n\t\t</TestResults>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   441
						"\t</TestSet>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   442
					"</TestRunInfo>")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   443
			tri_file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   444
			print "Tests uploaded to '" + self.upload_location + "' (" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   445
					branch + ")"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   446
		except OSError, e:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   447
			print "SBS_TESTS: Error:", e
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   448
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   449
class SuiteRun(TestRun):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   450
	""" Represents a 'run' of a number of test suites """
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   451
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   452
	def __init__(self, suitepattern = None, testpattern = None,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   453
			upload_location = None):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   454
		TestRun.__init__(self)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   455
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   456
		# Add common directory to list of paths to search for modules
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   457
		sys.path.append(raptor_tests.ReplaceEnvs("$(SBS_HOME)/test/common"))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   458
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   459
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   460
		if suitepattern:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   461
			self.suite_regex = re.compile(".*" + suitepattern + ".*", re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   462
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   463
			self.suite_regex = re.compile(".*\_suite$", re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   464
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   465
		if testpattern:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   466
			self.test_file_regex = re.compile(".*" + testpattern + ".*",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   467
					re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   468
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   469
			self.test_file_regex = None
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   470
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   471
		self.suitepattern = suitepattern
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   472
		self.testpattern = testpattern
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   473
		self.upload_location = upload_location
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   474
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   475
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   476
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   477
	def run_tests(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   478
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   479
		Run all the tests in the specified suite (directory)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   480
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   481
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   482
		suites = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   483
		for dir in os.listdir("."):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   484
			name_match = self.suite_regex.match(dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   485
			# Each folder that matches the suite pattern will be looked into
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   486
			# Also checks to make sure the found entry is actually a directory
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   487
			if name_match is not None and os.path.isdir(dir):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   488
				s = Suite(dir, self)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   489
				s.run()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   490
				self.aggregate(s)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   491
				suites.append(dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   492
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   493
		# Print which options were used
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   494
		if options.test_home == None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   495
			options_dir = "defaults)"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   496
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   497
			options_dir = "'" + options.test_home + "' options file)"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   498
		print "\n(Tests run using %s" %options_dir
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   499
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   500
		# Summarise the entire test run
9
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   501
		if self.suitepattern and (len(suites) < 1):
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   502
			print "\nNo suites matched specification '" + self.suitepattern + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   503
					"'\n"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   504
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   505
			print "Overall summary (%d suites, %d tests):" \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   506
					%(len(suites), self.test_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   507
			self.show()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   508
			self.what_failed()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   509
	        
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   510
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   511
	def sort_dict(self, input_dict):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   512
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   513
		This method sorts values in a dictionary
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   514
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   515
		keys = input_dict.keys()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   516
		keys.sort()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   517
		return keys
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   518
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   519
135
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   520
# Make SBS_HOME, EPOCROOT have uppercase drive letters to match os.getcwd() and
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   521
# thus stop all those insane test problems which result from one being uppercase
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   522
# and the other lowercase
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   523
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   524
if sys.platform.startswith("win"):
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   525
	sh = os.environ['SBS_HOME']
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   526
	if sh[1] == ':':
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   527
		os.environ['SBS_HOME'] = sh[0].upper() + sh[1:]
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   528
	er = os.environ['EPOCROOT']
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   529
	if er[1] == ':':
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   530
		os.environ['EPOCROOT'] = er[0].upper() + er[1:]
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   531
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   532
# Clean epocroot before running tests
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   533
raptor_tests.clean_epocroot()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   534
run_tests = SuiteRun(suitepattern = options.suite, testpattern = options.tests,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   535
		upload_location = options.upload)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   536
run_tests.run_tests()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   537