sbsv2/raptor/test/common/run_tests.py
author Jon Chatten
Thu, 25 Mar 2010 17:00:05 +0000
branchfix
changeset 404 d420eba6f3a0
parent 135 0092642f198e
permissions -rw-r--r--
release note: DPDEF143101 - GCC-E isn't tested as part of the tool checking mechanism
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     1
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     2
# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     3
# All rights reserved.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     4
# This component and the accompanying materials are made available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     5
# under the terms of the License "Eclipse Public License v1.0"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     6
# which accompanies this distribution, and is available
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     7
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     8
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
     9
# Initial Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    10
# Nokia Corporation - initial contribution.
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    11
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    12
# Contributors:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    13
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    14
# Description: 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    15
#
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    16
# Runs the specified suite of raptor tests
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    17
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    18
import os
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    19
import sys
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    20
import re
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    21
import imp
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    22
import datetime
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    23
import traceback
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    24
raptor_tests = imp.load_source("raptor_tests", "common/raptor_tests.py")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    25
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    26
# Command line options ########################################################
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    27
from optparse import OptionParser
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    28
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    29
parser = OptionParser(
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    30
		prog = "run",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    31
		usage = "%prog [Options]")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    32
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    33
parser.add_option("-u", "--upload", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    34
		dest = "upload", default = None,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    35
		help = "Path for uploading results (Can be UNC path)")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    36
parser.add_option("-b", "--branch", action = "store", type = "choice",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    37
		dest = "branch", choices = ["master", "m", "fix", "f", "wip", "w"],
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    38
		help = "string indicating which branch is being tested:\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    39
		"master, fix or wip. Default is 'fix'")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    40
parser.add_option("-s", "--suite", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    41
		dest = "suite", help = "regex to use for selecting test suites")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    42
parser.add_option("-t", "--tests", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    43
		dest = "tests", help = "regex to use for selecting tests")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    44
parser.add_option("-d", "--debug", action = "store_true", dest = "debug_mode",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    45
		default = False, help = "Turns on debug-mode")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    46
parser.add_option("--test-home", action = "store", type = "string",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    47
		dest = "test_home",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    48
		help = "Location of custom .sbs_init.xml (name of directory in " +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    49
		"'custom_options'): test/custom_options/<test_home>/.sbs_init.xml")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    50
parser.add_option("--what-failed", action = "store_true", dest = "what_failed",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    51
		help = "Re-run all the tests that failed in the previous test run")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    52
parser.add_option("--clean", action = "store_true", dest = "clean",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    53
		help = "Clean EPOCROOT after each test is run")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    54
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    55
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    56
(options, args) = parser.parse_args()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    57
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    58
# Check for --what-failed and override '-s' and '-t' (including flagless regex)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    59
if options.what_failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    60
	try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    61
		what_failed_file = open("what_failed", "r")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    62
		what_failed = what_failed_file.readline()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    63
		what_failed_file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    64
		print "Running: run " + what_failed
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    65
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    66
		first = what_failed.find('"')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    67
		second = what_failed.find('"', (first + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    68
		options.suite = what_failed[(first + 1):second]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    69
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    70
		first = what_failed.find('"', (second + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    71
		second = what_failed.find('"', (first + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    72
		options.tests = what_failed[(first + 1):second]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    73
	except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    74
		# If no file exists, nothing failed, so run as usual
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    75
		pass
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    76
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    77
# Allow flagless test regex
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    78
if (options.tests == None) and (len(args) > 0):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    79
	options.tests = args[len(args) - 1]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    80
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    81
if options.upload != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    82
	if options.branch != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    83
		if options.branch == "m":
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    84
			branch = "master"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    85
		elif options.branch == "f":
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    86
			branch = "fix"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    87
		elif options.branch == "w":
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    88
			branch = "wip"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    89
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    90
			branch = options.branch
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    91
	else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    92
		print "Warning: Test branch not set - Use " + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    93
				"'-b [master|fix|wip]'\n Using default of 'Fix'..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    94
		branch = "fix"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    95
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    96
if options.debug_mode:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    97
	raptor_tests.activate_debug()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    98
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
    99
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   100
# Set $HOME environment variable for finding a custom .sbs_init.xml 
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   101
if options.test_home != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   102
	home_dir = options.test_home
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   103
	if home_dir in os.listdir("./custom_options"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   104
		os.environ["HOME"] = os.environ["SBS_HOME"] + "/test/custom_options/" \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   105
				+ home_dir + "/"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   106
	else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   107
		print "Warning: Path to custom .sbs_init.xml file not found (" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   108
				home_dir + ")\nUsing defaults..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   109
		options.test_home = None
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   110
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   111
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   112
def format_milliseconds(microseconds):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   113
	""" format a microsecond time in milliseconds """
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   114
	milliseconds = (microseconds / 1000)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   115
	if milliseconds == 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   116
		return "000"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   117
	elif milliseconds < 10:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   118
		return "00" + str(milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   119
	elif milliseconds < 100:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   120
		return "0" + str(milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   121
	return milliseconds
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   122
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   123
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   124
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   125
class TestRun(object):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   126
	"""Represents any series of tests"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   127
	def __init__(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   128
		self.test_set = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   129
		self.failed_tests = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   130
		self.error_tests = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   131
		self.pass_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   132
		self.fail_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   133
		self.skip_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   134
		self.exception_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   135
		self.test_total = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   136
		# For --what-failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   137
		self.suites_failed = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   138
		self.tests_failed = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   139
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   140
	def aggregate(self, atestrun):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   141
		""" Aggregate other test results into this one """
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   142
		self.test_set.append(atestrun)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   143
		self.test_total += len(atestrun.test_set)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   144
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   145
	def show(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   146
		for test_set in self.test_set:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   147
			print "\n\n" + str(test_set.suite_dir) + ":\n"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   148
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   149
			# If a suite has failed/erroneous tests, add it to what_failed
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   150
			if (test_set.fail_total + test_set.exception_total) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   151
				self.suites_failed.append(test_set.suite_dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   152
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   153
			if len(test_set.test_set) < 1:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   154
				print "No tests run"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   155
			else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   156
				print "PASSED: " + str(test_set.pass_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   157
				print "FAILED: " + str(test_set.fail_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   158
				if test_set.skip_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   159
					print "SKIPPED: " + str(test_set.skip_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   160
				if test_set.exception_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   161
					print "EXCEPTIONS: " + str(test_set.exception_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   162
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   163
				if test_set.fail_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   164
					print "\nFAILED TESTS:"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   165
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   166
					# Add each failed test to what_failed and print it
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   167
					for test in test_set.failed_tests:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   168
						self.tests_failed.append("^" + test + ".py")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   169
						print "\t", test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   170
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   171
				if test_set.exception_total > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   172
					print "\nERRONEOUS TESTS:"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   173
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   174
					# Add each erroneous test to what_failed and print it
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   175
					for test in test_set.error_tests:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   176
						first = test.find("'")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   177
						second = test.find("'", (first + 1))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   178
						self.tests_failed.append("^" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   179
								test[(first + 1):second] + ".py")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   180
						print "\t", test
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   181
						
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   182
	def what_failed(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   183
		"Create the file for --what-failed if there were failing tests"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   184
		if len(self.suites_failed) > 0:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   185
			self.what_failed = open("what_failed", "w")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   186
			# Add the suites and tests to the file as command-line options
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   187
			self.what_failed.write('-s "')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   188
			loop_number = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   189
			for suite in self.suites_failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   190
				loop_number += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   191
				self.what_failed.write(suite)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   192
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   193
				# If this is not the last suite, prepare to add another
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   194
				if loop_number < len(self.suites_failed):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   195
					self.what_failed.write("|")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   196
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   197
			self.what_failed.write('" -t "')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   198
			loop_number = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   199
			for test in self.tests_failed:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   200
				loop_number += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   201
				self.what_failed.write(test)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   202
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   203
				# If this is not the last test, prepare to add another
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   204
				if loop_number < len(self.tests_failed):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   205
					self.what_failed.write("|")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   206
			self.what_failed.write('"')
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   207
			self.what_failed.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   208
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   209
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   210
			# If there were no failing tests this time, remove any previous file
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   211
			try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   212
				os.remove("what_failed")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   213
			except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   214
				try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   215
					os.chmod("what_failed", stat.S_IRWXU)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   216
					os.remove("what_failed")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   217
				except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   218
					pass
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   219
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   220
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   221
class Suite(TestRun):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   222
	"""A test suite"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   223
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   224
	python_file_regex = re.compile("(.*)\.py$", re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   225
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   226
	def __init__(self, dir, parent):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   227
		TestRun.__init__(self)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   228
		self.suite_dir = dir
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   229
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   230
		# Upload directory (if set)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   231
		self.upload_location = parent.upload_location
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   232
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   233
		# Regex for searching for tests
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   234
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   235
		self.test_file_regex = parent.test_file_regex
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   236
		self.test_pattern = parent.testpattern
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   237
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   238
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   239
	def run(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   240
		"""run the suite"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   241
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   242
		self.time_stamp = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   243
		self.results = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   244
		self.start_times = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   245
		self.end_times = {}
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   246
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   247
		print "\n\nRunning " + str(self.suite_dir) + "..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   248
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   249
		# Iterate through all files in specified directory
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   250
		for test in os.listdir(self.suite_dir):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   251
			# Only check '*.py' files
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   252
			name_match = self.python_file_regex.match(test)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   253
			if name_match is not None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   254
				if self.test_file_regex is not None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   255
					# Each file that matches -t input is imported if any
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   256
					name_match = self.test_file_regex.match(test)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   257
				else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   258
					name_match = 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   259
				if name_match is not None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   260
					import_name = test[:-3]
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   261
					try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   262
						self.test_set.append(imp.load_source(import_name,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   263
								(raptor_tests.ReplaceEnvs(self.suite_dir
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   264
								+ "/" + test))))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   265
					except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   266
						print "\n", (sys.exc_type.__name__ + ":"), \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   267
								sys.exc_value, "\n", \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   268
								traceback.print_tb(sys.exc_traceback)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   269
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   270
		test_number = 0
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   271
		test_total = len(self.test_set)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   272
		if test_total < 1:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   273
			print "No tests in suite "+self.suite_dir+" matched by specification '"+self.test_pattern+"' (regex: /.*"+self.test_pattern+".*/)\n";
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   274
		# Run each test, capturing all its details and its results
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   275
		for test in self.test_set:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   276
			test_number += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   277
			# Save start/end times and save in dictionary for TMS
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   278
			start_time = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   279
			try:
9
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   280
				test_number_text = "\n\nTEST " + str(test_number) + "/" + \
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   281
						str(test_total) + ":"
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   282
				
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   283
				if self.fail_total > 0:
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   284
					test_number_text += "    So far " + str(self.fail_total) + \
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   285
							" FAILED"
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   286
				if self.exception_total > 0:
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   287
					test_number_text += "    So far " + str(self.exception_total) + \
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   288
							" ERRONEOUS"
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   289
				
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   290
				print test_number_text
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   291
				
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   292
				test_object = test.run()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   293
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   294
				end_time = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   295
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   296
				# Add leading 0s
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   297
				test_object.id = raptor_tests.fix_id(test_object.id)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   298
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   299
				# No millisecond function, so need to use microseconds/1000
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   300
				start_milliseconds = start_time.microsecond
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   301
				end_milliseconds = end_time.microsecond
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   302
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   303
				# Add trailing 0's if required
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   304
				start_milliseconds = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   305
						format_milliseconds(start_milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   306
				end_milliseconds = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   307
						format_milliseconds(end_milliseconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   308
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   309
				self.start_times[test_object.id] = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   310
						start_time.strftime("%H:%M:%S:" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   311
						str(start_milliseconds))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   312
				self.end_times[test_object.id] = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   313
						end_time.strftime("%H:%M:%S:" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   314
						str(end_milliseconds))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   315
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   316
				run_time = (end_time - start_time)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   317
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   318
				run_time_seconds = (str(run_time.seconds) + "." + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   319
						str(format_milliseconds(run_time.microseconds)))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   320
				print ("RunTime: " + run_time_seconds + "s")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   321
				# Add to pass/fail count and save result to dictionary
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   322
				if test_object.result == raptor_tests.SmokeTest.PASS:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   323
					self.pass_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   324
					self.results[test_object.id] = "Passed"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   325
				elif test_object.result == raptor_tests.SmokeTest.FAIL:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   326
					self.fail_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   327
					self.results[test_object.id] = "Failed"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   328
					self.failed_tests.append(test_object.name)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   329
				elif test_object.result == raptor_tests.SmokeTest.SKIP:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   330
					self.skip_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   331
				# Clean epocroot after running each test if --clean option is specified
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   332
				if options.clean:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   333
					print "\nCLEANING TEST RESULTS..."
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   334
					raptor_tests.clean_epocroot()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   335
					
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   336
			except:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   337
				print "\nTEST ERROR:"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   338
				print (sys.exc_type.__name__ + ":"), \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   339
						sys.exc_value, "\n", \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   340
						traceback.print_tb(sys.exc_traceback)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   341
				self.exception_total += 1
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   342
				self.error_tests.append(str(self.test_set[test_number - 1]))
9
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   343
								
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   344
				
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   345
		if self.upload_location != None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   346
			self.create_csv()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   347
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   348
		end_time_stamp = datetime.datetime.now()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   349
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   350
		runtime = end_time_stamp - self.time_stamp
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   351
		seconds = (str(runtime.seconds) + "." + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   352
				str(format_milliseconds(runtime.microseconds)))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   353
		if options.upload:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   354
			self.create_tri(seconds)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   355
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   356
		print ("\n" + str(self.suite_dir) + " RunTime: " + seconds + "s")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   357
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   358
	def create_csv(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   359
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   360
		This method will create a CSV file with the smoke test's output
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   361
				in order to successfully upload results to TMS QC
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   362
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   363
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   364
		# This sorts the dictionaries by their key values (Test IDs)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   365
		id_list = run_tests.sort_dict(self.results)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   366
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   367
		self.test_file_name = (self.suite_dir + "_" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   368
				self.time_stamp.strftime("%Y-%m-%d_%H-%M-%S") + "_" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   369
				branch + "_results.csv")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   370
		# This is the path for file-creation on the server. Includes
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   371
		self.test_path = (self.upload_location + "/csv/" + self.suite_dir + "/"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   372
				+ self.test_file_name)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   373
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   374
		try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   375
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   376
			if not os.path.isdir(self.upload_location + "/csv/" +
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   377
					self.suite_dir):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   378
				os.makedirs(self.upload_location + "/csv/" + self.suite_dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   379
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   380
			csv_file = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   381
					open(raptor_tests.ReplaceEnvs(os.path.normpath(self.test_path)),
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   382
					"w")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   383
			csv_file.write("TestCaseID,StartTime,EndTime,Result\n")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   384
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   385
			for test_id in id_list:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   386
				csv_file.write("PCT-SBSV2-" + self.suite_dir + "-" + test_id + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   387
						"," + str(self.start_times[test_id]) + "," + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   388
						str(self.end_times[test_id]) + "," + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   389
						self.results[test_id] + "\n")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   390
			csv_file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   391
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   392
		except OSError, e:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   393
			print "SBS_TESTS: Error:", e
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   394
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   395
			
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   396
	def create_tri(self, overall_seconds):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   397
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   398
		This method will create a TRI (xml) file containing the location of the
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   399
				CSV file in order to successfully upload results to TMS QC
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   400
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   401
		# Path for the tri file
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   402
		tri_path = (self.upload_location + "/new/" + self.suite_dir + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   403
				"_" + self.time_stamp.strftime("%Y-%m-%d_%H-%M-%S") + ".xml")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   404
		run_name_timestamp = self.time_stamp.strftime(self.suite_dir + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   405
				"%Y-%m-%d_%H-%M-%S")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   406
		date_time_timestamp = self.time_stamp.strftime("%d.%m.%Y %H:%M:%S")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   407
		test_set_name = "Root\\Product Creation Tools\\Regression\\" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   408
				"SBS v2 (Raptor)\\" + self.suite_dir + "_"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   409
		if sys.platform.startswith("win"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   410
			test_set_name += ("WinXP_" + branch)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   411
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   412
			test_set_name += ("Linux_" + branch)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   413
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   414
		# /mnt/ -> // Fixes the difference in paths for lon-rhdev mounts vs. win
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   415
		if not sys.platform.startswith("win"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   416
			if self.test_path.startswith("/mnt/"):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   417
				self.test_path = self.test_path.replace("mnt", "", 1)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   418
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   419
		try:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   420
			tri_file = \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   421
					open(raptor_tests.ReplaceEnvs(os.path.normpath(tri_path)), \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   422
					"w")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   423
			tri_file.write(
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   424
					"<TestRunInfo>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   425
						"\t<RunName>\n\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   426
							run_name_timestamp + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   427
						"\n\t</RunName>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   428
						"\t<TestGroup>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   429
							"\t\tSBSv2 (Non-SITK)\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   430
						"\t</TestGroup>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   431
						"\t<DateTime>\n\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   432
							date_time_timestamp + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   433
						"\n\t</DateTime>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   434
						"\t<RunDuration>\n\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   435
							overall_seconds + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   436
						"\n\t</RunDuration>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   437
						'\t<TestSet name="' + test_set_name + '">\n' + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   438
							"\t\t<TestResults>\n\t\t\t" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   439
								self.test_path + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   440
							"\n\t\t</TestResults>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   441
						"\t</TestSet>\n" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   442
					"</TestRunInfo>")
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   443
			tri_file.close()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   444
			print "Tests uploaded to '" + self.upload_location + "' (" + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   445
					branch + ")"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   446
		except OSError, e:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   447
			print "SBS_TESTS: Error:", e
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   448
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   449
class SuiteRun(TestRun):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   450
	""" Represents a 'run' of a number of test suites """
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   451
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   452
	def __init__(self, suitepattern = None, testpattern = None,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   453
			upload_location = None):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   454
		TestRun.__init__(self)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   455
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   456
		# Add common directory to list of paths to search for modules
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   457
		sys.path.append(raptor_tests.ReplaceEnvs("$(SBS_HOME)/test/common"))
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   458
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   459
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   460
		if suitepattern:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   461
			self.suite_regex = re.compile(".*" + suitepattern + ".*", re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   462
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   463
			self.suite_regex = re.compile(".*\_suite$", re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   464
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   465
		if testpattern:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   466
			self.test_file_regex = re.compile(".*" + testpattern + ".*",
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   467
					re.I)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   468
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   469
			self.test_file_regex = None
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   470
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   471
		self.suitepattern = suitepattern
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   472
		self.testpattern = testpattern
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   473
		self.upload_location = upload_location
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   474
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   475
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   476
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   477
	def run_tests(self):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   478
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   479
		Run all the tests in the specified suite (directory)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   480
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   481
	
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   482
		suites = []
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   483
		for dir in os.listdir("."):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   484
			name_match = self.suite_regex.match(dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   485
			# Each folder that matches the suite pattern will be looked into
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   486
			# Also checks to make sure the found entry is actually a directory
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   487
			if name_match is not None and os.path.isdir(dir):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   488
				s = Suite(dir, self)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   489
				s.run()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   490
				self.aggregate(s)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   491
				suites.append(dir)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   492
		
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   493
		# Print which options were used
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   494
		if options.test_home == None:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   495
			options_dir = "defaults)"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   496
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   497
			options_dir = "'" + options.test_home + "' options file)"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   498
		print "\n(Tests run using %s" %options_dir
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   499
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   500
		# Summarise the entire test run
9
b211d87c390a Update to 2.11.0 from perforce
timothy.murphy@nokia.com
parents: 3
diff changeset
   501
		if self.suitepattern and (len(suites) < 1):
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   502
			print "\nNo suites matched specification '" + self.suitepattern + \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   503
					"'\n"
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   504
		else:
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   505
			print "Overall summary (%d suites, %d tests):" \
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   506
					%(len(suites), self.test_total)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   507
			self.show()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   508
			self.what_failed()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   509
	        
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   510
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   511
	def sort_dict(self, input_dict):
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   512
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   513
		This method sorts values in a dictionary
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   514
		"""
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   515
		keys = input_dict.keys()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   516
		keys.sort()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   517
		return keys
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   518
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   519
135
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   520
# Make SBS_HOME, EPOCROOT have uppercase drive letters to match os.getcwd() and
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   521
# thus stop all those insane test problems which result from one being uppercase
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   522
# and the other lowercase
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   523
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   524
if sys.platform.startswith("win"):
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   525
	sh = os.environ['SBS_HOME']
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   526
	if sh[1] == ':':
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   527
		os.environ['SBS_HOME'] = sh[0].upper() + sh[1:]
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   528
	er = os.environ['EPOCROOT']
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   529
	if er[1] == ':':
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   530
		os.environ['EPOCROOT'] = er[0].upper() + er[1:]
0092642f198e Elimitate those testing messups that result from the case of the drive letter in SBS_HOME or EPOCROOT
raptorbot <raptorbot@systemstesthead.symbian.intra>
parents: 9
diff changeset
   531
3
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   532
# Clean epocroot before running tests
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   533
raptor_tests.clean_epocroot()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   534
run_tests = SuiteRun(suitepattern = options.suite, testpattern = options.tests,
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   535
		upload_location = options.upload)
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   536
run_tests.run_tests()
e1eecf4d390d Team sf branch.
tnmurphy@4GBL06592.nokia.com
parents:
diff changeset
   537