--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sbsv2/raptor/test/common/raptor_tests.py Mon Nov 16 09:46:46 2009 +0000
@@ -0,0 +1,649 @@
+#
+# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# All rights reserved.
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Nokia Corporation - initial contribution.
+#
+# Contributors:
+#
+# Description:
+#
+
+# run the smoke tests
+
+import os
+import re
+import stat
+import sys
+import subprocess
+import traceback
+from shutil import rmtree
+
+sys.path.append(os.environ["SBS_HOME"]+"/python")
+from raptor_meta import BldInfFile
+
+logDir = "$(EPOCROOT)/epoc32/build/smoketestlogs"
+
+debug_mode_active = False
+
+# Environment #################################################################
+
+# On MYS there is USERNAME but not USER
+if 'USER' not in os.environ:
+ os.environ['USER'] = os.environ['USERNAME']
+
+def activate_debug():
+ """
+ Activate debug-mode remotely
+ """
+ global debug_mode_active
+ debug_mode_active = True
+
+# Determine the OS version in the epocroot we're testing
+# since some tests expect different outcomes for 9.4 and 9.5
+def getsymbianversion():
+ epocroot = os.environ['EPOCROOT']
+ b = open (epocroot+"/epoc32/data/buildinfo.txt","r")
+ binfo = " ".join(b.readlines())
+ vmatch = (re.compile("v(9\.[0-9])")).search(binfo)
+ if vmatch:
+ osversion = vmatch.group(1)
+ else:
+ osversion = '9.4'
+ return osversion
+
+envRegex = re.compile("\$\((.+?)\)")
+fixEnvironment = ['EPOCROOT', 'SBS_HOME', 'SBS_CYGWIN', 'SBS_MINGW', 'SBS_PYTHON']
+
+def ReplaceEnvs(item):
+
+ envs = envRegex.findall(item)
+
+ for e in set(envs):
+ try:
+ val = os.environ[e]
+ if e in fixEnvironment:
+ # Raptor "fixes up" EPOCROOT etc. so we must do the same:
+ # add the drive letter (make absolute)
+ val = os.path.abspath(val)
+ # use forward slashes
+ val = val.replace("\\", "/")
+ # remove trailing slashes
+ val = val.rstrip("/")
+ item = item.replace("$(" + e + ")", val)
+ except KeyError:
+ print e, "is not set in the environment"
+ raise ValueError
+
+ return item
+
+# Utility functions ###########################################################
+
+
+
+def where(input_file):
+ """Search for 'input_file' in the system path"""
+ locations = []
+ if sys.platform.startswith("win"):
+ if not input_file.lower().endswith(".exe"):
+ input_file += ".exe"
+ for current_file in [loop_number + "\\" + input_file for loop_number in
+ os.environ["PATH"].split(";")]:
+ try:
+ stat = os.stat(current_file)
+ locations.append(current_file)
+ except OSError, error:
+ pass
+ else:
+ (comIn, comOut) = os.popen4("which " + input_file)
+ output = comOut.read()
+ if len(output) > 0:
+ locations.append(output[0:(len(output) - 1)])
+
+ if len(locations) == 0:
+ print "Error: " + input_file + " not defined in PATH environment variable"
+ else:
+ return locations[0]
+
+def clean_epocroot():
+ """
+ This method walks through epocroot and cleans every file and folder that is
+ not present in the manifest file
+ """
+ all_files = {} # dictionary to hold all files
+ folders = [] # holds all unique folders in manifest
+ try:
+ mani = "./epocroot/manifest"
+ manifest = open(ReplaceEnvs(mani), "r")
+ # This is a fast algorithm to read the manifest file
+ while 1:
+ # The file is close to 3000 lines.
+ # If this value changes, increment the number to include all lines
+ lines = manifest.readlines(3000)
+ if not lines:
+ break
+ for line in lines:
+ # Get rid of newline char and add to dictionary
+ all_files[line.rstrip("\n")] = True
+ # This bit makes a record of unique folders into a list
+ end = 0
+ while end != -1: # Look through the parent folders
+ end = line.rfind("/")
+ line = line[:end]
+ if line not in folders:
+ folders.append(line)
+ # This algorithm walks through epocroot and handles files and folders
+ walkpath = "./epocroot"
+ for (root, dirs, files) in os.walk(ReplaceEnvs(walkpath), topdown =
+ False):
+ # This loop handles all files
+ for name in files:
+ name = os.path.join(root, name).replace("\\", "/")
+ name = name.replace("./epocroot/", "./")
+
+ if name not in all_files:
+ try:
+ name = ReplaceEnvs(name.replace("./", "./epocroot/"))
+ os.remove(name)
+ except:
+ # chmod to rw and try again
+ try:
+ os.chmod(name, stat.S_IRWXU)
+ os.remove(name)
+ except:
+ print "\nEPOCROOT-CLEAN ERROR:"
+ print (sys.exc_type.__name__ + ":"), \
+ sys.exc_value, "\n", \
+ traceback.print_tb(sys.exc_traceback)
+
+ # This loop handles folders
+ for name in dirs:
+ name = os.path.join(root, name).replace("\\", "/")
+ name = name.replace("./epocroot/", "./")
+ if name not in all_files and name not in folders:
+ # Remove the folder fully with no errors if full
+ try:
+ rmtree(ReplaceEnvs(name.replace("./", "./epocroot/")))
+ except:
+ print "\nEPOCROOT-CLEAN ERROR:"
+ print (sys.exc_type.__name__ + ":"), \
+ sys.exc_value, "\n", \
+ traceback.print_tb(sys.exc_traceback)
+ except IOError,e:
+ print e
+
+def fix_id(input_id):
+ return input_id.zfill(4)
+
+# Test classes ################################################################
+
+class SmokeTest(object):
+ """Base class for Smoke Test objects.
+
+ Each test is defined (minimally) by,
+ 1) a raptor command-line
+ 2) a list of target files that should get built
+
+ The run() method will,
+ 1) delete all the listed target files
+ 2) execute the raptor command
+ 3) check that the target files were created
+ 4) count the warnings and errors reported
+ """
+
+ PASS = "pass"
+ FAIL = "fail"
+ SKIP = "skip"
+
+ def __init__(self):
+
+ self.id = 0
+ self.name = "smoketest"
+ self.description = ""
+ self.command = "sbs --do_what_i_want"
+ self.targets = []
+ self.missing = 0
+ self.warnings = 0
+ self.errors = 0
+ self.exceptions = 0
+ self.returncode = 0
+
+ self.onWindows = sys.platform.startswith("win")
+
+ # These variables are for tests that treat the text as a list of lines. In
+ # particular, "." will not match end-of-line. This means that, for example,
+ # "abc.*def" will only match if "abc" and "def" appear on the same line.
+ self.mustmatch = []
+ self.mustnotmatch = []
+ self.mustmatch_singleline = []
+ self.mustnotmatch_singleline = []
+
+ # These variables are for tests that treat the text as a single string of
+ # characters. The pattern "." will match anything, including end-of-line.
+ self.mustmatch_multiline = []
+ self.mustnotmatch_multiline = []
+
+ self.countmatch = []
+
+ self.outputok = True
+ self.usebash = False
+ self.failsbecause = None
+ self.result = SmokeTest.SKIP
+ self.environ = {} # Allow tests to set the environment in which commands run.
+ self.sbs_build_dir = "$(EPOCROOT)/epoc32/build"
+
+ def run(self, platform = "all"):
+ previousResult = self.result
+ self.id = fix_id(self.id)
+ try:
+ if self.runnable(platform):
+
+ if not self.pretest():
+ self.result = SmokeTest.FAIL
+
+ elif not self.test():
+ self.result = SmokeTest.FAIL
+
+ elif not self.posttest():
+ self.result = SmokeTest.FAIL
+
+ else:
+ self.result = SmokeTest.PASS
+ else:
+ self.skip(platform)
+ except Exception, e:
+ print e
+ self.result = SmokeTest.FAIL
+
+ # print the result of this run()
+ self.print_result(True)
+
+ # if a previous run() failed then the overall result is a FAIL
+ if previousResult == SmokeTest.FAIL:
+ self.result = SmokeTest.FAIL
+
+ def print_result(self, internal = False, value = ""):
+ # the test passed :-)
+
+ result = self.result
+
+ if value != "":
+ print value
+ else:
+ string = ""
+ if not internal:
+ string += "\n" + self.name + ": "
+ if result == SmokeTest.PASS:
+ string += "PASSED"
+ elif result == SmokeTest.FAIL:
+ string += "FAILED"
+
+ print string
+
+ def runnable(self, platform):
+ # can this test run on this platform?
+ if platform == "all":
+ return True
+
+ isWin = self.onWindows
+ wantWin = platform.startswith("win")
+
+ return (isWin == wantWin)
+
+ def skip(self, platform):
+ print "\nSKIPPING:", self.name, "for", platform
+
+ def logfileOption(self):
+ return "-f " + self.logfile();
+
+ def logfile(self):
+ return logDir + "/" + self.name + ".log"
+
+ def makefileOption(self):
+ return "-m " + self.makefile();
+
+ def makefile(self):
+ return logDir + "/" + self.name + ".mk"
+
+ def removeFiles(self, files):
+ for t in files:
+ tgt = os.path.normpath(ReplaceEnvs(t))
+
+ if os.path.exists(tgt):
+ try:
+ os.chmod(tgt, stat.S_IRWXU)
+ if os.path.isdir(tgt):
+ rmtree(tgt)
+ else:
+ os.remove(tgt)
+ except OSError:
+ print "Could not remove", tgt, "before the test"
+ return False
+ return True
+
+
+ def clean(self):
+ # remove all the target files
+
+ # flatten any lists first (only 1 level of flattenening expected)
+ # these indicate alternative files - one of them will exist after a build
+ removables = []
+ for i in self.targets:
+ if type(i) is not list:
+ removables.append(i)
+ else:
+ removables.extend(i)
+
+ return self.removeFiles(removables)
+
+ def pretest(self):
+ # what to do before the test runs
+
+ print "\nID:", self.id
+ print "TEST:", self.name
+
+ return self.clean()
+
+ def test(self):
+ # run the actual test
+
+ # put the makefile and log in $EPOCROOT/build/smoketestlogs
+ if self.usebash:
+ command = ReplaceEnvs(self.command)
+ else:
+ command = ReplaceEnvs(self.command +
+ " " + self.makefileOption() +
+ " " + self.logfileOption())
+
+ print "COMMAND:", command
+
+
+ # Any environment settings specific to this test
+ shellenv = os.environ.copy()
+ for ev in self.environ:
+ shellenv[ev] = self.environ[ev]
+
+ if self.usebash:
+ shellpath = shellenv['PATH']
+ if self.onWindows:
+ if 'SBS_CYGWIN' in shellenv:
+ shellpath = ReplaceEnvs("$(SBS_CYGWIN)/bin") + ";" + shellpath
+ BASH=ReplaceEnvs("$(SBS_CYGWIN)/bin/bash.exe")
+ else:
+ shellpath = ReplaceEnvs("$(SBS_HOME)/win32/cygwin/bin") + ";" + shellpath
+ BASH=ReplaceEnvs("$(SBS_HOME)/win32/cygwin/bin/bash.exe")
+ else:
+ BASH=ReplaceEnvs("$(SBS_HOME)/$(HOSTPLATFORM_DIR)/bin/bash")
+
+ shellenv['SBSMAKEFILE']=ReplaceEnvs(self.makefile())
+ shellenv['SBSLOGFILE']=ReplaceEnvs(self.logfile())
+ shellenv['PATH']=shellpath
+ shellenv['PYTHON_HOME'] = ""
+ shellenv['CYGWIN']="nontsec nosmbntsec"
+
+ p = subprocess.Popen(args=[BASH, '-c', command],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=shellenv,
+ shell=False,
+ universal_newlines=True)
+
+ self.output = p.communicate()[0]
+ else:
+ p = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=shellenv,
+ shell=True,
+ universal_newlines=True)
+
+ self.output = p.communicate()[0]
+
+ if debug_mode_active:
+ print self.output
+
+ if p.returncode != self.returncode:
+ print "RETURN: got", p.returncode, "expected", self.returncode
+ return False
+
+ return True
+
+ def posttest(self):
+ # what to do after the test has run
+
+ # count the targets that got built
+ found = 0
+ missing = []
+ for t in self.targets:
+ if type(t) is not list:
+ target_alternatives=[t]
+
+ found_flag = False
+ for alt in target_alternatives:
+ tgt = os.path.normpath(ReplaceEnvs(alt))
+ if os.path.exists(tgt):
+ found_flag = True
+ break
+ if found_flag:
+ found += 1
+ else:
+ missing.append(tgt)
+
+ # count the errors and warnings
+ warn = 0
+ error = 0
+ exception = 0
+ lines = self.output.split("\n")
+
+ for line in lines:
+ if line.find("sbs: warning:") != -1 or line.find("<warning") != -1:
+ warn += 1
+ elif line.find("sbs: error:") != -1 or line.find("<error") != -1:
+ error += 1
+ elif line.startswith("Traceback"):
+ exception += 1
+
+ # Check the output for required, forbidden and counted regexp matches
+ self.outputok = True
+
+ for expr in self.mustmatch_singleline + self.mustmatch:
+ if not re.search(expr, self.output, re.MULTILINE):
+ self.outputok = False
+ print "OUTPUTMISMATCH: output did not match: %s" % expr
+
+ for expr in self.mustnotmatch_singleline + self.mustnotmatch:
+ if re.search(expr, self.output, re.MULTILINE):
+ self.outputok = False
+ print "OUTPUTMISMATCH: output should not have matched: %s" % expr
+
+ for expr in self.mustmatch_multiline:
+ if not re.search(expr, self.output, re.DOTALL):
+ self.outputok = False
+ print "OUTPUTMISMATCH: output did not match: %s" % expr
+
+ for expr in self.mustnotmatch_multiline:
+ if re.search(expr, self.output, re.DOTALL):
+ self.outputok = False
+ print "OUTPUTMISMATCH: output should not have matched: %s" % expr
+
+ for (expr,num) in self.countmatch:
+ expr_re = re.compile(expr)
+ matchnum = len(expr_re.findall(self.output))
+ if matchnum != num:
+ print "OUTPUTMISMATCH: %d matches occurred when %d were expected: %s" % (matchnum, num, expr)
+ self.outputok = False
+
+ # Ignore errors/warnings if they are set to (-1)
+ if self.errors == (-1):
+ self.errors = error
+ if self.warnings == (-1):
+ self.warnings= warn
+
+ # all as expected?
+ if self.missing == len(missing) \
+ and self.warnings == warn \
+ and self.errors == error \
+ and self.exceptions == exception \
+ and self.outputok:
+ return True
+
+ # something was wrong :-(
+
+ if len(missing) != self.missing:
+ print "MISSING: %d, expected %s" % (len(missing), self.missing)
+ for file in missing:
+ print "\t%s" % (file)
+
+ if warn != self.warnings:
+ print "WARNINGS: %d, expected %d" % (warn, self.warnings)
+
+ if error != self.errors:
+ print "ERRORS: %d, expected %d" % ( error, self.errors)
+
+ if exception != self.exceptions:
+ print "EXCEPTIONS: %d, expected %d" % (exception, self.exceptions)
+
+ return False
+
+ def addbuildtargets(self, bldinfsourcepath, targetsuffixes):
+ """Add targets that are under epoc32/build whose path
+ can change based on an md5 hash of the path to the bld.inf.
+ """
+
+ fragment = BldInfFile.outputPathFragment(bldinfsourcepath)
+
+ for t in targetsuffixes:
+ if type(t) is not list:
+ newt=self.sbs_build_dir+'/'+fragment+"/"+t
+ self.targets.append(newt)
+ else:
+ self.targets.append([self.sbs_build_dir+'/'+fragment+"/"+x for x in t])
+ return
+
+# derived class for tests that invoke some process, which have no log file and no makefile
+# e.g. unit tests
+
+class GenericSmokeTest(SmokeTest):
+
+ def __init__(self):
+ SmokeTest.__init__(self)
+
+ def logfileOption(self):
+ return ""
+
+ def makefileOption(self):
+ return ""
+
+ def posttest(self):
+ # dump the standard output to a log file
+ dir = ReplaceEnvs(logDir)
+ logfile = os.path.join(dir, self.name + ".log")
+ try:
+ if not os.path.exists(dir):
+ os.makedirs(dir)
+ file = open(logfile, "w")
+ file.write(self.output)
+ file.close()
+ except:
+ print "Could not save stdout in", logfile
+ return False
+
+ # do the base class things too
+ return SmokeTest.posttest(self)
+
+# derived class for --check, --what and .whatlog tests - these all write to stdout, but may
+# not actually build anything
+
+class CheckWhatSmokeTest(SmokeTest):
+
+ def __init__(self):
+ SmokeTest.__init__(self)
+
+ # regular expression match object to restrict comparisons to specific lines
+ self.regexlinefilter = None
+
+ # paths in --what output are tailored to the host OS, hence slashes are converted appropriately
+ # .whatlog output is used verbatim from the build/TEM/EM output
+ self.hostossensitive = True
+
+ def posttest(self):
+ outlines = self.output.splitlines()
+
+ ok = True
+ seen = []
+
+ # check for lines that we expected to see, optionally filtered
+ for line in self.stdout:
+ if self.regexlinefilter and not self.regexlinefilter.match(line):
+ continue
+ line = ReplaceEnvs(line)
+ if self.hostossensitive and self.onWindows:
+ line = line.replace("/", "\\")
+
+ if line in outlines:
+ seen.append(line)
+ else:
+ print "OUTPUT NOT FOUND:", line
+ ok = False
+
+ # and check for extra lines that we didn't expect, optionally filtered
+ for line in outlines:
+ if self.regexlinefilter and not self.regexlinefilter.match(line):
+ continue
+ if not line in seen:
+ print "UNEXPECTED OUTPUT:", line
+ ok = False
+
+ # do the base class things too
+ return (SmokeTest.posttest(self) and ok)
+
+# derived class for tests that also need to make sure that certain files
+# are NOT created - sort of anti-targets.
+
+class AntiTargetSmokeTest(SmokeTest):
+
+ def __init__(self):
+ SmokeTest.__init__(self)
+ self.antitargets = []
+
+ def pretest(self):
+ """ Prepare for the test """
+ # parent pretest first
+ ok = SmokeTest.pretest(self)
+
+ # remove all the anti-target files
+ return (self.removeFiles(self.antitargets) and ok)
+
+ def posttest(self):
+ """ look for antitargets """
+ ok = True
+ for t in self.antitargets:
+ tgt = os.path.normpath(ReplaceEnvs(t))
+ if os.path.exists(tgt):
+ print "UNWANTED", tgt
+ ok = False
+
+ # do the base class things too
+ return (SmokeTest.posttest(self) and ok)
+
+ def addbuildantitargets(self, bldinfsourcepath, targetsuffixes):
+ """Add targets that are under epoc32/build whose path
+ can change based on an md5 hash of the path to the bld.inf.
+ """
+
+ fragment = BldInfFile.outputPathFragment(bldinfsourcepath)
+
+ for t in targetsuffixes:
+ if type(t) is not list:
+ newt="$(EPOCROOT)/epoc32/build/"+fragment+"/"+t
+ self.antitargets.append(newt)
+ else:
+ self.antitargets.append(["$(EPOCROOT)/epoc32/build/"+fragment+"/"+x for x in t])
+ return
+
+# the end