--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/tools/regrtest_log_to_cruisecontrol.py Tue Feb 16 10:07:05 2010 +0530
@@ -0,0 +1,462 @@
+# Copyright (c) 2008 Nokia Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script parses the output of the regrtest.py script and
+# transforms it into the XML format suitable for the CruiseControl web
+# interface.
+
+import re
+import sys
+import os
+import pprint
+
+expected_skip_list = ['test_aepack',
+ 'test_al',
+ 'test_applesingle',
+ 'test_cd',
+ 'test_cl',
+ 'test_cmd_line',
+ 'test_commands',
+ 'test_crypt',
+ 'test_ctypes',
+ 'test_hotshot',
+ 'test_plistlib',
+ 'test_sundry',
+ 'test_bsddb',
+ 'test_bsddb185',
+ 'test_bsddb3',
+ 'test_bz2',
+ 'test_dbm',
+ 'test_gdbm',
+ 'test_gl',
+ 'test_imageop',
+ 'test_rgbimg',
+ 'test_audioop',
+ 'test_gettext',
+ 'test_curses',
+ 'test_dl',
+ 'test_fork1',
+ 'test_grp',
+ 'test_imgfile',
+ 'test_ioctl',
+ 'test_largefile',
+ 'test_linuxaudiodev',
+ 'test_macfs',
+ 'test_macostools',
+ 'test_macpath',
+ 'test_mhlib',
+ 'test_mmap',
+ 'test_nis',
+ 'test_openpty',
+ 'test_ossaudiodev',
+ 'test_pep277',
+ 'test_poll',
+ 'test_popen',
+ 'test_popen2',
+ 'test_pty',
+ 'test_pwd',
+ 'test_resource',
+ 'test_scriptpackages',
+ 'test_signal',
+ 'test_startfile',
+ 'test_sqlite',
+ 'test_subprocess',
+ 'test_sunaudiodev',
+ 'test_tcl',
+ 'test_threadsignals',
+ 'test_wait3',
+ 'test_wait4',
+ 'test_winreg',
+ 'test_winsound',
+ 'test_zipfile64']
+
+
+def replaceXMLentities(s):
+ s = s.replace('&', '&')
+ s = s.replace('<', '<')
+ s = s.replace('>', '>')
+ s = s.replace("'", ''')
+ s = s.replace('"', '"')
+
+ def replace_non_printable(obj):
+ if ord(obj.group(0)) > 31 and ord(obj.group(0)) < 126:
+ return obj.group(0)
+ else:
+ return '?'
+ return re.sub('[^ a-zA-Z0-9\n]', replace_non_printable, s)
+
+# This regexp matches both the plain test start header "test_foo" and
+# the "Re-running test 'test_foo' in verbose mode" header.
+re_run = re.compile("^(Re-running test '|)(test_[^ \n]+)(' in verbose mode|)$")
+# Matches the end of test outputs and beginning of reporting
+re_finish = re.compile("^[0-9]+ tests (OK.|skipped:)$")
+re_unexpected = re.compile("test (test_.*) produced unexpected output:(.*)")
+re_skipped = re.compile("(test_.*) skipped --")
+re_crashed = re.compile("test (test_.*) crashed --")
+re_failed = re.compile("test (test_.*) failed --")
+re_time = re.compile("Ran ([0-9]+) test[s]{0,1} in (.*)s")
+re_unexp_skips = re.compile("(.*) skip[s]{0,1} unexpected on")
+re_sis_build_time = re.compile("Sis build time :(.*)")
+
+# Build information that has to be displayed on CC should be in the
+# regrtest_emu log in this format
+re_build_info = re.compile("Build Info -- Name : <(.*)>, Value : <(.*)>")
+
+# A metric should be printed to regrtest log in the format :
+# "Measurement -- Name : <>, Value : <>, Unit : <>, Threshold : <>,
+# higher_is_better : <>"
+# Note : All special characters in the above string are compulsary
+# Threshold should always be a positive number
+# higher_is_better should be either 'yes' or 'no'
+re_measurement = re.compile("Measurement -- Name : <(.*)>, Value : <(.*)>," +
+ " Unit : <(.*)>, Threshold : <(.*)>, higher_is_better : <(.*)>")
+
+new_state = old_state = {'passed': [],
+ 'failed': [],
+ 'skipped_expected': [],
+ 'skipped_unexpected': []}
+changeset = {}
+results = {}
+current_case = None
+unexp_skips = None
+unexp_skip_list = []
+measurements = {}
+sis_build_time = ''
+dev_metrics_log = None
+build_info = {}
+
+# Remove 'regrtest_' & '_xxx.log' from the log name
+# Some sample target names : aalto, emulator, merlin
+target_name = sys.argv[1][9:-8]
+
+# regrtest_xxx.log and the corresponding xml file is placed here.
+base_dir = 'build\\test'
+
+log_directory = 'C:\\Program Files\\CruiseControl\\logs'
+if not os.path.exists(log_directory):
+ os.makedirs(log_directory)
+dev_metrics_log_file = os.path.join(log_directory, 'ngopy_metrics.log')
+
+# sys.argv[1] is the log filename passed when calling this script
+# Eg: regrtest_aino_3_1.log, regrtest_merlin_3_2.log
+regrtest_log = open(os.path.join(base_dir, sys.argv[1]), 'rt')
+# XML filenames will be -- aalto_resuls.xml, merlin_results.xml etc.
+regrtest_xml = open(os.path.join(base_dir, target_name + '_results.xml'), 'w')
+
+# state_file contains the state (passed/failed/skipped) of each test case
+# in the previous build
+try:
+ state_file = open(os.path.join(log_directory, 'state_' +
+ target_name + '.txt'), 'rt')
+except IOError:
+ state_file = None
+
+# Logging of metrics not required for emulator and linux builds
+if target_name not in ['linux', 'emulator']:
+ dev_metrics_log = open(dev_metrics_log_file, 'a+')
+
+# First split output file based on the case names
+for line in regrtest_log:
+ if re_finish.match(line):
+ current_case=None
+ continue
+ m = re_run.match(line)
+ if m:
+ # beginning of processing for new case
+ case_name = m.group(2)
+ assert case_name
+ if case_name != 'test_pystone':
+ current_case = {'time': "1", 'output': []}
+ results[case_name] = current_case
+ else:
+ x1 = re_time.match(line)
+ if x1:
+ current_case['time'] = x1.group(2)
+ if current_case:
+ current_case['output'].append(line)
+
+ sis_build_time_match = re_sis_build_time.match(line)
+ if sis_build_time_match:
+ sis_build_time = sis_build_time_match.group(1)
+ continue
+
+ re_build_info_match = re_build_info.match(line)
+ if re_build_info_match:
+ build_info[re_build_info_match.group(1)] = \
+ re_build_info_match.group(2)
+ continue
+
+ re_measurement_match = re_measurement.match(line)
+ if re_measurement_match:
+ measurements[re_measurement_match.group(1)] = {}
+ measurements[re_measurement_match.group(1)]['value'] = \
+ float(re_measurement_match.group(2))
+ measurements[re_measurement_match.group(1)]['unit'] = \
+ re_measurement_match.group(3)
+ measurements[re_measurement_match.group(1)]['threshold'] = \
+ float(re_measurement_match.group(4))
+ measurements[re_measurement_match.group(1)]['higher_is_better'] = \
+ re_measurement_match.group(5)
+ measurements[re_measurement_match.group(1)]['direction'] = 'Neutral'
+ measurements[re_measurement_match.group(1)]['delta'] = 0
+ continue
+ # For linux we believe regrtest, but for emu and devices we decide what is
+ # expected and unexpected skip (refer : expected_skip_list)
+ if target_name == 'linux':
+ if unexp_skips:
+ unexp_skip_list.extend(line.strip().rsplit(' '))
+ unexp_skips=None
+ unexp_skips = re_unexp_skips.match(line)
+regrtest_log.close()
+
+# Analyze further to determine result from the test
+count_unexpected = 0
+count_skipped = 0
+count_crashed = 0
+count_failed = 0
+count_passed = 0
+count_skipped_unexpected = 0
+count_skipped_expected = 0
+
+for case_name, result in results.items():
+ out = ''.join(result['output'])
+ result['output'] = out
+ # if others don't match, the case is assumed to be passed
+ if re_unexpected.search(out):
+ result['state'] = 'unexpected_output'
+ count_unexpected += 1
+ elif re_skipped.search(out):
+ result['state'] = 'skipped'
+ count_skipped += 1
+ elif re_crashed.search(out):
+ result['state'] = 'crashed'
+ count_crashed += 1
+ elif re_failed.search(out):
+ result['state'] = 'failed'
+ count_failed += 1
+ else:
+ result['state'] = 'passed'
+ count_passed += 1
+
+# Report results
+print "Full results:"
+pprint.pprint(results)
+states = set(results[x]['state'] for x in results)
+print "Summary:"
+for state in states:
+ cases = [x for x in results if results[x]['state'] == state]
+ print "%d %s: %s" % (len(cases), state, ' '.join(cases))
+
+total_cases = count_unexpected + count_skipped + count_crashed + \
+ count_failed + count_passed
+failed_testcases = count_unexpected + count_crashed + count_failed
+
+testcase_metric_names = ['Number of Failed Test Cases',
+'Number of Skipped Test Cases - Unexpected',
+'Number of Successful Test Cases',
+'Number of Skipped Test Cases - Expected']
+
+# Initialize the 'measurements' dictionary related data for test case counts
+# and then dump values. skipped_expected_count and skipped_unexpected_count
+# are assigned later as our expected and unexpected list is not the same as the
+# one calculated by regrtest in the log.
+for item in testcase_metric_names:
+ measurements[item] = {}
+ measurements[item]['value'] = 0
+ measurements[item]['threshold'] = 1
+ if item == 'Number of Successful Test Cases':
+ measurements[item]['higher_is_better'] = 'yes'
+ else:
+ measurements[item]['higher_is_better'] = 'no'
+ measurements[item]['delta'] = 0
+ measurements[item]['direction'] = 'Neutral'
+ measurements[item]['unit'] = ''
+
+measurements['Number of Failed Test Cases']['value'] = failed_testcases
+measurements['Number of Successful Test Cases']['value'] = count_passed
+
+regrtest_xml.write('<xml>\n')
+regrtest_xml.write('<testsuites>\n')
+regrtest_xml.write(' <testsuite_%(target)s name="testcases_%(target)s"\
+ tests="%(total_cases)s" time="%(time)s" >\n' % {'target': target_name,
+ 'total_cases': total_cases, 'time': sis_build_time})
+
+if state_file:
+ old_state = eval(state_file.read())
+ state_file.close()
+
+
+def check_state_change(testcase, state):
+ if testcase not in old_state[state]:
+ return 'yes'
+ else:
+ return 'no'
+
+# Each testcase has a 'new' attribute to indicate if it moved to this
+# state in this build
+for i in sorted(results.keys()):
+ if ((results[i]['state'] == 'failed') or
+ (results[i]['state'] == 'unexpected') or
+ (results[i]['state'] == 'crashed')):
+ state_changed = check_state_change(i, 'failed')
+ regrtest_xml.write(' <testcase name="' + i + '" time="1" ' +
+ 'new="' + state_changed + '"> <failure>' +
+ replaceXMLentities(results[i]['output']) +
+ '</failure></testcase>\n')
+ new_state['failed'].append(i)
+
+for i in sorted(results.keys()):
+ if (results[i]['state'] == 'skipped'):
+ if i not in expected_skip_list and target_name != 'linux':
+ unexp_skip_list.append(i)
+ if i in unexp_skip_list:
+ state_changed = check_state_change(i, 'skipped_unexpected')
+ regrtest_xml.write(' <testcase name="' + i + '" time="1" ' +
+ 'new="' + state_changed + '"> <skipped_unexpected>' +
+ replaceXMLentities(results[i]['output']) +
+ '</skipped_unexpected></testcase>\n')
+ new_state['skipped_unexpected'].append(i)
+ count_skipped_unexpected += 1
+
+for i in sorted(results.keys()):
+ if (results[i]['state'] == 'passed'):
+ state_changed = check_state_change(i, 'passed')
+ regrtest_xml.write(' <testcase name="' + i + '" time="1" ' +
+ 'new="' + state_changed + '"> <success>' +
+ replaceXMLentities(results[i]['output']) +
+ '</success></testcase>\n')
+ new_state['passed'].append(i)
+
+for i in sorted(results.keys()):
+ if (results[i]['state'] == 'skipped'):
+ if i not in unexp_skip_list:
+ state_changed = check_state_change(i, 'skipped_expected')
+ regrtest_xml.write(' <testcase name="' + i + '" time="1" ' +
+ 'new="' + state_changed + '"> <skipped_expected>' +
+ replaceXMLentities(results[i]['output']) +
+ '</skipped_expected></testcase>\n')
+ new_state['skipped_expected'].append(i)
+ count_skipped_expected += 1
+
+measurements['Number of Skipped Test Cases - Expected']['value'] = \
+ count_skipped_expected
+measurements['Number of Skipped Test Cases - Unexpected']['value'] = \
+ count_skipped_unexpected
+
+# 'measurements' dictionary's 'delta' will contain the difference in metrics
+# w.r.t the previous run. 'delta' can be positive or negative
+for item in measurements.keys():
+ new_state[item] = measurements[item]['value']
+ if item in old_state:
+ measurements[item]['delta'] = new_state[item] - old_state[item]
+
+# Identify the test cases which have changed states (Eg. Failed->Passed )
+for state in ['failed', 'passed', 'skipped_expected', 'skipped_unexpected']:
+ s1 = set(new_state[state])
+ s2 = set(old_state[state])
+ s1.difference_update(s2)
+ changeset[state] = list(s1)
+
+# Adding the test cases that have changed states as a measurement and also in
+# state_changes tag. 'threshold' and 'direction' are omitted as color coding
+# in CC is not necessary.
+regrtest_xml.write(' <state_changes>\n')
+for state in ['failed', 'passed', 'skipped_expected', 'skipped_unexpected']:
+ regrtest_xml.write(' <' + state + ' count="' +
+ str(len(changeset[state])) + '">\n')
+ regrtest_xml.write(repr(changeset[state]))
+ regrtest_xml.write(' </' + state + '>\n')
+ if changeset[state]:
+ measurements[state.capitalize() + ' - New'] = {}
+ measurements[state.capitalize() + ' - New']['value'] = \
+ repr(changeset[state])
+ measurements[state.capitalize() + ' - New']['delta'] = 0
+ measurements[state.capitalize() + ' - New']['unit'] = ''
+regrtest_xml.write(' </state_changes>\n')
+
+regrtest_xml.write(' <measurements>\n')
+# direction - dictates the color coding in CruiseControl. it will be set only
+# when the delta crosses threshold. direction can be 'Good', 'Bad', 'Neutral'
+for item in measurements.keys():
+ if measurements[item]['delta'] == 0:
+ measurements[item]['direction'] = 'Neutral'
+ elif measurements[item]['higher_is_better'] == 'yes':
+ if measurements[item]['delta'] >= measurements[item]['threshold']:
+ measurements[item]['direction'] = 'Good'
+ elif measurements[item]['delta'] <= -(measurements[item]['threshold']):
+ measurements[item]['direction'] = 'Bad'
+ else:
+ if measurements[item]['delta'] >= measurements[item]['threshold']:
+ measurements[item]['direction'] = 'Bad'
+ elif measurements[item]['delta'] <= -(measurements[item]['threshold']):
+ measurements[item]['direction'] = 'Good'
+
+
+def write_measurement(item, log_metrics=False):
+ regrtest_xml.write(' <measurement>\n')
+ regrtest_xml.write(' <name>' + item + '</name>\n')
+ regrtest_xml.write(' <value>' +
+ str(measurements[item]['value']) + ' ' +
+ str(measurements[item]['unit']) + '</value>\n')
+ regrtest_xml.write(' <direction>' +
+ measurements[item]['direction'] + '</direction>\n')
+ if measurements[item]['delta']:
+ regrtest_xml.write(' <delta>' +
+ str(measurements[item]['delta']) + '</delta>\n')
+ regrtest_xml.write(' </measurement>\n')
+
+ # Update the device specific log file, which is used to draw metrics graph
+ if dev_metrics_log and log_metrics:
+ dev_metrics_log.write(',%s=%s' %
+ (item, str(measurements[item]['value'])))
+
+# Write the testcase related metrics first and then the memory/time benchmark
+# related data so that CC also displays in that order
+s = set(measurements.keys())
+for item in testcase_metric_names:
+ write_measurement(item)
+s.difference_update(set(testcase_metric_names))
+
+if dev_metrics_log:
+ dev_metrics_log.write('Device=' + target_name)
+ dev_metrics_log.write(',Time=' + sis_build_time)
+
+for item in s:
+ write_measurement(item, True)
+regrtest_xml.write(' </measurements>\n')
+
+regrtest_xml.write(' <build_info>\n')
+for item in build_info.keys():
+ regrtest_xml.write(' <item>\n')
+ regrtest_xml.write(' <name>\n')
+ regrtest_xml.write(item)
+ regrtest_xml.write(' </name>\n')
+ regrtest_xml.write(' <value>\n')
+ regrtest_xml.write(str(build_info[item]) + '\n')
+ regrtest_xml.write(' </value>\n')
+ regrtest_xml.write(' </item>\n')
+regrtest_xml.write(' </build_info>\n')
+
+if dev_metrics_log:
+ dev_metrics_log.write('\n')
+ dev_metrics_log.close()
+
+regrtest_xml.write(' </testsuite_' + target_name + '>\n')
+state_file = open(os.path.join(log_directory, 'state_' +
+ target_name + '.txt'), 'wt')
+state_file.write(repr(new_state))
+state_file.close()
+regrtest_xml.write('</testsuites>\n')
+regrtest_xml.write('</xml>')
+regrtest_xml.close()