diff -r 000000000000 -r ca70ae20a155 src/tools/regrtest_log_to_cruisecontrol.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/tools/regrtest_log_to_cruisecontrol.py Tue Feb 16 10:07:05 2010 +0530 @@ -0,0 +1,462 @@ +# Copyright (c) 2008 Nokia Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script parses the output of the regrtest.py script and +# transforms it into the XML format suitable for the CruiseControl web +# interface. + +import re +import sys +import os +import pprint + +expected_skip_list = ['test_aepack', + 'test_al', + 'test_applesingle', + 'test_cd', + 'test_cl', + 'test_cmd_line', + 'test_commands', + 'test_crypt', + 'test_ctypes', + 'test_hotshot', + 'test_plistlib', + 'test_sundry', + 'test_bsddb', + 'test_bsddb185', + 'test_bsddb3', + 'test_bz2', + 'test_dbm', + 'test_gdbm', + 'test_gl', + 'test_imageop', + 'test_rgbimg', + 'test_audioop', + 'test_gettext', + 'test_curses', + 'test_dl', + 'test_fork1', + 'test_grp', + 'test_imgfile', + 'test_ioctl', + 'test_largefile', + 'test_linuxaudiodev', + 'test_macfs', + 'test_macostools', + 'test_macpath', + 'test_mhlib', + 'test_mmap', + 'test_nis', + 'test_openpty', + 'test_ossaudiodev', + 'test_pep277', + 'test_poll', + 'test_popen', + 'test_popen2', + 'test_pty', + 'test_pwd', + 'test_resource', + 'test_scriptpackages', + 'test_signal', + 'test_startfile', + 'test_sqlite', + 'test_subprocess', + 'test_sunaudiodev', + 'test_tcl', + 'test_threadsignals', + 'test_wait3', + 'test_wait4', + 'test_winreg', + 'test_winsound', + 'test_zipfile64'] + + +def replaceXMLentities(s): + s = s.replace('&', '&') + s = s.replace('<', '<') + s = s.replace('>', '>') + s = s.replace("'", ''') + s = s.replace('"', '"') + + def replace_non_printable(obj): + if ord(obj.group(0)) > 31 and ord(obj.group(0)) < 126: + return obj.group(0) + else: + return '?' + return re.sub('[^ a-zA-Z0-9\n]', replace_non_printable, s) + +# This regexp matches both the plain test start header "test_foo" and +# the "Re-running test 'test_foo' in verbose mode" header. +re_run = re.compile("^(Re-running test '|)(test_[^ \n]+)(' in verbose mode|)$") +# Matches the end of test outputs and beginning of reporting +re_finish = re.compile("^[0-9]+ tests (OK.|skipped:)$") +re_unexpected = re.compile("test (test_.*) produced unexpected output:(.*)") +re_skipped = re.compile("(test_.*) skipped --") +re_crashed = re.compile("test (test_.*) crashed --") +re_failed = re.compile("test (test_.*) failed --") +re_time = re.compile("Ran ([0-9]+) test[s]{0,1} in (.*)s") +re_unexp_skips = re.compile("(.*) skip[s]{0,1} unexpected on") +re_sis_build_time = re.compile("Sis build time :(.*)") + +# Build information that has to be displayed on CC should be in the +# regrtest_emu log in this format +re_build_info = re.compile("Build Info -- Name : <(.*)>, Value : <(.*)>") + +# A metric should be printed to regrtest log in the format : +# "Measurement -- Name : <>, Value : <>, Unit : <>, Threshold : <>, +# higher_is_better : <>" +# Note : All special characters in the above string are compulsary +# Threshold should always be a positive number +# higher_is_better should be either 'yes' or 'no' +re_measurement = re.compile("Measurement -- Name : <(.*)>, Value : <(.*)>," + + " Unit : <(.*)>, Threshold : <(.*)>, higher_is_better : <(.*)>") + +new_state = old_state = {'passed': [], + 'failed': [], + 'skipped_expected': [], + 'skipped_unexpected': []} +changeset = {} +results = {} +current_case = None +unexp_skips = None +unexp_skip_list = [] +measurements = {} +sis_build_time = '' +dev_metrics_log = None +build_info = {} + +# Remove 'regrtest_' & '_xxx.log' from the log name +# Some sample target names : aalto, emulator, merlin +target_name = sys.argv[1][9:-8] + +# regrtest_xxx.log and the corresponding xml file is placed here. +base_dir = 'build\\test' + +log_directory = 'C:\\Program Files\\CruiseControl\\logs' +if not os.path.exists(log_directory): + os.makedirs(log_directory) +dev_metrics_log_file = os.path.join(log_directory, 'ngopy_metrics.log') + +# sys.argv[1] is the log filename passed when calling this script +# Eg: regrtest_aino_3_1.log, regrtest_merlin_3_2.log +regrtest_log = open(os.path.join(base_dir, sys.argv[1]), 'rt') +# XML filenames will be -- aalto_resuls.xml, merlin_results.xml etc. +regrtest_xml = open(os.path.join(base_dir, target_name + '_results.xml'), 'w') + +# state_file contains the state (passed/failed/skipped) of each test case +# in the previous build +try: + state_file = open(os.path.join(log_directory, 'state_' + + target_name + '.txt'), 'rt') +except IOError: + state_file = None + +# Logging of metrics not required for emulator and linux builds +if target_name not in ['linux', 'emulator']: + dev_metrics_log = open(dev_metrics_log_file, 'a+') + +# First split output file based on the case names +for line in regrtest_log: + if re_finish.match(line): + current_case=None + continue + m = re_run.match(line) + if m: + # beginning of processing for new case + case_name = m.group(2) + assert case_name + if case_name != 'test_pystone': + current_case = {'time': "1", 'output': []} + results[case_name] = current_case + else: + x1 = re_time.match(line) + if x1: + current_case['time'] = x1.group(2) + if current_case: + current_case['output'].append(line) + + sis_build_time_match = re_sis_build_time.match(line) + if sis_build_time_match: + sis_build_time = sis_build_time_match.group(1) + continue + + re_build_info_match = re_build_info.match(line) + if re_build_info_match: + build_info[re_build_info_match.group(1)] = \ + re_build_info_match.group(2) + continue + + re_measurement_match = re_measurement.match(line) + if re_measurement_match: + measurements[re_measurement_match.group(1)] = {} + measurements[re_measurement_match.group(1)]['value'] = \ + float(re_measurement_match.group(2)) + measurements[re_measurement_match.group(1)]['unit'] = \ + re_measurement_match.group(3) + measurements[re_measurement_match.group(1)]['threshold'] = \ + float(re_measurement_match.group(4)) + measurements[re_measurement_match.group(1)]['higher_is_better'] = \ + re_measurement_match.group(5) + measurements[re_measurement_match.group(1)]['direction'] = 'Neutral' + measurements[re_measurement_match.group(1)]['delta'] = 0 + continue + # For linux we believe regrtest, but for emu and devices we decide what is + # expected and unexpected skip (refer : expected_skip_list) + if target_name == 'linux': + if unexp_skips: + unexp_skip_list.extend(line.strip().rsplit(' ')) + unexp_skips=None + unexp_skips = re_unexp_skips.match(line) +regrtest_log.close() + +# Analyze further to determine result from the test +count_unexpected = 0 +count_skipped = 0 +count_crashed = 0 +count_failed = 0 +count_passed = 0 +count_skipped_unexpected = 0 +count_skipped_expected = 0 + +for case_name, result in results.items(): + out = ''.join(result['output']) + result['output'] = out + # if others don't match, the case is assumed to be passed + if re_unexpected.search(out): + result['state'] = 'unexpected_output' + count_unexpected += 1 + elif re_skipped.search(out): + result['state'] = 'skipped' + count_skipped += 1 + elif re_crashed.search(out): + result['state'] = 'crashed' + count_crashed += 1 + elif re_failed.search(out): + result['state'] = 'failed' + count_failed += 1 + else: + result['state'] = 'passed' + count_passed += 1 + +# Report results +print "Full results:" +pprint.pprint(results) +states = set(results[x]['state'] for x in results) +print "Summary:" +for state in states: + cases = [x for x in results if results[x]['state'] == state] + print "%d %s: %s" % (len(cases), state, ' '.join(cases)) + +total_cases = count_unexpected + count_skipped + count_crashed + \ + count_failed + count_passed +failed_testcases = count_unexpected + count_crashed + count_failed + +testcase_metric_names = ['Number of Failed Test Cases', +'Number of Skipped Test Cases - Unexpected', +'Number of Successful Test Cases', +'Number of Skipped Test Cases - Expected'] + +# Initialize the 'measurements' dictionary related data for test case counts +# and then dump values. skipped_expected_count and skipped_unexpected_count +# are assigned later as our expected and unexpected list is not the same as the +# one calculated by regrtest in the log. +for item in testcase_metric_names: + measurements[item] = {} + measurements[item]['value'] = 0 + measurements[item]['threshold'] = 1 + if item == 'Number of Successful Test Cases': + measurements[item]['higher_is_better'] = 'yes' + else: + measurements[item]['higher_is_better'] = 'no' + measurements[item]['delta'] = 0 + measurements[item]['direction'] = 'Neutral' + measurements[item]['unit'] = '' + +measurements['Number of Failed Test Cases']['value'] = failed_testcases +measurements['Number of Successful Test Cases']['value'] = count_passed + +regrtest_xml.write('\n') +regrtest_xml.write('\n') +regrtest_xml.write(' \n' % {'target': target_name, + 'total_cases': total_cases, 'time': sis_build_time}) + +if state_file: + old_state = eval(state_file.read()) + state_file.close() + + +def check_state_change(testcase, state): + if testcase not in old_state[state]: + return 'yes' + else: + return 'no' + +# Each testcase has a 'new' attribute to indicate if it moved to this +# state in this build +for i in sorted(results.keys()): + if ((results[i]['state'] == 'failed') or + (results[i]['state'] == 'unexpected') or + (results[i]['state'] == 'crashed')): + state_changed = check_state_change(i, 'failed') + regrtest_xml.write(' ' + + replaceXMLentities(results[i]['output']) + + '\n') + new_state['failed'].append(i) + +for i in sorted(results.keys()): + if (results[i]['state'] == 'skipped'): + if i not in expected_skip_list and target_name != 'linux': + unexp_skip_list.append(i) + if i in unexp_skip_list: + state_changed = check_state_change(i, 'skipped_unexpected') + regrtest_xml.write(' ' + + replaceXMLentities(results[i]['output']) + + '\n') + new_state['skipped_unexpected'].append(i) + count_skipped_unexpected += 1 + +for i in sorted(results.keys()): + if (results[i]['state'] == 'passed'): + state_changed = check_state_change(i, 'passed') + regrtest_xml.write(' ' + + replaceXMLentities(results[i]['output']) + + '\n') + new_state['passed'].append(i) + +for i in sorted(results.keys()): + if (results[i]['state'] == 'skipped'): + if i not in unexp_skip_list: + state_changed = check_state_change(i, 'skipped_expected') + regrtest_xml.write(' ' + + replaceXMLentities(results[i]['output']) + + '\n') + new_state['skipped_expected'].append(i) + count_skipped_expected += 1 + +measurements['Number of Skipped Test Cases - Expected']['value'] = \ + count_skipped_expected +measurements['Number of Skipped Test Cases - Unexpected']['value'] = \ + count_skipped_unexpected + +# 'measurements' dictionary's 'delta' will contain the difference in metrics +# w.r.t the previous run. 'delta' can be positive or negative +for item in measurements.keys(): + new_state[item] = measurements[item]['value'] + if item in old_state: + measurements[item]['delta'] = new_state[item] - old_state[item] + +# Identify the test cases which have changed states (Eg. Failed->Passed ) +for state in ['failed', 'passed', 'skipped_expected', 'skipped_unexpected']: + s1 = set(new_state[state]) + s2 = set(old_state[state]) + s1.difference_update(s2) + changeset[state] = list(s1) + +# Adding the test cases that have changed states as a measurement and also in +# state_changes tag. 'threshold' and 'direction' are omitted as color coding +# in CC is not necessary. +regrtest_xml.write(' \n') +for state in ['failed', 'passed', 'skipped_expected', 'skipped_unexpected']: + regrtest_xml.write(' <' + state + ' count="' + + str(len(changeset[state])) + '">\n') + regrtest_xml.write(repr(changeset[state])) + regrtest_xml.write(' \n') + if changeset[state]: + measurements[state.capitalize() + ' - New'] = {} + measurements[state.capitalize() + ' - New']['value'] = \ + repr(changeset[state]) + measurements[state.capitalize() + ' - New']['delta'] = 0 + measurements[state.capitalize() + ' - New']['unit'] = '' +regrtest_xml.write(' \n') + +regrtest_xml.write(' \n') +# direction - dictates the color coding in CruiseControl. it will be set only +# when the delta crosses threshold. direction can be 'Good', 'Bad', 'Neutral' +for item in measurements.keys(): + if measurements[item]['delta'] == 0: + measurements[item]['direction'] = 'Neutral' + elif measurements[item]['higher_is_better'] == 'yes': + if measurements[item]['delta'] >= measurements[item]['threshold']: + measurements[item]['direction'] = 'Good' + elif measurements[item]['delta'] <= -(measurements[item]['threshold']): + measurements[item]['direction'] = 'Bad' + else: + if measurements[item]['delta'] >= measurements[item]['threshold']: + measurements[item]['direction'] = 'Bad' + elif measurements[item]['delta'] <= -(measurements[item]['threshold']): + measurements[item]['direction'] = 'Good' + + +def write_measurement(item, log_metrics=False): + regrtest_xml.write(' \n') + regrtest_xml.write(' ' + item + '\n') + regrtest_xml.write(' ' + + str(measurements[item]['value']) + ' ' + + str(measurements[item]['unit']) + '\n') + regrtest_xml.write(' ' + + measurements[item]['direction'] + '\n') + if measurements[item]['delta']: + regrtest_xml.write(' ' + + str(measurements[item]['delta']) + '\n') + regrtest_xml.write(' \n') + + # Update the device specific log file, which is used to draw metrics graph + if dev_metrics_log and log_metrics: + dev_metrics_log.write(',%s=%s' % + (item, str(measurements[item]['value']))) + +# Write the testcase related metrics first and then the memory/time benchmark +# related data so that CC also displays in that order +s = set(measurements.keys()) +for item in testcase_metric_names: + write_measurement(item) +s.difference_update(set(testcase_metric_names)) + +if dev_metrics_log: + dev_metrics_log.write('Device=' + target_name) + dev_metrics_log.write(',Time=' + sis_build_time) + +for item in s: + write_measurement(item, True) +regrtest_xml.write(' \n') + +regrtest_xml.write(' \n') +for item in build_info.keys(): + regrtest_xml.write(' \n') + regrtest_xml.write(' \n') + regrtest_xml.write(item) + regrtest_xml.write(' \n') + regrtest_xml.write(' \n') + regrtest_xml.write(str(build_info[item]) + '\n') + regrtest_xml.write(' \n') + regrtest_xml.write(' \n') +regrtest_xml.write(' \n') + +if dev_metrics_log: + dev_metrics_log.write('\n') + dev_metrics_log.close() + +regrtest_xml.write(' \n') +state_file = open(os.path.join(log_directory, 'state_' + + target_name + '.txt'), 'wt') +state_file.write(repr(new_state)) +state_file.close() +regrtest_xml.write('\n') +regrtest_xml.write('') +regrtest_xml.close()