--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/buildframework/helium/sf/python/pythoncore/lib/log2xml.py Tue Apr 27 08:33:08 2010 +0300
@@ -0,0 +1,415 @@
+#============================================================================
+#Name : log2xml.py
+#Part of : Helium
+
+#Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
+#All rights reserved.
+#This component and the accompanying materials are made available
+#under the terms of the License "Eclipse Public License v1.0"
+#which accompanies this distribution, and is available
+#at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+#Initial Contributors:
+#Nokia Corporation - initial contribution.
+#
+#Contributors:
+#
+#Description:
+#===============================================================================
+
+""" Symbian log converter.
+"""
+import xml.dom.minidom
+import sys
+import os
+import re
+import codecs
+import time
+import datetime
+from xml.sax import make_parser
+from xml.sax.handler import ContentHandler
+from xml.sax.saxutils import escape
+
+
+DEFAULT_CONFIGURATION = {"FATAL": [r"mingw_make.exe"],
+ "ERROR": [r'^(?:(?:\s*\d+\)\s*)|(?:\s*\*\*\*\s*))ERROR:',
+ r"^MISSING:",
+ r"Error:\s+",
+ r"^Error:",
+ r"'.+' is not recognized as an internal or external command",
+ r"FLEXlm error:",
+ r"(ABLD|BLDMAKE) ERROR:",
+ r"FATAL ERROR\(S\):",
+ r"fatal error U1077",
+ r"warning U4010",
+ r"^make(?:\[\d+\])?\: \*\*\*",
+ r"^make(?:\[\d+\])?:\s+.*\s+not\s+remade",
+ r"\"(.*)\", line (\d+): (Error: +(.\d+.*?):.*)$",
+ r"error: ((Internal fault):.*)$",
+ r"Exception: [A-Z0-9_]+",
+ r"target .* given more than once in the same rule",
+ r"^ERROR:",
+ r"^ERROR EC\d+:",
+ r"^ERROR\t",
+ r"syntax error at line",],
+ "CRITICAL": [r"[Ww]arning:?\s+(#111-D|#1166-D|#117-D|#128-D|#1293-D|#1441-D|#170-D|#174-D|#175-D|#185-D|#186-D|#223-D|#231-D|#257-D|#284-D|#368-D|#414-D|#430-D|#47-D|#514-D|#546-D|#68-D|#69-D|#830-D|#940-D|#836-D|A1495E|L6318W|C2874W|C4127|C4355|C4530|C4702|C4786|LNK4049)"],
+ "WARNING": [r'\): Missing file:',
+ r'^(\d+\))?\s*WARNING:', r'^MAKEDEF WARNING:',
+ r'line \d+: Warning:', r':\s+warning\s+\w+:',
+ r"\\\\(.*?)\(\d+\)\s:\sWarning:\s\(\d+\)",
+ r"^(BLDMAKE |MAKEDEF )?WARNING:",
+ r"WARNING\(S\)",
+ r"\(\d+\) : warning C",
+ r"LINK : warning",
+ r":\d+: warning:",
+ r"\"(.*)\", line (\d+): (Warning: +(?!A1495E)(.\d+.*?):.*)$",
+ r"Usage Warning:",
+ r"mwld.exe:",
+ r"^Command line warning",
+ r"ERROR: bad relocation:",
+ r"^(\d+) warning",
+ r"EventType:\s+Error\s+Source:\s+SweepNT",
+ r"^WARN\t",
+ ],
+ "REMARK": [r"Command line warning D4025 : ",
+ r"^REMARK: ",
+ r"^EventType:\s+Error\s+Source:\s+GNU\s+Make",
+ r":\d+: warning: cannot find matching deallocation function",
+ r"((:\d+)*: note: )",
+ ],
+ "INFO": [r"^INFO:"]
+ }
+
+
+def find_priority(line, config):
+ keys = config.keys()
+ keys.reverse()
+ for category in keys:
+ for rule in config[category]:
+ if rule.search(line) != None:
+ return category.lower()
+ return "stdout"
+
+class Stack:
+ """ Bottomless stack. If empty just pop a default element. """
+
+ def __init__(self, default):
+ self.__default = default
+ self.__stack = []
+
+ def pop(self):
+ result = None
+ try:
+ result = self.__stack.pop()
+ except IndexError, e:
+ result = self.__default
+ return result
+
+ def push(self, item):
+ self.__stack.append(item)
+
+ def __len__(self):
+ return len(self.__stack)
+
+def to_cdata(text):
+ """ Cleanup string to match CDATA requiements.
+ These are the only allowed characters: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF].
+ """
+ result = ""
+ for c in list(text):
+ v = ord(c)
+ if v == 0x9 or v == 0xa or v == 0xd:
+ result += c
+ elif v>=0x20 and v <= 0xd7ff:
+ result += c
+ elif v>=0xe000 and v <= 0xfffd:
+ result += c
+ elif v>=0x10000 and v <= 0x10ffff:
+ result += c
+ else:
+ result += " "
+ return result
+
+class LogWriter(object):
+ """ XML Log writer. """
+
+ def __init__(self, stream, filename):
+ self.__stream = stream
+ self.__stream.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
+ self.__stream.write("<log filename=\"%s\">\n" % filename)
+ self.__stream.write("\t<build>\n")
+ self.__indent = "\t"
+ self.__intask = 0
+
+ def close(self):
+ # closing open tasks...
+ while self.__intask > 0:
+ self.close_task()
+ self.__stream.write("\t</build>\n")
+ self.__stream.write("</log>\n")
+ self.__stream.close()
+
+ def open_task(self, name):
+ self.__indent += "\t"
+ self.__intask += 1
+ self.__stream.write("%s<task name=\"%s\">\n" % (self.__indent, name))
+
+ def close_task(self):
+ if self.__intask > 0:
+ self.__intask -= 1
+ self.__stream.write("%s</task>\n" % (self.__indent))
+ self.__indent = self.__indent[:-1]
+
+ def message(self, priority, msg):
+ try:
+ acdata = to_cdata(msg.decode('utf-8', 'ignore'))
+ self.__stream.write("%s<message priority=\"%s\"><![CDATA[%s]]></message>\n" % (self.__indent+"\t", priority, acdata))
+ except UnicodeDecodeError, e:
+ print e
+
+
+
+def convert(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
+ """ Convert an input log into an XML log and write an outputfile. """
+
+ # Compiling the regexp
+ built_config = {}
+ for category in configuration.keys():
+ built_config[category] = []
+ for rule in configuration[category]:
+ built_config[category].append(re.compile(rule))
+
+ # Generating the XML log
+ log = open(inputfile, 'r')
+ olog = codecs.open(outputfile, 'w+', 'utf-8', errors='ignore')
+ xmllog = LogWriter(olog, inputfile)
+
+
+ match_finnished = re.compile(r"^===\s+.+\s+finished")
+ match_started = re.compile(r"^===\s+(.+)\s+started")
+ match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
+ match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
+ #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started')
+ #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished')
+ match_ant_target_start = re.compile(r'^([^\s=\[\]]+):$')
+ match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
+ symbian = False
+ ant_has_open_task = False
+ # looping
+ for line in log:
+
+ # matching Ant logging
+ if not symbian and match_ant_target_end.match(line):
+ xmllog.close_task()
+ ant_has_open_task = False
+ continue
+ elif not symbian and match_ant_target_start.match(line):
+ result = match_ant_target_start.match(line)
+ if result != None:
+ if ant_has_open_task:
+ xmllog.close_task()
+ ant_has_open_task = False
+ xmllog.open_task(result.group(1))
+ ant_has_open_task = True
+ continue
+ # matching Symbian logging
+ line = match_logger_component.sub(r'', line)
+ line = line.strip()
+ if line.startswith("++ Finished at"):
+ xmllog.close_task()
+ elif line.startswith("=== "):
+ if match_finnished.match(line):
+ xmllog.close_task()
+ else:
+ # This is a symbian log
+ symbian = True
+ result = match_component.match(line)
+ if result != None:
+ xmllog.open_task(result.group(2))
+ # === cenrep_s60_32 started
+ result = match_started.match(line)
+ if result != None:
+ xmllog.open_task(result.group(1))
+ else:
+ # Type?
+ priority = find_priority(line, built_config)
+ if (fulllogging or priority != 'stdout'):
+ xmllog.message(priority, line)
+ # end file
+ xmllog.close()
+
+def convert_old(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
+ """ Convert an input log into an XML log and write an outputfile. """
+
+ # Compiling the regexp
+ built_config = {}
+ for category in configuration.keys():
+ built_config[category] = []
+ for rule in configuration[category]:
+ built_config[category].append(re.compile(rule))
+
+ # Generating the XML log
+ log = open (inputfile, 'r')
+ doc = xml.dom.minidom.Document()
+ root = doc.createElementNS("", "log")
+ root.setAttributeNS("", "name", inputfile)
+ doc.appendChild(root)
+ build = doc.createElementNS("", "build")
+ root.appendChild(build)
+ # current group/task
+ current = build
+ # bottomless stask, if losing sync all message will be at top level.
+ stack = Stack(build)
+
+ match_finnished = re.compile(r"^===\s+.+\s+finished")
+ match_started = re.compile(r"===\s+(.+)\s+started")
+ match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
+ match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
+ #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started')
+ #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished')
+ match_ant_target_start = re.compile(r'^([^\s=]+):$')
+ match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
+ # looping
+ for line in log:
+
+ # matching Ant logging
+ if match_ant_target_end.match(line):
+ current = stack.pop()
+ continue
+ elif match_ant_target_start.match(line):
+ result = match_ant_target_start.match(line)
+ if result != None:
+ stack.push(current)
+ task = doc.createElementNS("", "task")
+ task.setAttributeNS("", "name", result.group(1))
+ current.appendChild(task)
+ current = task
+ continue
+ # matching Symbian logging
+ line = match_logger_component.sub(r'', line)
+ line = line.strip()
+ if line.startswith("++ Finished at"):
+ current = stack.pop()
+ elif line.startswith("==="):
+ if match_finnished.match(line):
+ current = stack.pop()
+ else:
+ result = match_component.match(line)
+ if result != None:
+ stack.push(current)
+ task = doc.createElementNS("", "task")
+ task.setAttributeNS("", "name", result.group(2))
+ current.appendChild(task)
+ current = task
+ # === cenrep_s60_32 started
+ result = match_started.match(line)
+ if result != None:
+ task = doc.createElementNS("", "task")
+ task.setAttributeNS("", "name", result.group(1))
+ stack.push(current)
+ current.appendChild(task)
+ current = task
+ else:
+ msg = doc.createElementNS("", "message")
+ # Type?
+ priority = find_priority(line, built_config)
+ if (fulllogging or priority != 'stdout'):
+ msg.setAttributeNS("", "priority", priority)
+ msg.appendChild(doc.createCDATASection(to_cdata(line.decode("utf-8"))))
+ current.appendChild(msg)
+
+ file_object = codecs.open(outputfile, 'w', "utf_8")
+ file_object.write(doc.toprettyxml())
+ file_object.close()
+
+class ContentWriter(ContentHandler):
+ """ SAX Content writer. Parse and write an XML file. """
+ def __init__(self, os, indent=""):
+ self.os = os
+ self.indent = indent
+ self.__content = u""
+
+ def startElement(self, name, attrs):
+ self.os.write(self.indent + "<" + name)
+ if attrs.getLength() > 0:
+ self.os.write(" ")
+ self.os.write(" ".join(map(lambda x: "%s=\"%s\"" % (x, attrs.getValue(x)), attrs.getNames())))
+ self.os.write(">\n")
+ self.indent += "\t"
+ self.__content = ""
+
+ def endElement(self, name):
+ if len(self.__content) > 0:
+ self.os.write(self.indent + self.__content + "\n")
+ self.indent = self.indent[:-1]
+ self.os.write("%s</%s>\n" % (self.indent, name))
+ self.__content = ""
+
+ def characters(self, content):
+ self.__content += unicode(escape(content.strip()))
+
+class AppendSummary(ContentWriter):
+ """ SAX content handler to add an XML log to the summary. """
+ def __init__(self, output, xmllog):
+ ContentWriter.__init__(self, output)
+ self.xmllog = xmllog
+
+ def startDocument(self):
+ self.os.write('<?xml version="1.0" encoding="utf-8"?>\n')
+
+ def startElement(self, name, attrs):
+ ContentWriter.startElement(self, name, attrs)
+ if name == "logSummary":
+ parser = make_parser()
+ parser.setContentHandler(ContentWriter(self.os, self.indent))
+ parser.parse(open(self.xmllog, 'r'))
+
+
+def append_summary(summary, xmllog, maxmb=80):
+ """ Append content to the summary xml file. """
+ if os.path.getsize(summary) + os.path.getsize(xmllog) > (maxmb*1024*1024):
+ print 'Error: ' + summary + ' larger than ' + str(maxmb) + 'MB, not appending'
+ return
+
+ outfile = codecs.open(summary + ".tmp", 'w', "utf8")
+ parser = make_parser()
+ parser.setContentHandler(AppendSummary(outfile, xmllog))
+
+ input = open(summary, 'r')
+ parser.parse(input)
+ input.close()
+ outfile.close()
+ # Updating the summary file.
+ os.unlink(summary)
+ os.rename(summary + ".tmp", summary)
+
+
+def symbian_log_header(output, config, command, dir):
+ output.log("===-------------------------------------------------")
+ output.log("=== %s" % config)
+ output.log("===-------------------------------------------------")
+ output.log("=== %s started %s" % (config, datetime.datetime.now().ctime()))
+ output.log("=== %s == %s" % (config, dir))
+ output.log("-- %s" % command)
+ output.log("++ Started at %s" % datetime.datetime.now().ctime())
+ output.log("+++ HiRes Start %f" % time.time())
+ output.log("Chdir %s" % dir)
+
+
+def symbian_log_footer(output):
+ output.log("+++ HiRes End %f" % time.time())
+ output.log("++ Finished at %s" % datetime.datetime.now().ctime())
+
+
+if __name__ == "__main__":
+ convert(sys.argv[1], "%s.xml" % sys.argv[1], fulllogging=False)
+ """ An empty summary:
+ <?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>
+ """
+ #s = open(r"z:\summary.xml", "w")
+ #s.write("""<?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>""")
+ #s.close()
+ #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1.mc_5132_2_build.log2.xml')
+ #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1_BOM.xml')
+
+
\ No newline at end of file