Module log2xml
[hide private]
[frames] | no frames]

Source Code for Module log2xml

  1  #============================================================================  
  2  #Name        : log2xml.py  
  3  #Part of     : Helium  
  4   
  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
  6  #All rights reserved. 
  7  #This component and the accompanying materials are made available 
  8  #under the terms of the License "Eclipse Public License v1.0" 
  9  #which accompanies this distribution, and is available 
 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
 11  # 
 12  #Initial Contributors: 
 13  #Nokia Corporation - initial contribution. 
 14  # 
 15  #Contributors: 
 16  # 
 17  #Description: 
 18  #=============================================================================== 
 19   
 20  """ Symbian log converter. 
 21  """ 
 22  import xml.dom.minidom 
 23  import sys 
 24  import os 
 25  import re 
 26  import shutil 
 27  import codecs 
 28  import time 
 29  import datetime 
 30  from xml.sax import make_parser  
 31  from xml.sax.handler import ContentHandler  
 32  from xml.sax.saxutils import escape 
 33   
 34   
 35  DEFAULT_CONFIGURATION = {"FATAL": [r"mingw_make.exe"], 
 36                           "ERROR": [r'^(?:(?:\s*\d+\)\s*)|(?:\s*\*\*\*\s*))ERROR:', 
 37                                     r"^MISSING:", 
 38                                     r"Error:\s+", 
 39                                     r"^Error:", 
 40                                     r"'.+' is not recognized as an internal or external command", 
 41                                     r"FLEXlm error:", 
 42                                     r"(ABLD|BLDMAKE) ERROR:", 
 43                                     r"FATAL ERROR\(S\):", 
 44                                     r"fatal error U1077", 
 45                                     r"warning U4010", 
 46                                     r"^make(?:\[\d+\])?\: \*\*\*", 
 47                                     r"^make(?:\[\d+\])?:\s+.*\s+not\s+remade", 
 48                                     r"\"(.*)\", line (\d+): (Error: +(.\d+.*?):.*)$", 
 49                                     r"error: ((Internal fault):.*)$", 
 50                                     r"Exception: [A-Z0-9_]+", 
 51                                     r"target .* given more than once in the same rule", 
 52                                     r"^ERROR:", 
 53                                     r"^ERROR EC\d+:", 
 54                                     r"^ERROR\t",], 
 55                           "CRITICAL": [r"[Ww]arning:?\s+(#111-D|#1166-D|#117-D|#128-D|#1293-D|#1441-D|#170-D|#174-D|#175-D|#185-D|#186-D|#223-D|#231-D|#257-D|#284-D|#368-D|#414-D|#430-D|#47-D|#514-D|#546-D|#68-D|#69-D|#830-D|#940-D|#836-D|A1495E|L6318W|C2874W|C4127|C4355|C4530|C4702|C4786|LNK4049)"], 
 56                           "WARNING": [r'\): Missing file:', 
 57                                        r'^(\d+\))?\s*WARNING:', r'^MAKEDEF WARNING:', 
 58                                        r'line \d+: Warning:', r':\s+warning\s+\w+:', 
 59                                        r"\\\\(.*?)\(\d+\)\s:\sWarning:\s\(\d+\)", 
 60                                        r"^(BLDMAKE |MAKEDEF )?WARNING:", 
 61                                        r"WARNING\(S\)", 
 62                                        r"\(\d+\) : warning C", 
 63                                        r"LINK : warning", 
 64                                        r":\d+: warning:", 
 65                                        r"\"(.*)\", line (\d+): (Warning: +(?!A1495E)(.\d+.*?):.*)$", 
 66                                        r"Usage Warning:", 
 67                                        r"mwld.exe:", 
 68                                        r"^Command line warning", 
 69                                        r"ERROR: bad relocation:", 
 70                                        r"^(\d+) warning", 
 71                                        r"EventType:\s+Error\s+Source:\s+SweepNT", 
 72                                        r"^WARN\t", 
 73                                        ], 
 74                          "REMARK": [r"Command line warning D4025 : ", 
 75                                     r"^REMARK: ", 
 76                                     r"^EventType:\s+Error\s+Source:\s+GNU\s+Make", 
 77                                     r":\d+: warning: cannot find matching deallocation function", 
 78                                     r"((:\d+)*: note: )", 
 79                                     ], 
 80                          "INFO": [r"^INFO:"] 
 81                          } 
 82   
 83   
84 -def find_priority(line, config):
85 keys = config.keys() 86 keys.reverse() 87 for category in keys: 88 for rule in config[category]: 89 if rule.search(line) != None: 90 return category.lower() 91 return "stdout"
92
93 -class Stack:
94 """ Bottomless stack. If empty just pop a default element. """ 95
96 - def __init__(self, default):
97 self.__default = default 98 self.__stack = []
99
100 - def pop(self):
101 result = None 102 try: 103 result = self.__stack.pop() 104 except IndexError, e: 105 result = self.__default 106 return result
107
108 - def push(self, item):
109 self.__stack.append(item)
110
111 - def __len__(self):
112 return len(self.__stack)
113
114 -def to_cdata(text):
115 """ Cleanup string to match CDATA requiements. 116 These are the only allowed characters: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]. 117 """ 118 result = "" 119 for c in list(text): 120 v = ord(c) 121 if v == 0x9 or v == 0xa or v == 0xd: 122 result += c 123 elif v>=0x20 and v <= 0xd7ff: 124 result += c 125 elif v>=0xe000 and v <= 0xfffd: 126 result += c 127 elif v>=0x10000 and v <= 0x10ffff: 128 result += c 129 else: 130 result += " " 131 return result
132
133 -class LogWriter(object):
134 """ XML Log writer. """ 135
136 - def __init__(self, stream, filename):
137 self.__stream = stream 138 self.__stream.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n") 139 self.__stream.write("<log filename=\"%s\">\n" % filename) 140 self.__stream.write("\t<build>\n") 141 self.__indent = "\t" 142 self.__intask = 0
143
144 - def close(self):
145 # closing open tasks... 146 while self.__intask > 0: 147 self.close_task() 148 self.__stream.write("\t</build>\n") 149 self.__stream.write("</log>\n") 150 self.__stream.close()
151
152 - def open_task(self, name):
153 self.__indent += "\t" 154 self.__intask += 1 155 self.__stream.write("%s<task name=\"%s\">\n" % (self.__indent, name))
156
157 - def close_task(self):
158 if self.__intask > 0: 159 self.__intask -= 1 160 self.__stream.write("%s</task>\n" % (self.__indent)) 161 self.__indent = self.__indent[:-1]
162
163 - def message(self, priority, msg):
164 try: 165 acdata = to_cdata(msg.decode('utf-8', 'ignore')) 166 self.__stream.write("%s<message priority=\"%s\"><![CDATA[%s]]></message>\n" % (self.__indent+"\t", priority, acdata)) 167 except UnicodeDecodeError, e: 168 print e
169 170 171
172 -def convert(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
173 """ Convert an input log into an XML log and write an outputfile. """ 174 175 # Compiling the regexp 176 built_config = {} 177 for category in configuration.keys(): 178 built_config[category] = [] 179 for rule in configuration[category]: 180 built_config[category].append(re.compile(rule)) 181 182 # Generating the XML log 183 log = open(inputfile, 'r') 184 olog = codecs.open(outputfile, 'w+', 'utf-8', errors='ignore') 185 xmllog = LogWriter(olog, inputfile) 186 187 188 match_finnished = re.compile(r"^===\s+.+\s+finished") 189 match_started = re.compile(r"^===\s+(.+)\s+started") 190 match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)") 191 match_logger_component = re.compile(r'^\s*\[.+?\]\s*') 192 #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started') 193 #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished') 194 match_ant_target_start = re.compile(r'^([^\s=\[\]]+):$') 195 match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration') 196 symbian = False 197 ant_has_open_task = False 198 # looping 199 for line in log: 200 201 # matching Ant logging 202 if not symbian and match_ant_target_end.match(line): 203 xmllog.close_task() 204 ant_has_open_task = False 205 continue 206 elif not symbian and match_ant_target_start.match(line): 207 result = match_ant_target_start.match(line) 208 if result != None: 209 if ant_has_open_task: 210 xmllog.close_task() 211 ant_has_open_task = False 212 xmllog.open_task(result.group(1)) 213 ant_has_open_task = True 214 continue 215 # matching Symbian logging 216 line = match_logger_component.sub(r'', line) 217 line = line.strip() 218 if line.startswith("++ Finished at"): 219 xmllog.close_task() 220 elif line.startswith("=== "): 221 if match_finnished.match(line): 222 xmllog.close_task() 223 else: 224 # This is a symbian log 225 symbian = True 226 result = match_component.match(line) 227 if result != None: 228 xmllog.open_task(result.group(2)) 229 # === cenrep_s60_32 started 230 result = match_started.match(line) 231 if result != None: 232 xmllog.open_task(result.group(1)) 233 else: 234 # Type? 235 priority = find_priority(line, built_config) 236 if (fulllogging or priority != 'stdout'): 237 xmllog.message(priority, line) 238 # end file 239 xmllog.close()
240
241 -def convert_old(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
242 """ Convert an input log into an XML log and write an outputfile. """ 243 244 # Compiling the regexp 245 built_config = {} 246 for category in configuration.keys(): 247 built_config[category] = [] 248 for rule in configuration[category]: 249 built_config[category].append(re.compile(rule)) 250 251 # Generating the XML log 252 log = open (inputfile, 'r') 253 doc = xml.dom.minidom.Document() 254 root = doc.createElementNS("", "log") 255 root.setAttributeNS("", "name", inputfile) 256 doc.appendChild(root) 257 build = doc.createElementNS("", "build") 258 root.appendChild(build) 259 # current group/task 260 current = build 261 # bottomless stask, if losing sync all message will be at top level. 262 stack = Stack(build) 263 264 match_finnished = re.compile(r"^===\s+.+\s+finished") 265 match_started = re.compile(r"===\s+(.+)\s+started") 266 match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)") 267 match_logger_component = re.compile(r'^\s*\[.+?\]\s*') 268 #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started') 269 #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished') 270 match_ant_target_start = re.compile(r'^([^\s=]+):$') 271 match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration') 272 # looping 273 for line in log: 274 275 # matching Ant logging 276 if match_ant_target_end.match(line): 277 current = stack.pop() 278 continue 279 elif match_ant_target_start.match(line): 280 result = match_ant_target_start.match(line) 281 if result != None: 282 stack.push(current) 283 task = doc.createElementNS("", "task") 284 task.setAttributeNS("", "name", result.group(1)) 285 current.appendChild(task) 286 current = task 287 continue 288 # matching Symbian logging 289 line = match_logger_component.sub(r'', line) 290 line = line.strip() 291 if line.startswith("++ Finished at"): 292 current = stack.pop() 293 elif line.startswith("==="): 294 if match_finnished.match(line): 295 current = stack.pop() 296 else: 297 result = match_component.match(line) 298 if result != None: 299 stack.push(current) 300 task = doc.createElementNS("", "task") 301 task.setAttributeNS("", "name", result.group(2)) 302 current.appendChild(task) 303 current = task 304 # === cenrep_s60_32 started 305 result = match_started.match(line) 306 if result != None: 307 task = doc.createElementNS("", "task") 308 task.setAttributeNS("", "name", result.group(1)) 309 stack.push(current) 310 current.appendChild(task) 311 current = task 312 else: 313 msg = doc.createElementNS("", "message") 314 # Type? 315 priority = find_priority(line, built_config) 316 if (fulllogging or priority != 'stdout'): 317 msg.setAttributeNS("", "priority", priority) 318 msg.appendChild(doc.createCDATASection(to_cdata(line.decode("utf-8")))) 319 current.appendChild(msg) 320 321 file_object = codecs.open(outputfile, 'w', "utf_8") 322 file_object.write(doc.toprettyxml()) 323 file_object.close()
324
325 -class ContentWriter(ContentHandler):
326 """ SAX Content writer. Parse and write an XML file. """
327 - def __init__(self, os, indent=""):
328 self.os = os 329 self.indent = indent 330 self.__content = u""
331
332 - def startElement(self, name, attrs):
333 self.os.write(self.indent + "<" + name) 334 if attrs.getLength() > 0: 335 self.os.write(" ") 336 self.os.write(" ".join(map(lambda x: "%s=\"%s\"" % (x, attrs.getValue(x)), attrs.getNames()))) 337 self.os.write(">\n") 338 self.indent += "\t" 339 self.__content = ""
340
341 - def endElement(self, name):
342 if len(self.__content) > 0: 343 self.os.write(self.indent + self.__content + "\n") 344 self.indent = self.indent[:-1] 345 self.os.write("%s</%s>\n" % (self.indent, name)) 346 self.__content = ""
347
348 - def characters(self, content):
349 self.__content += unicode(escape(content.strip()))
350
351 -class AppendSummary(ContentWriter):
352 """ SAX content handler to add an XML log to the summary. """
353 - def __init__(self, output, xmllog):
354 ContentWriter.__init__(self, output) 355 self.xmllog = xmllog
356
357 - def startDocument(self):
358 self.os.write('<?xml version="1.0" encoding="utf-8"?>\n')
359
360 - def startElement(self, name, attrs):
361 ContentWriter.startElement(self, name, attrs) 362 if name == "logSummary": 363 parser = make_parser() 364 parser.setContentHandler(ContentWriter(self.os, self.indent)) 365 parser.parse(open(self.xmllog, 'r'))
366 367
368 -def append_summary(summary, xmllog, maxmb=80):
369 """ Append content to the summary xml file. """ 370 if os.path.getsize(summary) + os.path.getsize(xmllog) > (maxmb*1024*1024): 371 print 'Error: ' + summary + ' larger than ' + str(maxmb) + 'MB, not appending' 372 return 373 374 outfile = codecs.open(summary + ".tmp", 'w', "utf8") 375 parser = make_parser() 376 parser.setContentHandler(AppendSummary(outfile, xmllog)) 377 378 input = open(summary, 'r') 379 parser.parse(input) 380 input.close() 381 outfile.close() 382 # Updating the summary file. 383 os.unlink(summary) 384 os.rename(summary + ".tmp", summary)
385 386
387 -def symbian_log_header(output, config, command, dir):
388 output.log("===-------------------------------------------------") 389 output.log("=== %s" % config) 390 output.log("===-------------------------------------------------") 391 output.log("=== %s started %s" % (config, datetime.datetime.now().ctime())) 392 output.log("=== %s == %s" % (config, dir)) 393 output.log("-- %s" % command) 394 output.log("++ Started at %s" % datetime.datetime.now().ctime()) 395 output.log("+++ HiRes Start %f" % time.time()) 396 output.log("Chdir %s" % dir)
397 398 402 403 404 if __name__ == "__main__": 405 convert(sys.argv[1], "%s.xml" % sys.argv[1], fulllogging=False) 406 """ An empty summary: 407 <?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/> 408 """ 409 #s = open(r"z:\summary.xml", "w") 410 #s.write("""<?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>""") 411 #s.close() 412 #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1.mc_5132_2_build.log2.xml') 413 #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1_BOM.xml') 414