1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 """ Symbian log converter.
21 """
22 import xml.dom.minidom
23 import sys
24 import os
25 import re
26 import shutil
27 import codecs
28 import time
29 import datetime
30 from xml.sax import make_parser
31 from xml.sax.handler import ContentHandler
32 from xml.sax.saxutils import escape
33
34
35 DEFAULT_CONFIGURATION = {"FATAL": [r"mingw_make.exe"],
36 "ERROR": [r'^(?:(?:\s*\d+\)\s*)|(?:\s*\*\*\*\s*))ERROR:',
37 r"^MISSING:",
38 r"Error:\s+",
39 r"^Error:",
40 r"'.+' is not recognized as an internal or external command",
41 r"FLEXlm error:",
42 r"(ABLD|BLDMAKE) ERROR:",
43 r"FATAL ERROR\(S\):",
44 r"fatal error U1077",
45 r"warning U4010",
46 r"^make(?:\[\d+\])?\: \*\*\*",
47 r"^make(?:\[\d+\])?:\s+.*\s+not\s+remade",
48 r"\"(.*)\", line (\d+): (Error: +(.\d+.*?):.*)$",
49 r"error: ((Internal fault):.*)$",
50 r"Exception: [A-Z0-9_]+",
51 r"target .* given more than once in the same rule",
52 r"^ERROR:",
53 r"^ERROR EC\d+:",
54 r"^ERROR\t",],
55 "CRITICAL": [r"[Ww]arning:?\s+(#111-D|#1166-D|#117-D|#128-D|#1293-D|#1441-D|#170-D|#174-D|#175-D|#185-D|#186-D|#223-D|#231-D|#257-D|#284-D|#368-D|#414-D|#430-D|#47-D|#514-D|#546-D|#68-D|#69-D|#830-D|#940-D|#836-D|A1495E|L6318W|C2874W|C4127|C4355|C4530|C4702|C4786|LNK4049)"],
56 "WARNING": [r'\): Missing file:',
57 r'^(\d+\))?\s*WARNING:', r'^MAKEDEF WARNING:',
58 r'line \d+: Warning:', r':\s+warning\s+\w+:',
59 r"\\\\(.*?)\(\d+\)\s:\sWarning:\s\(\d+\)",
60 r"^(BLDMAKE |MAKEDEF )?WARNING:",
61 r"WARNING\(S\)",
62 r"\(\d+\) : warning C",
63 r"LINK : warning",
64 r":\d+: warning:",
65 r"\"(.*)\", line (\d+): (Warning: +(?!A1495E)(.\d+.*?):.*)$",
66 r"Usage Warning:",
67 r"mwld.exe:",
68 r"^Command line warning",
69 r"ERROR: bad relocation:",
70 r"^(\d+) warning",
71 r"EventType:\s+Error\s+Source:\s+SweepNT",
72 r"^WARN\t",
73 ],
74 "REMARK": [r"Command line warning D4025 : ",
75 r"^REMARK: ",
76 r"^EventType:\s+Error\s+Source:\s+GNU\s+Make",
77 r":\d+: warning: cannot find matching deallocation function",
78 r"((:\d+)*: note: )",
79 ],
80 "INFO": [r"^INFO:"]
81 }
82
83
85 keys = config.keys()
86 keys.reverse()
87 for category in keys:
88 for rule in config[category]:
89 if rule.search(line) != None:
90 return category.lower()
91 return "stdout"
92
94 """ Bottomless stack. If empty just pop a default element. """
95
97 self.__default = default
98 self.__stack = []
99
101 result = None
102 try:
103 result = self.__stack.pop()
104 except IndexError, e:
105 result = self.__default
106 return result
107
108 - def push(self, item):
110
112 return len(self.__stack)
113
115 """ Cleanup string to match CDATA requiements.
116 These are the only allowed characters: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF].
117 """
118 result = ""
119 for c in list(text):
120 v = ord(c)
121 if v == 0x9 or v == 0xa or v == 0xd:
122 result += c
123 elif v>=0x20 and v <= 0xd7ff:
124 result += c
125 elif v>=0xe000 and v <= 0xfffd:
126 result += c
127 elif v>=0x10000 and v <= 0x10ffff:
128 result += c
129 else:
130 result += " "
131 return result
132
134 """ XML Log writer. """
135
137 self.__stream = stream
138 self.__stream.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
139 self.__stream.write("<log filename=\"%s\">\n" % filename)
140 self.__stream.write("\t<build>\n")
141 self.__indent = "\t"
142 self.__intask = 0
143
145
146 while self.__intask > 0:
147 self.close_task()
148 self.__stream.write("\t</build>\n")
149 self.__stream.write("</log>\n")
150 self.__stream.close()
151
153 self.__indent += "\t"
154 self.__intask += 1
155 self.__stream.write("%s<task name=\"%s\">\n" % (self.__indent, name))
156
158 if self.__intask > 0:
159 self.__intask -= 1
160 self.__stream.write("%s</task>\n" % (self.__indent))
161 self.__indent = self.__indent[:-1]
162
164 try:
165 acdata = to_cdata(msg.decode('utf-8', 'ignore'))
166 self.__stream.write("%s<message priority=\"%s\"><![CDATA[%s]]></message>\n" % (self.__indent+"\t", priority, acdata))
167 except UnicodeDecodeError, e:
168 print e
169
170
171
173 """ Convert an input log into an XML log and write an outputfile. """
174
175
176 built_config = {}
177 for category in configuration.keys():
178 built_config[category] = []
179 for rule in configuration[category]:
180 built_config[category].append(re.compile(rule))
181
182
183 log = open(inputfile, 'r')
184 olog = codecs.open(outputfile, 'w+', 'utf-8', errors='ignore')
185 xmllog = LogWriter(olog, inputfile)
186
187
188 match_finnished = re.compile(r"^===\s+.+\s+finished")
189 match_started = re.compile(r"^===\s+(.+)\s+started")
190 match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
191 match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
192
193
194 match_ant_target_start = re.compile(r'^([^\s=\[\]]+):$')
195 match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
196 symbian = False
197 ant_has_open_task = False
198
199 for line in log:
200
201
202 if not symbian and match_ant_target_end.match(line):
203 xmllog.close_task()
204 ant_has_open_task = False
205 continue
206 elif not symbian and match_ant_target_start.match(line):
207 result = match_ant_target_start.match(line)
208 if result != None:
209 if ant_has_open_task:
210 xmllog.close_task()
211 ant_has_open_task = False
212 xmllog.open_task(result.group(1))
213 ant_has_open_task = True
214 continue
215
216 line = match_logger_component.sub(r'', line)
217 line = line.strip()
218 if line.startswith("++ Finished at"):
219 xmllog.close_task()
220 elif line.startswith("=== "):
221 if match_finnished.match(line):
222 xmllog.close_task()
223 else:
224
225 symbian = True
226 result = match_component.match(line)
227 if result != None:
228 xmllog.open_task(result.group(2))
229
230 result = match_started.match(line)
231 if result != None:
232 xmllog.open_task(result.group(1))
233 else:
234
235 priority = find_priority(line, built_config)
236 if (fulllogging or priority != 'stdout'):
237 xmllog.message(priority, line)
238
239 xmllog.close()
240
242 """ Convert an input log into an XML log and write an outputfile. """
243
244
245 built_config = {}
246 for category in configuration.keys():
247 built_config[category] = []
248 for rule in configuration[category]:
249 built_config[category].append(re.compile(rule))
250
251
252 log = open (inputfile, 'r')
253 doc = xml.dom.minidom.Document()
254 root = doc.createElementNS("", "log")
255 root.setAttributeNS("", "name", inputfile)
256 doc.appendChild(root)
257 build = doc.createElementNS("", "build")
258 root.appendChild(build)
259
260 current = build
261
262 stack = Stack(build)
263
264 match_finnished = re.compile(r"^===\s+.+\s+finished")
265 match_started = re.compile(r"===\s+(.+)\s+started")
266 match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
267 match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
268
269
270 match_ant_target_start = re.compile(r'^([^\s=]+):$')
271 match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
272
273 for line in log:
274
275
276 if match_ant_target_end.match(line):
277 current = stack.pop()
278 continue
279 elif match_ant_target_start.match(line):
280 result = match_ant_target_start.match(line)
281 if result != None:
282 stack.push(current)
283 task = doc.createElementNS("", "task")
284 task.setAttributeNS("", "name", result.group(1))
285 current.appendChild(task)
286 current = task
287 continue
288
289 line = match_logger_component.sub(r'', line)
290 line = line.strip()
291 if line.startswith("++ Finished at"):
292 current = stack.pop()
293 elif line.startswith("==="):
294 if match_finnished.match(line):
295 current = stack.pop()
296 else:
297 result = match_component.match(line)
298 if result != None:
299 stack.push(current)
300 task = doc.createElementNS("", "task")
301 task.setAttributeNS("", "name", result.group(2))
302 current.appendChild(task)
303 current = task
304
305 result = match_started.match(line)
306 if result != None:
307 task = doc.createElementNS("", "task")
308 task.setAttributeNS("", "name", result.group(1))
309 stack.push(current)
310 current.appendChild(task)
311 current = task
312 else:
313 msg = doc.createElementNS("", "message")
314
315 priority = find_priority(line, built_config)
316 if (fulllogging or priority != 'stdout'):
317 msg.setAttributeNS("", "priority", priority)
318 msg.appendChild(doc.createCDATASection(to_cdata(line.decode("utf-8"))))
319 current.appendChild(msg)
320
321 file_object = codecs.open(outputfile, 'w', "utf_8")
322 file_object.write(doc.toprettyxml())
323 file_object.close()
324
325 -class ContentWriter(ContentHandler):
326 """ SAX Content writer. Parse and write an XML file. """
327 - def __init__(self, os, indent=""):
328 self.os = os
329 self.indent = indent
330 self.__content = u""
331
332 - def startElement(self, name, attrs):
333 self.os.write(self.indent + "<" + name)
334 if attrs.getLength() > 0:
335 self.os.write(" ")
336 self.os.write(" ".join(map(lambda x: "%s=\"%s\"" % (x, attrs.getValue(x)), attrs.getNames())))
337 self.os.write(">\n")
338 self.indent += "\t"
339 self.__content = ""
340
341 - def endElement(self, name):
342 if len(self.__content) > 0:
343 self.os.write(self.indent + self.__content + "\n")
344 self.indent = self.indent[:-1]
345 self.os.write("%s</%s>\n" % (self.indent, name))
346 self.__content = ""
347
348 - def characters(self, content):
349 self.__content += unicode(escape(content.strip()))
350
352 """ SAX content handler to add an XML log to the summary. """
356
358 self.os.write('<?xml version="1.0" encoding="utf-8"?>\n')
359
366
367
369 """ Append content to the summary xml file. """
370 if os.path.getsize(summary) + os.path.getsize(xmllog) > (maxmb*1024*1024):
371 print 'Error: ' + summary + ' larger than ' + str(maxmb) + 'MB, not appending'
372 return
373
374 outfile = codecs.open(summary + ".tmp", 'w', "utf8")
375 parser = make_parser()
376 parser.setContentHandler(AppendSummary(outfile, xmllog))
377
378 input = open(summary, 'r')
379 parser.parse(input)
380 input.close()
381 outfile.close()
382
383 os.unlink(summary)
384 os.rename(summary + ".tmp", summary)
385
386
388 output.log("===-------------------------------------------------")
389 output.log("=== %s" % config)
390 output.log("===-------------------------------------------------")
391 output.log("=== %s started %s" % (config, datetime.datetime.now().ctime()))
392 output.log("=== %s == %s" % (config, dir))
393 output.log("-- %s" % command)
394 output.log("++ Started at %s" % datetime.datetime.now().ctime())
395 output.log("+++ HiRes Start %f" % time.time())
396 output.log("Chdir %s" % dir)
397
398
402
403
404 if __name__ == "__main__":
405 convert(sys.argv[1], "%s.xml" % sys.argv[1], fulllogging=False)
406 """ An empty summary:
407 <?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>
408 """
409
410
411
412
413
414