|
28
|
1 |
|
|
|
2 |
# Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
|
|
|
3 |
# All rights reserved.
|
|
|
4 |
# This component and the accompanying materials are made available
|
|
|
5 |
# under the terms of the License "Eclipse Public License v1.0"
|
|
|
6 |
# which accompanies this distribution, and is available
|
|
|
7 |
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
|
|
|
8 |
|
|
|
9 |
'''
|
|
|
10 |
Filter class for generating HTML summary pages
|
|
|
11 |
'''
|
|
|
12 |
|
|
|
13 |
import os
|
|
|
14 |
import re
|
|
|
15 |
import csv
|
|
|
16 |
import sys
|
|
|
17 |
import shutil
|
|
|
18 |
import tempfile
|
|
|
19 |
import time
|
|
|
20 |
import filter_interface
|
|
|
21 |
|
|
|
22 |
class HTML(filter_interface.FilterSAX):
|
|
|
23 |
|
|
|
24 |
def __init__(self, params = []):
|
|
|
25 |
"""parameters to this filter are..."""
|
|
|
26 |
super(HTML, self).__init__()
|
|
|
27 |
|
|
|
28 |
# FilterSAX method overrides
|
|
|
29 |
|
|
|
30 |
def startDocument(self):
|
|
|
31 |
|
|
|
32 |
if self.params.logFileName:
|
|
|
33 |
self.dirname = str(self.params.logFileName).replace("%TIME", self.params.timestring) + "_html"
|
|
|
34 |
else:
|
|
|
35 |
self.dirname = "html" # writing to stdout doesn't make sense
|
|
|
36 |
|
|
|
37 |
# read regular expressions from the first file on the config path
|
|
|
38 |
self.regex = []
|
|
|
39 |
for p in self.params.configPath:
|
|
|
40 |
if not p.isAbsolute():
|
|
|
41 |
p = self.params.home.Append(p)
|
|
|
42 |
|
|
|
43 |
csv = p.Append("logfile_regex.csv")
|
|
|
44 |
if csv.isFile():
|
|
|
45 |
self.regex = self.readregex(str(csv))
|
|
|
46 |
break
|
|
|
47 |
|
|
|
48 |
# regexes for important "make" errors
|
|
|
49 |
self.noruletomake = re.compile("No rule to make target `(.+)', needed by `(.+)'")
|
|
|
50 |
|
|
|
51 |
# all our lists are empty
|
|
|
52 |
self.elements = []
|
|
|
53 |
self.recipe_tag = None
|
|
|
54 |
self.error_tag = None
|
|
|
55 |
self.warning_tag = None
|
|
|
56 |
|
|
|
57 |
self.components = {}
|
|
|
58 |
self.configurations = {}
|
|
|
59 |
self.missed_depends = {}
|
|
|
60 |
self.parse_start = {}
|
|
|
61 |
self.totals = Records()
|
|
|
62 |
|
|
|
63 |
self.progress_started = 0
|
|
|
64 |
self.progress_stopped = 0
|
|
|
65 |
|
|
|
66 |
# create all the directories
|
|
|
67 |
for s in Records.CLASSES:
|
|
|
68 |
dir = os.path.join(self.dirname, s)
|
|
|
69 |
if not os.path.isdir(dir):
|
|
|
70 |
try:
|
|
|
71 |
os.makedirs(dir)
|
|
|
72 |
except:
|
|
|
73 |
return self.err("could not create directory '%s'" % dir)
|
|
|
74 |
|
|
|
75 |
# create an index.html
|
|
|
76 |
try:
|
|
|
77 |
indexname = os.path.join(self.dirname, "index.html")
|
|
|
78 |
|
|
|
79 |
self.index = open(indexname, "w")
|
|
|
80 |
self.index.write("""<html>
|
|
|
81 |
<head>
|
|
|
82 |
<title>Raptor Build Summary</title>
|
|
|
83 |
<link type="text/css" rel="stylesheet" href="style.css">
|
|
|
84 |
</head>
|
|
|
85 |
<body>
|
|
|
86 |
<h1>Raptor Build Summary</h1>
|
|
|
87 |
""")
|
|
|
88 |
except:
|
|
|
89 |
return self.err("could not create file '%s'" % indexname)
|
|
|
90 |
|
|
|
91 |
# copy over a style file if none exists in the output already
|
|
|
92 |
css = os.path.join(self.dirname, "style.css")
|
|
|
93 |
if not os.path.isfile(css):
|
|
|
94 |
try:
|
|
|
95 |
style = str(self.params.home.Append("style/filter_html.css"))
|
|
|
96 |
shutil.copyfile(style, css)
|
|
|
97 |
except:
|
|
|
98 |
self.moan("could not copy '%s' to '%s'" % (style, css))
|
|
|
99 |
|
|
|
100 |
# create a temporary file to record all the "what" files in. We can
|
|
|
101 |
# only test the files for existence after "make" has finished running.
|
|
|
102 |
try:
|
|
|
103 |
self.tmp = tempfile.TemporaryFile()
|
|
|
104 |
except:
|
|
|
105 |
return self.err("could not create temporary file")
|
|
|
106 |
|
|
|
107 |
return self.ok
|
|
|
108 |
|
|
|
109 |
def startElement(self, name, attributes):
|
|
|
110 |
"call the start handler for this element if we defined one."
|
|
|
111 |
|
|
|
112 |
ns_name = name.replace(":", "_")
|
|
|
113 |
self.generic_start(ns_name) # tracks element nesting
|
|
|
114 |
|
|
|
115 |
function_name = "start_" + ns_name
|
|
|
116 |
try:
|
|
|
117 |
HTML.__dict__[function_name](self, attributes)
|
|
|
118 |
except KeyError:
|
|
|
119 |
pass
|
|
|
120 |
|
|
|
121 |
def characters(self, char):
|
|
|
122 |
"process [some of] the body text for the current element."
|
|
|
123 |
|
|
|
124 |
function_name = "char_" + self.elements[-1]
|
|
|
125 |
try:
|
|
|
126 |
HTML.__dict__[function_name](self, char)
|
|
|
127 |
except KeyError:
|
|
|
128 |
pass
|
|
|
129 |
|
|
|
130 |
def endElement(self, name):
|
|
|
131 |
"call the end handler for this element if we defined one."
|
|
|
132 |
|
|
|
133 |
function_name = "end_" + name.replace(":", "_")
|
|
|
134 |
try:
|
|
|
135 |
HTML.__dict__[function_name](self)
|
|
|
136 |
except KeyError:
|
|
|
137 |
pass
|
|
|
138 |
|
|
|
139 |
self.generic_end() # tracks element nesting
|
|
|
140 |
|
|
|
141 |
def endDocument(self):
|
|
|
142 |
|
|
|
143 |
self.existencechecks()
|
|
|
144 |
self.dumptotals()
|
|
|
145 |
try:
|
|
|
146 |
if self.progress_started > 0:
|
|
|
147 |
t_from = time.asctime(time.localtime(self.progress_started))
|
|
|
148 |
t_to = time.asctime(time.localtime(self.progress_stopped))
|
|
|
149 |
self.index.write("<p>start: " + t_from + "\n")
|
|
|
150 |
self.index.write("<br>end : " + t_to + "\n")
|
|
|
151 |
|
|
|
152 |
self.index.write("<p><table><tr><th></th>")
|
|
|
153 |
|
|
|
154 |
for title in Records.TITLES:
|
|
|
155 |
self.index.write('<th class="numbers">%s</th>' % title)
|
|
|
156 |
|
|
|
157 |
self.index.write("</tr>")
|
|
|
158 |
self.index.write(self.totals.tablerow("total"))
|
|
|
159 |
self.index.write("</table>")
|
|
|
160 |
|
|
|
161 |
|
|
|
162 |
self.index.write("<h2>by configuration</h2>")
|
|
|
163 |
self.index.write("<p><table><tr><th></th>")
|
|
|
164 |
|
|
|
165 |
for title in Records.TITLES:
|
|
|
166 |
self.index.write('<th class="numbers">%s</th>' % title)
|
|
|
167 |
|
|
|
168 |
self.index.write("</tr>")
|
|
|
169 |
|
|
|
170 |
# the list of configuration names in alphabetical order
|
|
|
171 |
names = self.configurations.keys()
|
|
|
172 |
names.sort()
|
|
|
173 |
|
|
|
174 |
# print the "unknown" configuration results first
|
|
|
175 |
if 'unknown' in names:
|
|
|
176 |
self.index.write(self.configurations['unknown'].tablerow("no specific configuration"))
|
|
|
177 |
names.remove('unknown')
|
|
|
178 |
|
|
|
179 |
# print the rest
|
|
|
180 |
for name in names:
|
|
|
181 |
self.index.write(self.configurations[name].tablerow(name))
|
|
|
182 |
|
|
|
183 |
self.index.write("</table>")
|
|
|
184 |
|
|
|
185 |
|
|
|
186 |
self.index.write("<h2>by component</h2>")
|
|
|
187 |
self.index.write("<p><table><tr><th></th>")
|
|
|
188 |
|
|
|
189 |
for title in Records.TITLES:
|
|
|
190 |
self.index.write('<th class="numbers">%s</th>' % title)
|
|
|
191 |
|
|
|
192 |
self.index.write("</tr>")
|
|
|
193 |
|
|
|
194 |
# the list of component names in alphabetical order
|
|
|
195 |
names = self.components.keys()
|
|
|
196 |
names.sort()
|
|
|
197 |
|
|
|
198 |
# print the "unknown" component results first
|
|
|
199 |
if 'unknown' in names:
|
|
|
200 |
self.index.write(self.components['unknown'].tablerow("no specific component"))
|
|
|
201 |
names.remove('unknown')
|
|
|
202 |
|
|
|
203 |
# print the rest
|
|
|
204 |
for name in names:
|
|
|
205 |
self.index.write(self.components[name].tablerow(name))
|
|
|
206 |
|
|
|
207 |
self.index.write("</table>")
|
|
|
208 |
self.index.write("</body></html>")
|
|
|
209 |
self.index.close()
|
|
|
210 |
except Exception, e:
|
|
|
211 |
return self.err("could not close index " + str(e))
|
|
|
212 |
|
|
|
213 |
# error and warning exception handlers for FilterSAX
|
|
|
214 |
|
|
|
215 |
def error(self, exception):
|
|
|
216 |
self.fatalError(exception) # all errors are fatal
|
|
|
217 |
|
|
|
218 |
def fatalError(self, exception):
|
|
|
219 |
self.err("exception " + str(exception))
|
|
|
220 |
|
|
|
221 |
def warning(self, exception):
|
|
|
222 |
"""only print warnings if no errors have occurred yet.
|
|
|
223 |
|
|
|
224 |
because after an error everything goes mad."""
|
|
|
225 |
if self.ok:
|
|
|
226 |
sys.stderr.write(self.formatWarning("HTML filter " + str(exception)))
|
|
|
227 |
|
|
|
228 |
# our error handling functions
|
|
|
229 |
|
|
|
230 |
def err(self, text):
|
|
|
231 |
"""only print the first error, then go quiet.
|
|
|
232 |
|
|
|
233 |
because after a fatal error there are usually hundreds of
|
|
|
234 |
meaningless repeats and/or garbage that doesn't help anyone."""
|
|
|
235 |
if self.ok:
|
|
|
236 |
sys.stderr.write(self.formatError("HTML filter " + text))
|
|
|
237 |
self.ok = False
|
|
|
238 |
return self.ok
|
|
|
239 |
|
|
|
240 |
def moan(self, text):
|
|
|
241 |
"""print a warning about something that is annoying but not fatal."""
|
|
|
242 |
if self.ok:
|
|
|
243 |
sys.stderr.write(self.formatWarning("HTML filter " + text))
|
|
|
244 |
return self.ok
|
|
|
245 |
|
|
|
246 |
# our content handling functions
|
|
|
247 |
|
|
|
248 |
def start_buildlog(self, attributes):
|
|
|
249 |
try:
|
|
|
250 |
self.index.write("<p><tt>sbs " + attributes['sbs_version'] + "</tt>")
|
|
|
251 |
except KeyError:
|
|
|
252 |
pass
|
|
|
253 |
|
|
|
254 |
def char_buildlog(self, char):
|
|
|
255 |
'''process text in the top-level element.
|
|
|
256 |
|
|
|
257 |
ideally all text will be inside <recipe> tags, but some will not.
|
|
|
258 |
"make" errors in particular appear inside the buildlog tag itself.'''
|
|
|
259 |
|
|
|
260 |
text = char.strip()
|
|
|
261 |
if text:
|
|
|
262 |
match = self.noruletomake.search(text)
|
|
|
263 |
if match:
|
|
|
264 |
target = match.group(2)
|
|
|
265 |
depend = match.group(1)
|
|
|
266 |
if target in self.missed_depends:
|
|
|
267 |
self.missed_depends[target].append(depend)
|
|
|
268 |
else:
|
|
|
269 |
self.missed_depends[target] = [depend]
|
|
|
270 |
|
|
|
271 |
def end_buildlog(self):
|
|
|
272 |
pass
|
|
|
273 |
|
|
|
274 |
def start_recipe(self, attributes):
|
|
|
275 |
self.recipe_tag = TaggedText(attributes)
|
|
|
276 |
|
|
|
277 |
def char_recipe(self, char):
|
|
|
278 |
self.recipe_tag.text += char
|
|
|
279 |
|
|
|
280 |
def end_recipe(self):
|
|
|
281 |
# an "ok" recipe may contain warnings / remarks
|
|
|
282 |
if self.recipe_tag.exit == 'ok':
|
|
|
283 |
self.record(self.recipe_tag, self.classify(self.recipe_tag.text))
|
|
|
284 |
|
|
|
285 |
# a "failed" recipe is always an error
|
|
|
286 |
elif self.recipe_tag.exit == 'failed':
|
|
|
287 |
self.record(self.recipe_tag, Records.ERROR)
|
|
|
288 |
|
|
|
289 |
# "retry" should just be ignored (for now)
|
|
|
290 |
# but will be recorded in a later version.
|
|
|
291 |
|
|
|
292 |
self.recipe_tag = None
|
|
|
293 |
|
|
|
294 |
def start_status(self, attributes):
|
|
|
295 |
try:
|
|
|
296 |
if self.recipe_tag:
|
|
|
297 |
self.recipe_tag.exit = attributes['exit']
|
|
|
298 |
self.recipe_tag.code = attributes['code']
|
|
|
299 |
else:
|
|
|
300 |
self.err("status element not inside a recipe element")
|
|
|
301 |
except KeyError:
|
|
|
302 |
pass
|
|
|
303 |
|
|
|
304 |
def start_time(self, attributes):
|
|
|
305 |
try:
|
|
|
306 |
if self.recipe_tag:
|
|
|
307 |
self.recipe_tag.time = float(attributes['elapsed'])
|
|
|
308 |
else:
|
|
|
309 |
self.err("status element not inside a recipe element")
|
|
|
310 |
except KeyError:
|
|
|
311 |
pass
|
|
|
312 |
|
|
|
313 |
def start_progress_start(self, attributes):
|
|
|
314 |
'''on progress:start note the parse starting timestamp.
|
|
|
315 |
|
|
|
316 |
and keep track of the earliest timestamp of all as that shows
|
|
|
317 |
us when the sbs command was run.'''
|
|
|
318 |
try:
|
|
|
319 |
t = float(attributes['time'])
|
|
|
320 |
if self.progress_started == 0 or t < self.progress_started:
|
|
|
321 |
self.progress_started = t
|
|
|
322 |
|
|
|
323 |
if attributes['task'] == 'parse':
|
|
|
324 |
self.parse_start[attributes['key']] = t
|
|
|
325 |
except KeyError:
|
|
|
326 |
pass
|
|
|
327 |
|
|
|
328 |
def start_progress_end(self, attributes):
|
|
|
329 |
'''on progress:end add the elapsed parse time to the total time.
|
|
|
330 |
|
|
|
331 |
also keep track of the latest timestamp of all as that shows
|
|
|
332 |
us when the sbs command finished.'''
|
|
|
333 |
try:
|
|
|
334 |
t = float(attributes['time'])
|
|
|
335 |
if t > self.progress_stopped:
|
|
|
336 |
self.progress_stopped = t
|
|
|
337 |
|
|
|
338 |
if attributes['task'] == 'parse':
|
|
|
339 |
elapsed = t - self.parse_start[attributes['key']]
|
|
|
340 |
self.totals.inc(Records.TIME, elapsed)
|
|
|
341 |
except KeyError:
|
|
|
342 |
pass
|
|
|
343 |
|
|
|
344 |
def start_error(self, attributes):
|
|
|
345 |
self.error_tag = TaggedText(attributes)
|
|
|
346 |
|
|
|
347 |
def char_error(self, char):
|
|
|
348 |
self.error_tag.text += char
|
|
|
349 |
|
|
|
350 |
def end_error(self):
|
|
|
351 |
self.record(self.error_tag, Records.ERROR)
|
|
|
352 |
self.error_tag = None
|
|
|
353 |
|
|
|
354 |
def start_warning(self, attributes):
|
|
|
355 |
self.warning_tag = TaggedText(attributes)
|
|
|
356 |
|
|
|
357 |
def char_warning(self, char):
|
|
|
358 |
self.warning_tag.text += char
|
|
|
359 |
|
|
|
360 |
def end_warning(self):
|
|
|
361 |
self.record(self.warning_tag, Records.WARNING)
|
|
|
362 |
self.warning_tag = None
|
|
|
363 |
|
|
|
364 |
def start_whatlog(self, attributes):
|
|
|
365 |
try:
|
|
|
366 |
for attrib in ['bldinf', 'config']:
|
|
|
367 |
self.tmp.write("|")
|
|
|
368 |
if attrib in attributes:
|
|
|
369 |
self.tmp.write(attributes[attrib])
|
|
|
370 |
self.tmp.write("\n")
|
|
|
371 |
except:
|
|
|
372 |
return self.err("could not write to temporary file")
|
|
|
373 |
|
|
|
374 |
def start_export(self, attributes):
|
|
|
375 |
try:
|
|
|
376 |
self.tmp.write(attributes['destination'] + "\n")
|
|
|
377 |
except:
|
|
|
378 |
return self.err("could not write to temporary file")
|
|
|
379 |
|
|
|
380 |
def start_resource(self, attributes):
|
|
|
381 |
self.resource_tag = ""
|
|
|
382 |
|
|
|
383 |
def char_resource(self, char):
|
|
|
384 |
self.resource_tag += char
|
|
|
385 |
|
|
|
386 |
def end_resource(self):
|
|
|
387 |
try:
|
|
|
388 |
self.tmp.write(self.resource_tag.strip() + "\n")
|
|
|
389 |
except:
|
|
|
390 |
return self.err("could not write to temporary file")
|
|
|
391 |
|
|
|
392 |
def start_bitmap(self, attributes):
|
|
|
393 |
self.bitmap_tag = ""
|
|
|
394 |
|
|
|
395 |
def char_bitmap(self, char):
|
|
|
396 |
self.bitmap_tag += char
|
|
|
397 |
|
|
|
398 |
def end_bitmap(self):
|
|
|
399 |
try:
|
|
|
400 |
self.tmp.write(self.bitmap_tag.strip() + "\n")
|
|
|
401 |
except:
|
|
|
402 |
return self.err("could not write to temporary file")
|
|
|
403 |
|
|
|
404 |
def start_stringtable(self, attributes):
|
|
|
405 |
self.stringtable_tag = ""
|
|
|
406 |
|
|
|
407 |
def char_stringtable(self, char):
|
|
|
408 |
self.stringtable_tag += char
|
|
|
409 |
|
|
|
410 |
def end_stringtable(self):
|
|
|
411 |
try:
|
|
|
412 |
self.tmp.write(self.stringtable_tag.strip() + "\n")
|
|
|
413 |
except:
|
|
|
414 |
return self.err("could not write to temporary file")
|
|
|
415 |
|
|
|
416 |
def start_member(self, attributes):
|
|
|
417 |
self.member_tag = ""
|
|
|
418 |
|
|
|
419 |
def char_member(self, char):
|
|
|
420 |
self.member_tag += char
|
|
|
421 |
|
|
|
422 |
def end_member(self):
|
|
|
423 |
try:
|
|
|
424 |
self.tmp.write(self.member_tag.strip() + "\n")
|
|
|
425 |
except:
|
|
|
426 |
return self.err("could not write to temporary file")
|
|
|
427 |
|
|
|
428 |
def start_build(self, attributes):
|
|
|
429 |
self.build_tag = ""
|
|
|
430 |
|
|
|
431 |
def char_build(self, char):
|
|
|
432 |
self.build_tag += char
|
|
|
433 |
|
|
|
434 |
def end_build(self):
|
|
|
435 |
try:
|
|
|
436 |
self.tmp.write(self.build_tag.strip() + "\n")
|
|
|
437 |
except:
|
|
|
438 |
return self.err("could not write to temporary file")
|
|
|
439 |
|
|
|
440 |
def start_clean(self, attributes):
|
|
|
441 |
try:
|
|
|
442 |
for attrib in ['bldinf', 'config']:
|
|
|
443 |
self.tmp.write("|")
|
|
|
444 |
if attrib in attributes:
|
|
|
445 |
self.tmp.write(attributes[attrib])
|
|
|
446 |
self.tmp.write("\n")
|
|
|
447 |
except:
|
|
|
448 |
return self.err("could not write to temporary file")
|
|
|
449 |
|
|
|
450 |
def start_file(self, attributes):
|
|
|
451 |
'''opening file tag.
|
|
|
452 |
|
|
|
453 |
in the temporary file we need to mark the "clean" targets with a
|
|
|
454 |
leading ">" character so they can be treated differently from
|
|
|
455 |
the "releasable" targets'''
|
|
|
456 |
self.file_tag = ">"
|
|
|
457 |
|
|
|
458 |
def char_file(self, char):
|
|
|
459 |
self.file_tag += char
|
|
|
460 |
|
|
|
461 |
def end_file(self):
|
|
|
462 |
try:
|
|
|
463 |
self.tmp.write(self.file_tag.strip() + "\n")
|
|
|
464 |
except:
|
|
|
465 |
return self.err("could not write to temporary file")
|
|
|
466 |
|
|
|
467 |
# even if we ignore an element we need to mark its coming and going
|
|
|
468 |
# so that we know which element any character data belongs to.
|
|
|
469 |
|
|
|
470 |
def generic_start(self, name):
|
|
|
471 |
self.elements.append(name)
|
|
|
472 |
|
|
|
473 |
def generic_end(self):
|
|
|
474 |
self.elements.pop()
|
|
|
475 |
|
|
|
476 |
# text classification
|
|
|
477 |
|
|
|
478 |
def classify(self, text):
|
|
|
479 |
"test the text for errors, warnings and remarks."
|
|
|
480 |
|
|
|
481 |
# there shouldn't actually be any errors in here because we
|
|
|
482 |
# are only looking at "ok" recipes... BUT there are bad tools
|
|
|
483 |
# out there which don't set an error code when they fail, so
|
|
|
484 |
# we should look out for those cases.
|
|
|
485 |
|
|
|
486 |
for line in text.splitlines():
|
|
|
487 |
if not line or line.startswith("+"):
|
|
|
488 |
continue # it is a blank line or a command, not its output
|
|
|
489 |
|
|
|
490 |
# the first expression that matches wins
|
|
|
491 |
for r in self.regex:
|
|
|
492 |
if r[0].search(line):
|
|
|
493 |
return r[1]
|
|
|
494 |
|
|
|
495 |
return Records.OK
|
|
|
496 |
|
|
|
497 |
# reporting of "errors" to separate files
|
|
|
498 |
|
|
|
499 |
def record(self, taggedtext, type):
|
|
|
500 |
if self.totals.isempty(type):
|
|
|
501 |
self.createoverallfile(type)
|
|
|
502 |
self.appendoverallfile(type, taggedtext)
|
|
|
503 |
|
|
|
504 |
configuration = taggedtext.config
|
|
|
505 |
|
|
|
506 |
if configuration in self.configurations:
|
|
|
507 |
if self.configurations[configuration].isempty(type):
|
|
|
508 |
self.createconfigurationfile(configuration, type)
|
|
|
509 |
|
|
|
510 |
self.appendconfigurationfile(configuration, type, taggedtext)
|
|
|
511 |
else:
|
|
|
512 |
# first time for configuration
|
|
|
513 |
self.configurations[configuration] = Records()
|
|
|
514 |
self.createconfigurationfile(configuration, type)
|
|
|
515 |
self.appendconfigurationfile(configuration, type, taggedtext)
|
|
|
516 |
|
|
|
517 |
component = taggedtext.bldinf
|
|
|
518 |
|
|
|
519 |
if component in self.components:
|
|
|
520 |
if self.components[component].isempty(type):
|
|
|
521 |
self.createcomponentfile(component, type)
|
|
|
522 |
|
|
|
523 |
self.appendcomponentfile(component, type, taggedtext)
|
|
|
524 |
else:
|
|
|
525 |
# first time for component
|
|
|
526 |
self.components[component] = Records()
|
|
|
527 |
self.createcomponentfile(component, type)
|
|
|
528 |
self.appendcomponentfile(component, type, taggedtext)
|
|
|
529 |
|
|
|
530 |
def createoverallfile(self, type):
|
|
|
531 |
if type == Records.OK:
|
|
|
532 |
# we don't want to show successes, just count them
|
|
|
533 |
return
|
|
|
534 |
|
|
|
535 |
linkname = os.path.join(Records.CLASSES[type], "overall.html")
|
|
|
536 |
filename = os.path.join(self.dirname, linkname)
|
|
|
537 |
title = Records.TITLES[type] + " for all configurations"
|
|
|
538 |
try:
|
|
|
539 |
file = open(filename, "w")
|
|
|
540 |
file.write("<html><head><title>%s</title>" % title)
|
|
|
541 |
file.write('<link type="text/css" rel="stylesheet" href="../style.css"></head><body>')
|
|
|
542 |
file.write("<h1>%s</h1>" % title)
|
|
|
543 |
file.close()
|
|
|
544 |
except:
|
|
|
545 |
return self.err("cannot create file '%s'" % filename)
|
|
|
546 |
|
|
|
547 |
self.totals.set_filename(type, filename)
|
|
|
548 |
self.totals.set_linkname(type, linkname)
|
|
|
549 |
|
|
|
550 |
def appendoverallfile(self, type, taggedtext):
|
|
|
551 |
self.totals.inc(type) # one more and counting
|
|
|
552 |
self.totals.inc(Records.TIME, taggedtext.time)
|
|
|
553 |
|
|
|
554 |
if type == Records.OK:
|
|
|
555 |
# we don't want to show successes, just count them
|
|
|
556 |
return
|
|
|
557 |
|
|
|
558 |
filename = self.totals.get_filename(type)
|
|
|
559 |
try:
|
|
|
560 |
file = open(filename, "a")
|
|
|
561 |
file.write("<p>component: %s " % taggedtext.bldinf)
|
|
|
562 |
file.write("config: %s\n" % taggedtext.config)
|
|
|
563 |
file.write("<pre>" + taggedtext.text.strip() + "</pre>")
|
|
|
564 |
file.close()
|
|
|
565 |
except:
|
|
|
566 |
return self.err("cannot append to file '%s'" % filename)
|
|
|
567 |
|
|
|
568 |
def createconfigurationfile(self, configuration, type):
|
|
|
569 |
if type == Records.OK:
|
|
|
570 |
# we don't want to show successes, just count them
|
|
|
571 |
return
|
|
|
572 |
|
|
|
573 |
linkname = os.path.join(Records.CLASSES[type], "cfg_" + configuration + ".html")
|
|
|
574 |
filename = os.path.join(self.dirname, linkname)
|
|
|
575 |
title = Records.TITLES[type] + " for configuration " + configuration
|
|
|
576 |
try:
|
|
|
577 |
file = open(filename, "w")
|
|
|
578 |
file.write("<html><head><title>%s</title>" % title)
|
|
|
579 |
file.write('<link type="text/css" rel="stylesheet" href="../style.css"></head><body>')
|
|
|
580 |
file.write("<h1>%s</h1>" % title)
|
|
|
581 |
file.close()
|
|
|
582 |
except:
|
|
|
583 |
return self.err("cannot create file '%s'" % filename)
|
|
|
584 |
|
|
|
585 |
self.configurations[configuration].set_filename(type, filename)
|
|
|
586 |
self.configurations[configuration].set_linkname(type, linkname)
|
|
|
587 |
|
|
|
588 |
def appendconfigurationfile(self, configuration, type, taggedtext):
|
|
|
589 |
self.configurations[configuration].inc(type) # one more and counting
|
|
|
590 |
self.configurations[configuration].inc(Records.TIME, taggedtext.time)
|
|
|
591 |
|
|
|
592 |
if type == Records.OK:
|
|
|
593 |
# we don't want to show successes, just count them
|
|
|
594 |
return
|
|
|
595 |
|
|
|
596 |
filename = self.configurations[configuration].get_filename(type)
|
|
|
597 |
try:
|
|
|
598 |
file = open(filename, "a")
|
|
|
599 |
file.write("<p>component: %s\n" % taggedtext.bldinf)
|
|
|
600 |
file.write("<pre>" + taggedtext.text.strip() + "</pre>")
|
|
|
601 |
file.close()
|
|
|
602 |
except:
|
|
|
603 |
return self.err("cannot append to file '%s'" % filename)
|
|
|
604 |
|
|
|
605 |
def createcomponentfile(self, component, type):
|
|
|
606 |
if type == Records.OK:
|
|
|
607 |
# we don't want to show successes, just count them
|
|
|
608 |
return
|
|
|
609 |
|
|
|
610 |
linkname = os.path.join(Records.CLASSES[type], "bld_" + re.sub("[/:]","_",component) + ".html")
|
|
|
611 |
filename = os.path.join(self.dirname, linkname)
|
|
|
612 |
title = Records.TITLES[type] + " for component " + component
|
|
|
613 |
try:
|
|
|
614 |
file = open(filename, "w")
|
|
|
615 |
file.write("<html><head><title>%s</title>" % title)
|
|
|
616 |
file.write('<link type="text/css" rel="stylesheet" href="../style.css"></head><body>')
|
|
|
617 |
file.write("<h1>%s</h1>" % title)
|
|
|
618 |
file.close()
|
|
|
619 |
except:
|
|
|
620 |
return self.err("cannot create file '%s'" % filename)
|
|
|
621 |
|
|
|
622 |
self.components[component].set_filename(type, filename)
|
|
|
623 |
self.components[component].set_linkname(type, linkname)
|
|
|
624 |
|
|
|
625 |
def appendcomponentfile(self, component, type, taggedtext):
|
|
|
626 |
self.components[component].inc(type) # one more and counting
|
|
|
627 |
self.components[component].inc(Records.TIME, taggedtext.time)
|
|
|
628 |
|
|
|
629 |
if type == Records.OK:
|
|
|
630 |
# we don't want to show successes, just count them
|
|
|
631 |
return
|
|
|
632 |
|
|
|
633 |
filename = self.components[component].get_filename(type)
|
|
|
634 |
try:
|
|
|
635 |
file = open(filename, "a")
|
|
|
636 |
file.write("<p>config: %s\n" % taggedtext.config)
|
|
|
637 |
file.write("<pre>" + taggedtext.text.strip() + "</pre>")
|
|
|
638 |
file.close()
|
|
|
639 |
except:
|
|
|
640 |
return self.err("cannot append to file '%s'" % filename)
|
|
|
641 |
|
|
|
642 |
def existencechecks(self):
|
|
|
643 |
try:
|
|
|
644 |
self.tmp.flush() # write what is left in the buffer
|
|
|
645 |
self.tmp.seek(0) # rewind to the beginning
|
|
|
646 |
|
|
|
647 |
missing_tag = TaggedText({})
|
|
|
648 |
missed = set() # only report missing files once
|
|
|
649 |
|
|
|
650 |
for line in self.tmp.readlines():
|
|
|
651 |
if line.startswith("|"):
|
|
|
652 |
parts = line.split("|")
|
|
|
653 |
attribs = { 'bldinf' : parts[1].strip(),
|
|
|
654 |
'config' : parts[2].strip() }
|
|
|
655 |
missing_tag = TaggedText(attribs)
|
|
|
656 |
else:
|
|
|
657 |
filename = line.strip()
|
|
|
658 |
if filename.startswith(">"):
|
|
|
659 |
# a clean target, so we don't care if it exists
|
|
|
660 |
# but we care if it has a missing dependency
|
|
|
661 |
filename = filename[1:]
|
|
|
662 |
else:
|
|
|
663 |
# a releasable target so it must exist
|
|
|
664 |
if not filename in missed and not os.path.isfile(filename):
|
|
|
665 |
missing_tag.text = filename
|
|
|
666 |
self.record(missing_tag, Records.MISSING)
|
|
|
667 |
missed.add(filename)
|
|
|
668 |
|
|
|
669 |
if filename in self.missed_depends:
|
|
|
670 |
missing_tag.text = filename + \
|
|
|
671 |
"\n\nrequired the following files which could not be found,\n\n"
|
|
|
672 |
for dep in self.missed_depends[filename]:
|
|
|
673 |
missing_tag.text += dep + "\n"
|
|
|
674 |
self.record(missing_tag, Records.ERROR)
|
|
|
675 |
del self.missed_depends[filename]
|
|
|
676 |
|
|
|
677 |
self.tmp.close() # this also deletes the temporary file
|
|
|
678 |
|
|
|
679 |
# any missed dependencies left over are not attributable to any
|
|
|
680 |
# specific component but should still be reported
|
|
|
681 |
missing_tag = TaggedText({})
|
|
|
682 |
for filename in self.missed_depends:
|
|
|
683 |
missing_tag.text = filename + \
|
|
|
684 |
"\n\nrequired the following files which could not be found,\n\n"
|
|
|
685 |
for dep in self.missed_depends[filename]:
|
|
|
686 |
missing_tag.text += dep + "\n"
|
|
|
687 |
self.record(missing_tag, Records.ERROR)
|
|
|
688 |
|
|
|
689 |
except Exception,e:
|
|
|
690 |
return self.err("could not close temporary file " + str(e))
|
|
|
691 |
|
|
|
692 |
def dumptotals(self):
|
|
|
693 |
"""write the numbers of errors, warnings etc. into a text file.
|
|
|
694 |
|
|
|
695 |
so that a grand summariser can tie together individual log summaries
|
|
|
696 |
into one big summary page."""
|
|
|
697 |
try:
|
|
|
698 |
filename = os.path.join(self.dirname, "totals.txt")
|
|
|
699 |
file = open(filename, "w")
|
|
|
700 |
file.write(self.totals.textdump())
|
|
|
701 |
file.close()
|
|
|
702 |
except:
|
|
|
703 |
self.err("cannot write totals file '%s'" % filename)
|
|
|
704 |
|
|
|
705 |
def readregex(self, csvfile):
|
|
|
706 |
"""read the list of regular expressions from a csv file.
|
|
|
707 |
|
|
|
708 |
the file format is TYPE,REGEX,DESCRIPTION
|
|
|
709 |
|
|
|
710 |
If the description is "ignorecase" then the regular expression is
|
|
|
711 |
compiled with re.IGNORECASE and will match case-insensitively.
|
|
|
712 |
"""
|
|
|
713 |
regexlist = []
|
|
|
714 |
try:
|
|
|
715 |
reader = csv.reader(open(csvfile, "rb"))
|
|
|
716 |
for row in reader:
|
|
|
717 |
try:
|
|
|
718 |
type = None
|
|
|
719 |
|
|
|
720 |
if row[0] == "ERROR":
|
|
|
721 |
type = Records.ERROR
|
|
|
722 |
elif row[0] == "CRITICAL":
|
|
|
723 |
type = Records.CRITICAL
|
|
|
724 |
elif row[0] == "WARNING":
|
|
|
725 |
type = Records.WARNING
|
|
|
726 |
elif row[0] == "REMARK":
|
|
|
727 |
type = Records.REMARK
|
|
|
728 |
|
|
|
729 |
# there are other types like INFO that we don't
|
|
|
730 |
# care about so silently ignore them.
|
|
|
731 |
if type:
|
|
|
732 |
if row[2].lower() == "ignorecase":
|
|
|
733 |
regex = re.compile(row[1], re.I)
|
|
|
734 |
else:
|
|
|
735 |
regex = re.compile(row[1])
|
|
|
736 |
regexlist.append((regex, type))
|
|
|
737 |
except:
|
|
|
738 |
self.moan("ignored bad regex '%s' in file '%s'" % (row[1], csvfile))
|
|
|
739 |
except Exception, ex:
|
|
|
740 |
self.err("cannot read regex file '%s': %s" % (csvfile, str(ex)))
|
|
|
741 |
return []
|
|
|
742 |
|
|
|
743 |
return regexlist
|
|
|
744 |
|
|
|
745 |
class CountItem(object):
|
|
|
746 |
def __init__(self):
|
|
|
747 |
self.N = 0
|
|
|
748 |
self.filename = None
|
|
|
749 |
self.linkname = None
|
|
|
750 |
|
|
|
751 |
def num_str(self):
|
|
|
752 |
return str(self.N)
|
|
|
753 |
|
|
|
754 |
class TimeItem(CountItem):
|
|
|
755 |
def num_str(self):
|
|
|
756 |
return time.strftime("%H:%M:%S", time.gmtime(self.N + 0.5))
|
|
|
757 |
|
|
|
758 |
class Records(object):
|
|
|
759 |
"a group of related records e.g. errors, warnings and remarks."
|
|
|
760 |
|
|
|
761 |
# the different types of record we want to group together
|
|
|
762 |
TIME = 0
|
|
|
763 |
OK = 1
|
|
|
764 |
ERROR = 2
|
|
|
765 |
CRITICAL = 3
|
|
|
766 |
WARNING = 4
|
|
|
767 |
REMARK = 5
|
|
|
768 |
MISSING = 6
|
|
|
769 |
|
|
|
770 |
CLASSES = [ "time", "ok", "error", "critical", "warning", "remark", "missing" ]
|
|
|
771 |
TITLES = [ "CPU Time", "OK", "Errors", "Criticals", "Warnings", "Remarks", "Missing files" ]
|
|
|
772 |
|
|
|
773 |
def __init__(self):
|
|
|
774 |
self.data = [ TimeItem(), CountItem(), CountItem(), CountItem(), CountItem(), CountItem(), CountItem() ]
|
|
|
775 |
|
|
|
776 |
def get_filename(self, index):
|
|
|
777 |
return self.data[index].filename
|
|
|
778 |
|
|
|
779 |
def inc(self, index, increment=1):
|
|
|
780 |
self.data[index].N += increment
|
|
|
781 |
|
|
|
782 |
def isempty(self, index):
|
|
|
783 |
return (self.data[index].N == 0)
|
|
|
784 |
|
|
|
785 |
def set_filename(self, index, value):
|
|
|
786 |
self.data[index].filename = value
|
|
|
787 |
|
|
|
788 |
def set_linkname(self, index, value):
|
|
|
789 |
self.data[index].linkname = value
|
|
|
790 |
|
|
|
791 |
def tablerow(self, name):
|
|
|
792 |
row = '<tr><td class="name">%s</td>' % name
|
|
|
793 |
|
|
|
794 |
for i,datum in enumerate(self.data):
|
|
|
795 |
if datum.N == 0:
|
|
|
796 |
row += '<td class="zero">0</td>'
|
|
|
797 |
else:
|
|
|
798 |
row += '<td class="' + Records.CLASSES[i] + '">'
|
|
|
799 |
if datum.linkname:
|
|
|
800 |
row += '<a href="%s">%s</a></td>' % (datum.linkname,datum.num_str())
|
|
|
801 |
else:
|
|
|
802 |
row += '%s</td>' % datum.num_str()
|
|
|
803 |
|
|
|
804 |
row += "</tr>"
|
|
|
805 |
return row
|
|
|
806 |
|
|
|
807 |
def textdump(self):
|
|
|
808 |
text = ""
|
|
|
809 |
for i,datum in enumerate(self.data):
|
|
|
810 |
if datum.N == 0:
|
|
|
811 |
style = "zero"
|
|
|
812 |
else:
|
|
|
813 |
style = Records.CLASSES[i]
|
|
|
814 |
text += str(i) + ',' + style + "," + str(datum.N) + "\n"
|
|
|
815 |
return text
|
|
|
816 |
|
|
|
817 |
class TaggedText(object):
|
|
|
818 |
def __init__(self, attributes):
|
|
|
819 |
|
|
|
820 |
for attrib in ['bldinf', 'config']:
|
|
|
821 |
self.__dict__[attrib] = "unknown"
|
|
|
822 |
if attrib in attributes:
|
|
|
823 |
value = attributes[attrib]
|
|
|
824 |
if value:
|
|
|
825 |
self.__dict__[attrib] = value
|
|
|
826 |
|
|
|
827 |
self.text = ""
|
|
|
828 |
self.time = 0.0
|
|
|
829 |
|
|
|
830 |
# the end |