16 # |
16 # |
17 # display summary information about recipes from raptor logs |
17 # display summary information about recipes from raptor logs |
18 # e.g. total times and so on. |
18 # e.g. total times and so on. |
19 |
19 |
20 import time |
20 import time |
|
21 import __future__ |
|
22 |
|
23 |
|
24 |
21 |
25 |
22 class RecipeStats(object): |
26 class RecipeStats(object): |
|
27 def __init__(self, name, count, time): |
|
28 self.name=name |
|
29 self.count=count |
|
30 self.time=time |
|
31 |
|
32 def add(self, duration): |
|
33 self.time += duration |
|
34 |
|
35 class BuildStats(object): |
23 STAT_OK = 0 |
36 STAT_OK = 0 |
24 |
37 |
25 |
38 |
26 def __init__(self): |
39 def __init__(self): |
27 self.stats = {} |
40 self.stats = {} |
28 self.failcount = 0 |
41 self.failcount = 0 |
29 self.failtime = 0.0 |
42 self.failtime = 0.0 |
30 self.failtypes = {} |
43 self.failtypes = {} |
31 self.retryfails = 0 |
44 self.retryfails = 0 |
|
45 self.hosts = {} |
32 |
46 |
33 def add(self, starttime, duration, name, status): |
47 def add(self, starttime, duration, name, status, host, phase): |
34 if status != RecipeStats.STAT_OK: |
48 if status != BuildStats.STAT_OK: |
35 self.failcount += 1 |
49 self.failcount += 1 |
36 if name in self.failtypes: |
50 if name in self.failtypes: |
37 self.failtypes[name] += 1 |
51 self.failtypes[name] += 1 |
38 else: |
52 else: |
39 self.failtypes[name] = 1 |
53 self.failtypes[name] = 1 |
41 if status == 128: |
55 if status == 128: |
42 self.retryfails += 1 |
56 self.retryfails += 1 |
43 return |
57 return |
44 |
58 |
45 if name in self.stats: |
59 if name in self.stats: |
46 (count, time) = self.stats[name] |
60 r = self.stats[name] |
47 self.stats[name] = (count + 1, time + duration) |
61 r.add(duration) |
48 else: |
62 else: |
49 self.stats[name] = (1,duration) |
63 self.stats[name] = RecipeStats(name,1,duration) |
|
64 |
|
65 hp=host |
|
66 if hp in self.hosts: |
|
67 self.hosts[hp] += 1 |
|
68 else: |
|
69 self.hosts[hp] = 1 |
50 |
70 |
51 def recipe_csv(self): |
71 def recipe_csv(self): |
52 s = "# name, time, count\n" |
72 s = '"name", "time", "count"\n' |
53 for (name,(count,time)) in self.stats.iteritems(): |
73 l = sorted(self.stats.values(), key= lambda r: r.time, reverse=True) |
54 s += '"%s",%s,%d\n' % (name, str(time), count) |
74 for r in l: |
|
75 s += '"%s",%s,%d\n' % (r.name, str(r.time), r.count) |
55 return s |
76 return s |
56 |
77 |
|
78 def hosts_csv(self): |
|
79 s='"host","recipecount"\n' |
|
80 hs = self.hosts |
|
81 for h in sorted(hs.keys()): |
|
82 s += '"%s",%d\n' % (h,hs[h]) |
|
83 return s |
57 |
84 |
58 |
85 |
59 import sys |
86 import sys |
60 import re |
87 import re |
|
88 import os |
|
89 from optparse import OptionParser # for parsing command line parameters |
61 |
90 |
62 def main(): |
91 def main(): |
63 |
92 recipe_re = re.compile(".*<recipe name='([^']+)'.*host='([^']+)'.*") |
64 f = sys.stdin |
|
65 st = RecipeStats() |
|
66 |
|
67 recipe_re = re.compile(".*<recipe name='([^']+)'.*") |
|
68 time_re = re.compile(".*<time start='([0-9]+\.[0-9]+)' *elapsed='([0-9]+\.[0-9]+)'.*") |
93 time_re = re.compile(".*<time start='([0-9]+\.[0-9]+)' *elapsed='([0-9]+\.[0-9]+)'.*") |
69 status_re = re.compile(".*<status exit='(?P<exit>(ok|failed))'( *code='(?P<code>[0-9]+)')?.*") |
94 status_re = re.compile(".*<status exit='(?P<exit>(ok|failed))'( *code='(?P<code>[0-9]+)')?.*") |
|
95 phase_re = re.compile(".*<info>Making.*?([^\.]+\.[^\.]+)</info>") |
|
96 |
|
97 parser = OptionParser(prog = "recipestats", |
|
98 usage = """%prog --help [-b] [-f <logfilename>]""") |
|
99 |
|
100 parser.add_option("-b","--buildhosts",action="store_true",dest="buildhosts_flag", |
|
101 help="Lists which build hosts were active in each invocation of the build engine and how many recipes ran on each.", default = False) |
|
102 parser.add_option("-f","--logfile",action="store",dest="logfilename", help="Read from the file, not stdin", default = None) |
|
103 |
|
104 |
|
105 (options, stuff) = parser.parse_args(sys.argv[1:]) |
|
106 |
|
107 if options.logfilename is None: |
|
108 f = sys.stdin |
|
109 else: |
|
110 f = open(options.logfilename,"r") |
|
111 |
|
112 st = BuildStats() |
|
113 |
70 |
114 |
71 alternating = 0 |
115 alternating = 0 |
72 start_time = 0.0 |
116 start_time = 0.0 |
73 |
117 |
74 |
118 phase=None |
75 for l in f.xreadlines(): |
119 for l in f: |
76 l2 = l.rstrip("\n\r") |
120 l2 = l.rstrip("\n\r") |
|
121 |
77 rm = recipe_re.match(l2) |
122 rm = recipe_re.match(l2) |
78 |
123 |
79 if rm is not None: |
124 if rm is not None: |
80 rname = rm.groups()[0] |
125 (rname,host) = rm.groups() |
81 continue |
126 continue |
82 |
127 |
|
128 pm = phase_re.match(l2) |
|
129 |
|
130 if pm is not None: |
|
131 if phase is not None: |
|
132 if options.buildhosts_flag: |
|
133 print('"%s"\n' % phase) |
|
134 print(st.hosts_csv()) |
|
135 st.hosts = {} |
|
136 phase = pm.groups()[0] |
|
137 continue |
83 |
138 |
84 tm = time_re.match(l2) |
139 tm = time_re.match(l2) |
85 if tm is not None: |
140 if tm is not None: |
86 try: |
141 try: |
87 s = float(tm.groups()[0]) |
142 s = float(tm.groups()[0]) |
107 if sm.groupdict()['exit'] == 'ok': |
162 if sm.groupdict()['exit'] == 'ok': |
108 status = 0 |
163 status = 0 |
109 else: |
164 else: |
110 status = int(sm.groupdict()['code']) |
165 status = int(sm.groupdict()['code']) |
111 |
166 |
112 st.add(s, elapsed, rname, status) |
167 st.add(s, elapsed, rname, status, host, phase) |
113 |
168 |
114 print st.recipe_csv() |
169 if options.buildhosts_flag: |
|
170 print('"%s"\n' % phase) |
|
171 print(st.hosts_csv()) |
|
172 else: |
|
173 print(st.recipe_csv()) |
115 |
174 |
116 |
175 |
117 if __name__ == '__main__': main() |
176 if __name__ == '__main__': main() |