18 |
18 |
19 import hashlib |
19 import hashlib |
20 import os |
20 import os |
21 import random |
21 import random |
22 import raptor |
22 import raptor |
23 import raptor_data |
23 import raptor_timing |
24 import raptor_utilities |
24 import raptor_utilities |
25 import raptor_version |
25 import raptor_version |
|
26 import raptor_data |
26 import re |
27 import re |
27 import subprocess |
28 import subprocess |
28 import time |
29 import time |
29 from raptor_makefile import * |
30 from raptor_makefile import * |
|
31 import traceback |
|
32 import sys |
|
33 from xml.sax.saxutils import escape |
|
34 from xml.sax.saxutils import unescape |
|
35 |
|
36 |
|
37 class BadMakeEngineException(Exception): |
|
38 pass |
|
39 |
|
40 def string_following(prefix, str): |
|
41 """If str starts with prefix then return the rest of str, otherwise None""" |
|
42 if str.startswith(prefix): |
|
43 return str[len(prefix):] |
|
44 else: |
|
45 return None |
|
46 |
|
47 def XMLEscapeLog(stream): |
|
48 """ A generator that reads a raptor log from a stream and performs an XML escape |
|
49 on all text between tags, which is usually make output that could contain |
|
50 illegal characters that upset XML-based log parsers. |
|
51 This function yields "xml-safe" output line by line. |
|
52 """ |
|
53 inRecipe = False |
|
54 |
|
55 for line in stream: |
|
56 if line.startswith("<recipe"): |
|
57 inRecipe = True |
|
58 elif line.startswith("</recipe"): |
|
59 inRecipe = False |
|
60 |
|
61 # unless we are inside a "recipe", any line not starting |
|
62 # with "<" is free text that must be escaped. |
|
63 if inRecipe or line.startswith("<"): |
|
64 yield line |
|
65 else: |
|
66 yield escape(line) |
|
67 |
|
68 def AnnoFileParseOutput(annofile): |
|
69 """ A generator that extracts log output from an emake annotation file, |
|
70 perform an XML-unescape on it and "yields" it line by line. """ |
|
71 if isinstance(annofile,str): |
|
72 af = open(annofile, "r") |
|
73 else: |
|
74 af = annofile |
|
75 |
|
76 inOutput = False |
|
77 |
|
78 buildid = "" |
|
79 for line in af: |
|
80 line = line.rstrip("\n\r") |
|
81 |
|
82 |
|
83 if not inOutput: |
|
84 o = string_following("<output>", line) |
|
85 if not o: |
|
86 o = string_following('<output src="prog">', line) |
|
87 |
|
88 if o: |
|
89 inOutput = True |
|
90 yield unescape(o)+'\n' |
|
91 continue |
|
92 |
|
93 |
|
94 o = string_following('<build id="',line) |
|
95 if o: |
|
96 buildid = o[:o.find('"')] |
|
97 yield "Starting build: "+buildid+"\n" |
|
98 continue |
|
99 |
|
100 o = string_following('<metric name="duration">', line) |
|
101 if o: |
|
102 secs = int(o[:o.find('<')]) |
|
103 if secs != 0: |
|
104 duration = "%d:%d" % (secs/60, secs % 60) |
|
105 else: |
|
106 duration = "0:0" |
|
107 continue |
|
108 |
|
109 |
|
110 o = string_following('<metric name="clusterAvailability">', line) |
|
111 if o: |
|
112 availability = o[:o.find('<')] |
|
113 continue |
|
114 |
|
115 else: |
|
116 end_output = line.find("</output>") |
|
117 |
|
118 if end_output != -1: |
|
119 line = line[:end_output] |
|
120 inOutput = False |
|
121 |
|
122 if line != "": |
|
123 yield unescape(line)+'\n' |
|
124 |
|
125 yield "Finished build: %s Duration: %s (m:s) Cluster availability: %s%%\n" %(buildid,duration,availability) |
|
126 af.close() |
|
127 |
|
128 |
30 |
129 |
31 # raptor_make module classes |
130 # raptor_make module classes |
32 |
131 |
33 class MakeEngine(object): |
132 class MakeEngine(object): |
34 |
133 |
35 def __init__(self, Raptor): |
134 def __init__(self, Raptor, engine="make_engine"): |
36 self.raptor = Raptor |
135 self.raptor = Raptor |
37 self.valid = True |
136 self.valid = True |
38 self.makefileset = None |
|
39 self.descrambler = None |
137 self.descrambler = None |
40 self.descrambler_started = False |
138 self.descrambler_started = False |
41 |
139 |
42 engine = Raptor.makeEngine |
|
43 |
|
44 # look for an alias first as this gives end-users a chance to modify |
140 # look for an alias first as this gives end-users a chance to modify |
45 # the shipped variant rather than completely replacing it. |
141 # the shipped variant rather than completely replacing it. |
46 if engine in Raptor.cache.aliases: |
142 if engine in Raptor.cache.aliases: |
47 avar = Raptor.cache.FindNamedAlias(engine) |
143 avar = Raptor.cache.FindNamedAlias(engine) |
48 elif engine in Raptor.cache.variants: |
144 elif engine in Raptor.cache.variants: |
49 avar = Raptor.cache.FindNamedVariant(engine) |
145 avar = Raptor.cache.FindNamedVariant(engine) |
50 else: |
146 else: |
51 Raptor.Error("No settings found for build engine '%s'", engine) |
147 raise BadMakeEngineException("'%s' does not appear to be a make engine - no settings found for it" % engine) |
52 return |
148 |
|
149 if not avar.isDerivedFrom("make_engine", Raptor.cache): |
|
150 raise BadMakeEngineException("'%s' is not a build engine (it's a variant but it does not extend 'make_engine')" % engine) |
53 |
151 |
54 # find the variant and extract the values |
152 # find the variant and extract the values |
55 try: |
153 try: |
56 units = avar.GenerateBuildUnits() |
154 units = avar.GenerateBuildUnits(Raptor.cache) |
57 evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True) |
155 evaluator = Raptor.GetEvaluator( None, units[0] , gathertools=True) |
58 |
156 |
59 # shell |
157 # shell |
60 self.shellpath = evaluator.Get("DEFAULT_SHELL") |
158 self.shellpath = evaluator.Get("DEFAULT_SHELL") |
61 usetalon_s = evaluator.Get("USE_TALON") |
159 usetalon_s = evaluator.Get("USE_TALON") |
62 self.usetalon = usetalon_s is not None and usetalon_s != "" |
160 self.usetalon = usetalon_s is not None and usetalon_s != "" |
63 self.talonshell = str(evaluator.Get("TALON_SHELL")) |
161 self.talonshell = str(evaluator.Get("TALON_SHELL")) |
64 self.talontimeout = str(evaluator.Get("TALON_TIMEOUT")) |
162 self.talontimeout = str(evaluator.Get("TALON_TIMEOUT")) |
65 self.talonretries = str(evaluator.Get("TALON_RETRIES")) |
163 self.talonretries = str(evaluator.Get("TALON_RETRIES")) |
|
164 |
|
165 # work around for RVCT 2.2 failed compiles |
|
166 delete_on_failed_compile_s = evaluator.Get("DELETE_ON_FAILED_COMPILE") |
|
167 self.delete_on_failed_compile = "" |
|
168 if delete_on_failed_compile_s is not None and delete_on_failed_compile_s != "": |
|
169 self.delete_on_failed_compile = "1" |
66 |
170 |
67 # commands |
171 # commands |
68 self.initCommand = evaluator.Get("initialise") |
172 self.initCommand = evaluator.Get("initialise") |
69 self.buildCommand = evaluator.Get("build") |
173 self.buildCommand = evaluator.Get("build") |
70 self.shutdownCommand = evaluator.Get("shutdown") |
174 self.shutdownCommand = evaluator.Get("shutdown") |
72 # options |
176 # options |
73 self.makefileOption = evaluator.Get("makefile") |
177 self.makefileOption = evaluator.Get("makefile") |
74 self.keepGoingOption = evaluator.Get("keep_going") |
178 self.keepGoingOption = evaluator.Get("keep_going") |
75 self.jobsOption = evaluator.Get("jobs") |
179 self.jobsOption = evaluator.Get("jobs") |
76 self.defaultMakeOptions = evaluator.Get("defaultoptions") |
180 self.defaultMakeOptions = evaluator.Get("defaultoptions") |
|
181 |
|
182 # Logging |
|
183 # copylogfromannofile means, for emake, that we should ignore |
|
184 # emake's console output and instead extract output from its annotation |
|
185 # file. This is a workaround for a problem where some emake |
|
186 # console output is lost. The annotation file has a copy of this |
|
187 # output in the "parse" job and it turns out to be uncorrupted. |
|
188 self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true") |
|
189 self.annoFileName = None |
|
190 |
|
191 if self.copyLogFromAnnoFile: |
|
192 for o in self.raptor.makeOptions: |
|
193 self.annoFileName = string_following("--emake-annofile=", o) |
|
194 if self.annoFileName: |
|
195 self.raptor.Info("annofile: " + o) |
|
196 |
|
197 if not self.annoFileName: |
|
198 self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>") |
|
199 self.copyLogFromAnnoFile = False |
77 |
200 |
78 # buffering |
201 # buffering |
79 self.scrambled = (evaluator.Get("scrambled") == "true") |
202 self.scrambled = (evaluator.Get("scrambled") == "true") |
80 |
203 |
81 # check tool versions |
204 # check tool versions |
121 host='$$HOSTNAME'\ |
243 host='$$HOSTNAME'\ |
122 layer='$$COMPONENT_LAYER'\ |
244 layer='$$COMPONENT_LAYER'\ |
123 component='$$COMPONENT_NAME'\ |
245 component='$$COMPONENT_NAME'\ |
124 bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\ |
246 bldinf='$$COMPONENT_META' mmp='$$PROJECT_META'\ |
125 config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\ |
247 config='$$SBS_CONFIGURATION' platform='$$PLATFORM'\ |
126 phase='$$MAKEFILE_GROUP' source='$$SOURCE |
248 phase='$$MAKEFILE_GROUP' source='$$SOURCE' |
127 export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT |
249 export TALON_RECIPEATTRIBUTES TALON_SHELL TALON_TIMEOUT |
128 USE_TALON:=%s |
250 USE_TALON:=%s |
129 |
251 |
130 """ % (self.talonshell, self.talontimeout, "1") |
252 """ % (self.talonshell, self.talontimeout, "1") |
131 else: |
253 else: |
132 talon_settings=""" |
254 talon_settings=""" |
133 USE_TALON:= |
255 USE_TALON:= |
134 |
256 |
135 """ |
257 """ |
136 |
258 |
|
259 |
|
260 timing_start = "$(info " + \ |
|
261 raptor_timing.Timing.custom_string(tag = "start", |
|
262 object_type = "makefile", task = "parse", |
|
263 key = "$(THIS_FILENAME)", |
|
264 time="$(shell date +%s.%N)").rstrip("\n") + ")" |
|
265 |
|
266 timing_end = "$(info " + \ |
|
267 raptor_timing.Timing.custom_string(tag = "end", |
|
268 object_type = "makefile", task = "parse", |
|
269 key = "$(THIS_FILENAME)", |
|
270 time="$(shell date +%s.%N)").rstrip("\n") + ")" |
|
271 |
|
272 |
|
273 # Debugging on or off for make: |
|
274 # We need it at the very top level so that it can be used |
|
275 # to determine what extra info to put in recipe tags |
|
276 try: |
|
277 flmdebug_setting = os.environ["FLMDEBUG"] |
|
278 except KeyError: |
|
279 flmdebug_setting = "" |
137 |
280 |
138 self.makefile_prologue = """ |
281 self.makefile_prologue = """ |
|
282 |
139 # generated by %s %s |
283 # generated by %s %s |
140 |
284 |
141 HOSTPLATFORM:=%s |
285 HOSTPLATFORM:=%s |
142 HOSTPLATFORM_DIR:=%s |
286 HOSTPLATFORM_DIR:=%s |
143 OSTYPE:=%s |
287 OSTYPE:=%s |
144 FLMHOME:=%s |
288 FLMHOME:=%s |
145 SHELL:=%s |
289 SHELL:=%s |
|
290 THIS_FILENAME:=$(firstword $(MAKEFILE_LIST)) |
|
291 DELETE_ON_FAILED_COMPILE:=%s |
146 |
292 |
147 %s |
293 %s |
|
294 FLMDEBUG:=%s |
148 |
295 |
149 include %s |
296 include %s |
150 |
297 |
151 """ % ( raptor.name, raptor_version.Version(), |
298 """ % ( raptor.name, raptor_version.fullversion(), |
152 " ".join(raptor.hostplatform), |
299 " ".join(raptor.hostplatform), |
153 raptor.hostplatform_dir, |
300 raptor.hostplatform_dir, |
154 self.raptor.filesystem, |
301 self.raptor.filesystem, |
155 str(self.raptor.systemFLM), |
302 str(self.raptor.systemFLM), |
156 self.shellpath, |
303 self.shellpath, |
|
304 self.delete_on_failed_compile, |
157 talon_settings, |
305 talon_settings, |
|
306 flmdebug_setting, |
158 self.raptor.systemFLM.Append('globals.mk') ) |
307 self.raptor.systemFLM.Append('globals.mk') ) |
159 |
308 |
160 |
309 # Unless dependency processing has been eschewed via the CLI, use a .DEFAULT target to |
161 self.makefile_epilogue = """ |
310 # trap missing dependencies (ignoring user config files that we know are usually absent) |
|
311 if not (self.raptor.noDependGenerate or self.raptor.noDependInclude): |
|
312 self.makefile_prologue += """ |
|
313 |
|
314 $(FLMHOME)/user/final.mk: |
|
315 $(FLMHOME)/user/default.flm: |
|
316 $(FLMHOME)/user/globals.mk: |
|
317 |
|
318 .DEFAULT:: |
|
319 @echo "<warning>Missing dependency detected: $@</warning>" |
|
320 |
|
321 """ |
|
322 |
|
323 # Only output timings if requested on CLI |
|
324 if self.raptor.timing: |
|
325 self.makefile_prologue += "\n# Print Start-time of Makefile parsing\n" \ |
|
326 + timing_start + "\n\n" |
|
327 |
|
328 |
|
329 self.makefile_epilogue = "\n\n# Print End-time of Makefile parsing\n" \ |
|
330 + timing_end + "\n" |
|
331 else: |
|
332 self.makefile_epilogue = "" |
|
333 |
|
334 self.makefile_epilogue += """ |
162 |
335 |
163 include %s |
336 include %s |
164 |
337 |
165 """ % (self.raptor.systemFLM.Append('final.mk') ) |
338 """ % (self.raptor.systemFLM.Append('final.mk') ) |
166 |
339 |
167 def Write(self, toplevel, specs, configs): |
340 def Write(self, toplevel, specs, configs): |
168 """Generate a set of makefiles, or one big Makefile.""" |
341 """Generate a set of makefiles, or one big Makefile.""" |
169 |
342 |
170 if not self.valid: |
343 if not self.valid: |
171 return |
344 return None |
|
345 |
|
346 self.raptor.Debug("Writing Makefile '%s'" % (str(toplevel))) |
172 |
347 |
173 self.toplevel = toplevel |
348 self.toplevel = toplevel |
174 |
349 |
175 # create the top-level makefiles |
350 # create the top-level makefiles |
|
351 makefileset = None |
176 |
352 |
177 try: |
353 try: |
178 self.makefileset = MakefileSet(directory = str(toplevel.Dir()), |
354 makefileset = MakefileSet(directory = str(toplevel.Dir()), |
179 selectors = self.selectors, |
355 selectors = self.selectors, |
180 filenamebase = str(toplevel.File()), |
356 filenamebase = str(toplevel.File()), |
181 prologue = self.makefile_prologue, |
357 prologue = self.makefile_prologue, |
182 epilogue = self.makefile_epilogue, |
358 epilogue = self.makefile_epilogue, |
183 defaulttargets = self.defaultTargets) |
359 defaulttargets = self.defaultTargets) |
205 config_wide_spec = s |
380 config_wide_spec = s |
206 else: |
381 else: |
207 ordered_specs.append(s) |
382 ordered_specs.append(s) |
208 |
383 |
209 if config_wide_spec is not None: |
384 if config_wide_spec is not None: |
210 config_wide_spec.Configure(c) |
385 config_wide_spec.Configure(c, cache = self.raptor.cache) |
211 self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True) |
386 self.WriteConfiguredSpec(config_makefileset, config_wide_spec, c, True) |
212 |
387 |
213 for s in ordered_specs: |
388 for s in ordered_specs: |
214 s.Configure(c) |
389 s.Configure(c, cache = self.raptor.cache) |
215 self.WriteConfiguredSpec(config_makefileset, s, c, False) |
390 self.WriteConfiguredSpec(config_makefileset, s, c, False) |
216 |
391 |
217 self.makefileset.close() |
392 makefileset.close() |
218 except Exception,e: |
393 except Exception,e: |
219 self.raptor.Error("Failed to write makefile '%s': %s" % (str(toplevel),str(e))) |
394 tb = traceback.format_exc() |
|
395 if not self.raptor.debugOutput: |
|
396 tb="" |
|
397 self.raptor.Error("Failed to write makefile '%s': %s : %s" % (str(toplevel),str(e),tb)) |
|
398 makefileset = None |
|
399 |
|
400 return makefileset |
220 |
401 |
221 |
402 |
222 def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces): |
403 def WriteConfiguredSpec(self, parentMakefileSet, spec, config, useAllInterfaces): |
223 # ignore this spec if it is empty |
404 # ignore this spec if it is empty |
224 hasInterface = spec.HasInterface() |
405 hasInterface = spec.HasInterface() |
266 |
448 |
267 parameters.append((k, value)) |
449 parameters.append((k, value)) |
268 md5hash.update(value) |
450 md5hash.update(value) |
269 |
451 |
270 # parameters required by the interface |
452 # parameters required by the interface |
271 for p in iface.GetParams(): |
453 for p in iface.GetParams(self.raptor.cache): |
272 val = evaluator.Resolve(p.name) |
454 val = evaluator.Resolve(p.name) |
273 addparam(p.name,val,p.default) |
455 addparam(p.name,val,p.default) |
274 |
456 |
275 # Use Patterns to fetch a group of parameters |
457 # Use Patterns to fetch a group of parameters |
276 for g in iface.GetParamGroups(): |
458 for g in iface.GetParamGroups(self.raptor.cache): |
277 for k,v in evaluator.ResolveMatching(g.patternre): |
459 for k,v in evaluator.ResolveMatching(g.patternre): |
278 addparam(k,v,g.default) |
460 addparam(k,v,g.default) |
279 |
461 |
280 hash = md5hash.hexdigest() |
462 hash = md5hash.hexdigest() |
281 dupe = hash in self.hashes |
463 dupe = hash in self.hashes |
375 # Set default options first so that they can be overridden by |
560 # Set default options first so that they can be overridden by |
376 # ones set by the --mo option on the raptor commandline: |
561 # ones set by the --mo option on the raptor commandline: |
377 command += " " + self.defaultMakeOptions |
562 command += " " + self.defaultMakeOptions |
378 # Can supply options on the commandline to override default settings. |
563 # Can supply options on the commandline to override default settings. |
379 if len(self.raptor.makeOptions) > 0: |
564 if len(self.raptor.makeOptions) > 0: |
380 command += " " + " ".join(self.raptor.makeOptions) |
565 for o in self.raptor.makeOptions: |
|
566 if o.find(";") != -1 or o.find("\\") != -1: |
|
567 command += " " + "'" + o + "'" |
|
568 else: |
|
569 command += " " + o |
381 |
570 |
382 # Switch off dependency file including? |
571 # Switch off dependency file including? |
383 if self.raptor.noDependInclude: |
572 if self.raptor.noDependInclude or self.raptor.noDependGenerate: |
384 command += " NO_DEPEND_INCLUDE=1" |
573 command += " NO_DEPEND_INCLUDE=1" |
|
574 |
|
575 # Switch off dependency file generation (and, implicitly, inclusion)? |
|
576 if self.raptor.noDependGenerate: |
|
577 command += " NO_DEPEND_GENERATE=1" |
385 |
578 |
386 if self.usetalon: |
579 if self.usetalon: |
387 # use the descrambler if we set it up |
580 # use the descrambler if we set it up |
388 command += ' TALON_DESCRAMBLE=' |
581 command += ' TALON_DESCRAMBLE=' |
389 if self.scrambled: |
582 if self.scrambled: |
399 command += ' RECIPETRIES=' + str(self.raptor.tries) |
592 command += ' RECIPETRIES=' + str(self.raptor.tries) |
400 command += ' TALON_RETRIES=' + str(self.raptor.tries - 1) |
593 command += ' TALON_RETRIES=' + str(self.raptor.tries - 1) |
401 |
594 |
402 # targets go at the end, if the makefile supports them |
595 # targets go at the end, if the makefile supports them |
403 addTargets = self.raptor.targets[:] |
596 addTargets = self.raptor.targets[:] |
404 ignoreTargets = self.makefileset.ignoreTargets(makefile) |
597 ignoreTargets = makefileset.ignoreTargets(makefile) |
405 if addTargets and ignoreTargets: |
598 if addTargets and ignoreTargets: |
406 for target in self.raptor.targets: |
599 for target in self.raptor.targets: |
407 if re.match(ignoreTargets, target): |
600 if re.match(ignoreTargets, target): |
408 addTargets.remove(target) |
601 addTargets.remove(target) |
409 |
602 |
410 if addTargets: |
603 if addTargets: |
411 command += " " + " ".join(addTargets) |
604 command += " " + " ".join(addTargets) |
|
605 |
|
606 # Send stderr to a file so that it can't mess up the log (e.g. |
|
607 # clock skew messages from some build engines scatter their |
|
608 # output across our xml. |
|
609 stderrfilename = makefile+'.stderr' |
|
610 stdoutfilename = makefile+'.stdout' |
|
611 command += " 2>'%s' " % stderrfilename |
|
612 |
|
613 # Keep a copy of the stdout too in the case of using the |
|
614 # annofile - so that we can trap the problem that |
|
615 # makes the copy-log-from-annofile workaround necessary |
|
616 # and perhaps determine when we can remove it. |
|
617 if self.copyLogFromAnnoFile: |
|
618 command += " >'%s' " % stdoutfilename |
|
619 |
|
620 # Substitute the makefile name for any occurrence of #MAKEFILE# |
|
621 command = command.replace("#MAKEFILE#", str(makefile)) |
412 |
622 |
413 self.raptor.Info("Executing '%s'", command) |
623 self.raptor.Info("Executing '%s'", command) |
414 |
624 |
415 # execute the build. |
625 # execute the build. |
416 # the actual call differs between Windows and Unix. |
626 # the actual call differs between Windows and Unix. |
417 # bufsize=1 means "line buffered" |
627 # bufsize=1 means "line buffered" |
418 # |
628 # |
419 try: |
629 try: |
|
630 # Time the build |
|
631 self.raptor.InfoStartTime(object_type = "makefile", |
|
632 task = "build", key = str(makefile)) |
|
633 |
420 makeenv=os.environ.copy() |
634 makeenv=os.environ.copy() |
421 if self.usetalon: |
635 if self.usetalon: |
422 makeenv['TALON_RECIPEATTRIBUTES']="none" |
636 makeenv['TALON_RECIPEATTRIBUTES']="none" |
423 makeenv['TALON_SHELL']=self.talonshell |
637 makeenv['TALON_SHELL']=self.talonshell |
424 makeenv['TALON_BUILDID']=str(self.buildID) |
638 makeenv['TALON_BUILDID']=str(self.buildID) |
425 makeenv['TALON_TIMEOUT']=str(self.talontimeout) |
639 makeenv['TALON_TIMEOUT']=str(self.talontimeout) |
|
640 |
426 if self.raptor.filesystem == "unix": |
641 if self.raptor.filesystem == "unix": |
427 p = subprocess.Popen(command, bufsize=65535, |
642 p = subprocess.Popen([command], bufsize=65535, |
428 stdout=subprocess.PIPE, |
643 stdout=subprocess.PIPE, |
429 stderr=subprocess.STDOUT, |
644 stderr=subprocess.STDOUT, |
430 close_fds=True, env=makeenv, shell=True) |
645 close_fds=True, env=makeenv, shell=True) |
431 else: |
646 else: |
432 p = subprocess.Popen(command, bufsize=65535, |
647 p = subprocess.Popen(args = |
433 stdout=subprocess.PIPE, |
648 [raptor_data.ToolSet.shell, '-c', command], |
434 stderr=subprocess.STDOUT, |
649 bufsize=65535, |
435 universal_newlines=True, env=makeenv) |
650 stdout=subprocess.PIPE, |
|
651 stderr=subprocess.STDOUT, |
|
652 shell = False, |
|
653 universal_newlines=True, env=makeenv) |
436 stream = p.stdout |
654 stream = p.stdout |
437 |
655 |
438 |
656 inRecipe = False |
439 line = " " |
657 |
440 while line: |
658 if not self.copyLogFromAnnoFile: |
441 line = stream.readline() |
659 for l in XMLEscapeLog(stream): |
442 self.raptor.out.write(line) |
660 self.raptor.out.write(l) |
443 |
661 |
444 # should be done now |
662 returncode = p.wait() |
445 returncode = p.wait() |
663 else: |
446 |
664 returncode = p.wait() |
|
665 |
|
666 annofilename = self.annoFileName.replace("#MAKEFILE#", makefile) |
|
667 self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename) |
|
668 try: |
|
669 for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)): |
|
670 self.raptor.out.write(l) |
|
671 except Exception,e: |
|
672 self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e)) |
|
673 |
|
674 |
|
675 # Take all the stderr output that went into the .stderr file |
|
676 # and put it back into the log, but safely so it can't mess up |
|
677 # xml parsers. |
|
678 try: |
|
679 e = open(stderrfilename,"r") |
|
680 for line in e: |
|
681 self.raptor.out.write(escape(line)) |
|
682 e.close() |
|
683 except Exception,e: |
|
684 self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e)) |
|
685 # Report end-time of the build |
|
686 self.raptor.InfoEndTime(object_type = "makefile", |
|
687 task = "build", key = str(makefile)) |
447 |
688 |
448 if returncode != 0 and not self.raptor.keepGoing: |
689 if returncode != 0 and not self.raptor.keepGoing: |
449 self.Tidy() |
690 self.Tidy() |
450 return False |
691 return False |
451 |
692 |
452 except Exception,e: |
693 except Exception,e: |
453 self.raptor.Error("Exception '%s' during '%s'", str(e), command) |
694 self.raptor.Error("Exception '%s' during '%s'", str(e), command) |
454 self.Tidy() |
695 self.Tidy() |
|
696 # Still report end-time of the build |
|
697 self.raptor.InfoEndTime(object_type = "Building", task = "Makefile", |
|
698 key = str(makefile)) |
455 return False |
699 return False |
456 |
700 |
457 # run any shutdown script |
701 # run any shutdown script |
458 if self.shutdownCommand != None and self.shutdownCommand != "": |
702 if self.shutdownCommand != None and self.shutdownCommand != "": |
459 self.raptor.Info("Running %s", self.shutdownCommand) |
703 self.raptor.Info("Running %s", self.shutdownCommand) |