29 import time |
29 import time |
30 from raptor_makefile import * |
30 from raptor_makefile import * |
31 import traceback |
31 import traceback |
32 import sys |
32 import sys |
33 from xml.sax.saxutils import escape |
33 from xml.sax.saxutils import escape |
|
34 from xml.sax.saxutils import unescape |
34 |
35 |
35 |
36 |
36 class BadMakeEngineException(Exception): |
37 class BadMakeEngineException(Exception): |
37 pass |
38 pass |
|
39 |
|
40 def XMLEscapeLog(stream): |
|
41 inRecipe = False |
|
42 |
|
43 for line in stream: |
|
44 if line.startswith("<recipe"): |
|
45 inRecipe = True |
|
46 elif line.startswith("</recipe"): |
|
47 inRecipe = False |
|
48 |
|
49 # unless we are inside a "recipe", any line not starting |
|
50 # with "<" is free text that must be escaped. |
|
51 if inRecipe or line.startswith("<"): |
|
52 yield line |
|
53 else: |
|
54 yield escape(line) |
|
55 |
|
56 def AnnoFileParseOutput(annofile): |
|
57 af = open(annofile, "r") |
|
58 |
|
59 inOutput = False |
|
60 inParseJob = False |
|
61 for line in af: |
|
62 line = line.rstrip("\n\r") |
|
63 |
|
64 if not inOutput: |
|
65 if line.startswith("<output>"): |
|
66 inOutput = True |
|
67 yield unescape(line[8:])+'\n' |
|
68 # This is make output so don't unescape it. |
|
69 elif line.startswith('<output src="prog">'): |
|
70 line = line[19:] |
|
71 inOutput = True |
|
72 yield unescape(line)+'\n' |
|
73 else: |
|
74 end_output = line.find("</output>") |
|
75 |
|
76 if end_output != -1: |
|
77 line = line[:end_output] |
|
78 inOutput = False |
|
79 |
|
80 yield unescape(line)+'\n' |
|
81 |
|
82 af.close() |
|
83 |
|
84 |
38 |
85 |
39 # raptor_make module classes |
86 # raptor_make module classes |
40 |
87 |
41 class MakeEngine(object): |
88 class MakeEngine(object): |
42 |
89 |
79 # options |
126 # options |
80 self.makefileOption = evaluator.Get("makefile") |
127 self.makefileOption = evaluator.Get("makefile") |
81 self.keepGoingOption = evaluator.Get("keep_going") |
128 self.keepGoingOption = evaluator.Get("keep_going") |
82 self.jobsOption = evaluator.Get("jobs") |
129 self.jobsOption = evaluator.Get("jobs") |
83 self.defaultMakeOptions = evaluator.Get("defaultoptions") |
130 self.defaultMakeOptions = evaluator.Get("defaultoptions") |
|
131 |
|
132 # Logging |
|
133 # copylogfromannofile means, for emake, that we should ignore |
|
134 # emake's console output and instead extract output from its annotation |
|
135 # file. This is a workaround for a problem where some emake |
|
136 # console output is lost. The annotation file has a copy of this |
|
137 # output in the "parse" job and it turns out to be uncorrupted. |
|
138 self.copyLogFromAnnoFile = (evaluator.Get("copylogfromannofile") == "true") |
|
139 self.annoFileName = None |
|
140 |
|
141 if self.copyLogFromAnnoFile: |
|
142 for o in self.raptor.makeOptions: |
|
143 if o.startswith("--emake-annofile="): |
|
144 self.annoFileName = o[17:] |
|
145 self.raptor.Info("annofile: " + o) |
|
146 |
|
147 if not self.annoFileName: |
|
148 self.raptor.Info("Cannot copy log from annotation file as no annotation filename was specified via the option --mo=--emake-annofile=<filename>") |
|
149 self.copyLogFromAnnoFile = False |
84 |
150 |
85 # buffering |
151 # buffering |
86 self.scrambled = (evaluator.Get("scrambled") == "true") |
152 self.scrambled = (evaluator.Get("scrambled") == "true") |
87 |
153 |
88 # check tool versions |
154 # check tool versions |
477 |
543 |
478 # Send stderr to a file so that it can't mess up the log (e.g. |
544 # Send stderr to a file so that it can't mess up the log (e.g. |
479 # clock skew messages from some build engines scatter their |
545 # clock skew messages from some build engines scatter their |
480 # output across our xml. |
546 # output across our xml. |
481 stderrfilename = makefile+'.stderr' |
547 stderrfilename = makefile+'.stderr' |
|
548 stdoutfilename = makefile+'.stdout' |
482 command += " 2>'%s' " % stderrfilename |
549 command += " 2>'%s' " % stderrfilename |
|
550 |
|
551 # Keep a copy of the stdout too in the case of using the |
|
552 # annofile - so that we can trap the problem that |
|
553 # makes the copy-log-from-annofile workaround necessary |
|
554 # and perhaps determine when we can remove it. |
|
555 if self.copyLogFromAnnoFile: |
|
556 command += " >'%s' " % stdoutfilename |
483 |
557 |
484 # Substitute the makefile name for any occurrence of #MAKEFILE# |
558 # Substitute the makefile name for any occurrence of #MAKEFILE# |
485 command = command.replace("#MAKEFILE#", str(makefile)) |
559 command = command.replace("#MAKEFILE#", str(makefile)) |
486 |
560 |
487 self.raptor.Info("Executing '%s'", command) |
561 self.raptor.Info("Executing '%s'", command) |
516 shell = False, |
590 shell = False, |
517 universal_newlines=True, env=makeenv) |
591 universal_newlines=True, env=makeenv) |
518 stream = p.stdout |
592 stream = p.stdout |
519 |
593 |
520 inRecipe = False |
594 inRecipe = False |
521 line = " " |
595 |
522 while line: |
596 if not self.copyLogFromAnnoFile: |
523 line = stream.readline() |
597 for l in XMLEscapeLog(stream): |
524 |
598 self.raptor.out.write(l) |
525 if line.startswith("<recipe"): |
599 |
526 inRecipe = True |
600 returncode = p.wait() |
527 elif line.startswith("</recipe"): |
601 else: |
528 inRecipe = False |
602 returncode = p.wait() |
529 |
603 |
530 # unless we are inside a "recipe", any line not starting |
604 annofilename = self.annoFileName.replace("#MAKEFILE#", makefile) |
531 # with "<" is free text that must be escaped. |
605 self.raptor.Info("copylogfromannofile: Copying log from annotation file %s to work around a potential problem with the console output", annofilename) |
532 if inRecipe or line.startswith("<"): |
606 try: |
533 self.raptor.out.write(line) |
607 for l in XMLEscapeLog(AnnoFileParseOutput(annofilename)): |
534 else: |
608 self.raptor.out.write(l) |
535 self.raptor.out.write(escape(line)) |
609 except Exception,e: |
536 |
610 self.raptor.Error("Couldn't complete stdout output from annofile %s for %s - '%s'", annofilename, command, str(e)) |
537 # should be done now |
611 |
538 returncode = p.wait() |
|
539 |
|
540 # Report end-time of the build |
|
541 self.raptor.InfoEndTime(object_type = "makefile", |
|
542 task = "build", key = str(makefile)) |
|
543 |
612 |
544 # Take all the stderr output that went into the .stderr file |
613 # Take all the stderr output that went into the .stderr file |
545 # and put it back into the log, but safely so it can't mess up |
614 # and put it back into the log, but safely so it can't mess up |
546 # xml parsers. |
615 # xml parsers. |
547 try: |
616 try: |
549 for line in e: |
618 for line in e: |
550 self.raptor.out.write(escape(line)) |
619 self.raptor.out.write(escape(line)) |
551 e.close() |
620 e.close() |
552 except Exception,e: |
621 except Exception,e: |
553 self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e)) |
622 self.raptor.Error("Couldn't complete stderr output for %s - '%s'", command, str(e)) |
|
623 # Report end-time of the build |
|
624 self.raptor.InfoEndTime(object_type = "makefile", |
|
625 task = "build", key = str(makefile)) |
554 |
626 |
555 if returncode != 0 and not self.raptor.keepGoing: |
627 if returncode != 0 and not self.raptor.keepGoing: |
556 self.Tidy() |
628 self.Tidy() |
557 return False |
629 return False |
558 |
630 |
559 except Exception,e: |
631 except Exception,e: |
560 self.raptor.Error("Exception '%s' during '%s'", str(e), command) |
632 self.raptor.Error("Exception '%s' during '%s'", str(e), command) |
561 self.Tidy() |
633 self.Tidy() |
562 # Still report end-time of the build |
634 # Still report end-time of the build |
563 self.raptor.InfoEnd(object_type = "Building", task = "Makefile", |
635 self.raptor.InfoEndTime(object_type = "Building", task = "Makefile", |
564 key = str(makefile)) |
636 key = str(makefile)) |
565 return False |
637 return False |
566 |
638 |
567 # run any shutdown script |
639 # run any shutdown script |
568 if self.shutdownCommand != None and self.shutdownCommand != "": |
640 if self.shutdownCommand != None and self.shutdownCommand != "": |