Module log2xml |
[hide private]
[frames] | no frames]

# Source Code for Module log2xml

      1  #============================================================================
      2  #Name        : log2xml.py
      3  #Part of     : Helium
      4
      5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
      6  #All rights reserved.
      7  #This component and the accompanying materials are made available
      8  #under the terms of the License "Eclipse Public License v1.0"
      9  #which accompanies this distribution, and is available
     10  #at the URL "http://www.eclipse.org/legal/epl-v10.html".
     11  #
     12  #Initial Contributors:
     13  #Nokia Corporation - initial contribution.
     14  #
     15  #Contributors:
     16  #
     17  #Description:
     18  #===============================================================================
     19
     20  """ Symbian log converter.
     21  """
     22  import xml.dom.minidom
     23  import sys
     24  import os
     25  import re
     26  import shutil
     27  import codecs
     28  import time
     29  import datetime
     30  from xml.sax import make_parser
     31  from xml.sax.handler import ContentHandler
     32  from xml.sax.saxutils import escape
     33
     34
     35  DEFAULT_CONFIGURATION = {"FATAL": [r"mingw_make.exe"],
     36                           "ERROR": [r'^(?:(?:\s*\d+\)\s*)|(?:\s*\*\*\*\s*))ERROR:',
     37                                     r"^MISSING:",
     38                                     r"Error:\s+",
     39                                     r"^Error:",
     40                                     r"'.+' is not recognized as an internal or external command",
     41                                     r"FLEXlm error:",
     42                                     r"(ABLD|BLDMAKE) ERROR:",
     43                                     r"FATAL ERROR\(S\):",
     44                                     r"fatal error U1077",
     45                                     r"warning U4010",
     46                                     r"^make(?:\[\d+\])?\: \*\*\*",
     47                                     r"^make(?:\[\d+\])?:\s+.*\s+not\s+remade",
     48                                     r"\"(.*)\", line (\d+): (Error: +(.\d+.*?):.*)$",
     49                                     r"error: ((Internal fault):.*)$",
     50                                     r"Exception: [A-Z0-9_]+",
     51                                     r"target .* given more than once in the same rule",
     52                                     r"^ERROR:",
     53                                     r"^ERROR EC\d+:",
     54                                     r"^ERROR\t",],
     55                           "CRITICAL": [r"[Ww]arning:?\s+(#111-D|#1166-D|#117-D|#128-D|#1293-D|#1441-D|#170-D|#174-D|#175-D|#185-D|#186-D|#223-D|#231-D|#257-D|#284-D|#368-D|#414-D|#430-D|#47-D|#514-D|#546-D|#68-D|#69-D|#830-D|#940-D|#836-D|A1495E|L6318W|C2874W|C4127|C4355|C4530|C4702|C4786|LNK4049)"],
     56                           "WARNING": [r'\): Missing file:',
     57                                        r'^(\d+\))?\s*WARNING:', r'^MAKEDEF WARNING:',
     58                                        r'line \d+: Warning:', r':\s+warning\s+\w+:',
     59                                        r"\\\\(.*?)\(\d+\)\s:\sWarning:\s\(\d+\)",
     60                                        r"^(BLDMAKE |MAKEDEF )?WARNING:",
     61                                        r"WARNING\(S\)",
     62                                        r"\(\d+\) : warning C",
     63                                        r"LINK : warning",
     64                                        r":\d+: warning:",
     65                                        r"\"(.*)\", line (\d+): (Warning: +(?!A1495E)(.\d+.*?):.*)$",
     66                                        r"Usage Warning:",
     67                                        r"mwld.exe:",
     68                                        r"^Command line warning",
     69                                        r"ERROR: bad relocation:",
     70                                        r"^(\d+) warning",
     71                                        r"EventType:\s+Error\s+Source:\s+SweepNT",
     72                                        r"^WARN\t",
     73                                        ],
     74                          "REMARK": [r"Command line warning D4025 : ",
     75                                     r"^REMARK: ",
     76                                     r"^EventType:\s+Error\s+Source:\s+GNU\s+Make",
     77                                     r":\d+: warning: cannot find matching deallocation function",
     78                                     r"((:\d+)*: note: )",
     79                                     ],
     80                          "INFO": [r"^INFO:"]
     81                          }
     82
     83
     84 -def find_priority(line, config):
     85      keys = config.keys()
     86      keys.reverse()
     87      for category in keys:
     88          for rule in config[category]:
     89              if rule.search(line) != None:
     90                  return category.lower()
     91      return "stdout"
     92
     93 -class Stack:
     94      """ Bottomless stack. If empty just pop a default element. """
     95
     96 -    def __init__(self, default):
     97          self.__default = default
     98          self.__stack = []
     99
    100 -    def pop(self):
    101          result = None
    102          try:
    103              result = self.__stack.pop()
    104          except IndexError, e:
    105              result = self.__default
    106          return result
    107
    108 -    def push(self, item):
    109          self.__stack.append(item)
    110
    111 -    def __len__(self):
    112          return len(self.__stack)
    113
    114 -def to_cdata(text):
    115      """ Cleanup string to match CDATA requiements.
    116          These are the only allowed characters: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF].
    117      """
    118      result = ""
    119      for c in list(text):
    120          v = ord(c)
    121          if v == 0x9 or v == 0xa or v == 0xd:
    122              result += c
    123          elif v>=0x20 and v <= 0xd7ff:
    124              result += c
    125          elif v>=0xe000 and v <= 0xfffd:
    126              result += c
    127          elif v>=0x10000 and v <= 0x10ffff:
    128              result += c
    129          else:
    130              result += " "
    131      return result
    132
    133 -class LogWriter(object):
    134      """ XML Log writer. """
    135
    136 -    def __init__(self, stream, filename):
    137          self.__stream = stream
    138          self.__stream.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
    139          self.__stream.write("<log filename=\"%s\">\n" % filename)
    140          self.__stream.write("\t<build>\n")
    141          self.__indent = "\t"
    142          self.__intask = 0
    143
    144 -    def close(self):
    145          # closing open tasks...
    146          while self.__intask > 0:
    147              self.close_task()
    148          self.__stream.write("\t</build>\n")
    149          self.__stream.write("</log>\n")
    150          self.__stream.close()
    151
    152 -    def open_task(self, name):
    153          self.__indent += "\t"
    154          self.__intask += 1
    155          self.__stream.write("%s<task name=\"%s\">\n" % (self.__indent, name))
    156
    157 -    def close_task(self):
    158          if self.__intask > 0:
    159              self.__intask -= 1
    160              self.__stream.write("%s</task>\n" % (self.__indent))
    161              self.__indent = self.__indent[:-1]
    162
    163 -    def message(self, priority, msg):
    164          try:
    165              acdata = to_cdata(msg.decode('utf-8', 'ignore'))
    166              self.__stream.write("%s<message priority=\"%s\"><![CDATA[%s]]></message>\n" % (self.__indent+"\t", priority, acdata))
    167          except UnicodeDecodeError, e:
    168              print e
    169
    170
    171
    172 -def convert(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
    173      """ Convert an input log into an XML log and write an outputfile. """
    174
    175      # Compiling the regexp
    176      built_config = {}
    177      for category in configuration.keys():
    178          built_config[category] = []
    179          for rule in configuration[category]:
    180              built_config[category].append(re.compile(rule))
    181
    182      # Generating the XML log
    183      log = open(inputfile, 'r')
    184      olog = codecs.open(outputfile, 'w+', 'utf-8', errors='ignore')
    185      xmllog = LogWriter(olog, inputfile)
    186
    187
    188      match_finnished = re.compile(r"^===\s+.+\s+finished")
    189      match_started = re.compile(r"^===\s+(.+)\s+started")
    190      match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
    191      match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
    192      #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started')
    193      #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished')
    194      match_ant_target_start = re.compile(r'^([^\s=\[\]]+):$')
    195      match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
    196      symbian = False
    197      ant_has_open_task = False
    198      # looping
    199      for line in log:
    200
    201          # matching Ant logging
    202          if not symbian and match_ant_target_end.match(line):
    203              xmllog.close_task()
    204              ant_has_open_task = False
    205              continue
    206          elif not symbian and match_ant_target_start.match(line):
    207              result = match_ant_target_start.match(line)
    208              if result != None:
    209                  if ant_has_open_task:
    210                      xmllog.close_task()
    211                      ant_has_open_task = False
    212                  xmllog.open_task(result.group(1))
    213                  ant_has_open_task = True
    214              continue
    215          # matching Symbian logging
    216          line = match_logger_component.sub(r'', line)
    217          line = line.strip()
    218          if line.startswith("++ Finished at"):
    219              xmllog.close_task()
    220          elif line.startswith("=== "):
    221              if match_finnished.match(line):
    222                  xmllog.close_task()
    223              else:
    224                  # This is a symbian log
    225                  symbian = True
    226                  result = match_component.match(line)
    227                  if result != None:
    228                      xmllog.open_task(result.group(2))
    229                  # === cenrep_s60_32 started
    230                  result = match_started.match(line)
    231                  if result != None:
    232                      xmllog.open_task(result.group(1))
    233          else:
    234              # Type?
    235              priority = find_priority(line, built_config)
    236              if (fulllogging or priority != 'stdout'):
    237                  xmllog.message(priority, line)
    238      # end file
    239      xmllog.close()
    240
    241 -def convert_old(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
    242      """ Convert an input log into an XML log and write an outputfile. """
    243
    244      # Compiling the regexp
    245      built_config = {}
    246      for category in configuration.keys():
    247          built_config[category] = []
    248          for rule in configuration[category]:
    249              built_config[category].append(re.compile(rule))
    250
    251      # Generating the XML log
    252      log = open (inputfile, 'r')
    253      doc = xml.dom.minidom.Document()
    254      root = doc.createElementNS("", "log")
    255      root.setAttributeNS("", "name", inputfile)
    256      doc.appendChild(root)
    257      build = doc.createElementNS("", "build")
    258      root.appendChild(build)
    259      # current group/task
    260      current = build
    261      # bottomless stask, if losing sync all message will be at top level.
    262      stack = Stack(build)
    263
    264      match_finnished = re.compile(r"^===\s+.+\s+finished")
    265      match_started = re.compile(r"===\s+(.+)\s+started")
    266      match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
    267      match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
    268      #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started')
    269      #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished')
    270      match_ant_target_start = re.compile(r'^([^\s=]+):$')
    271      match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
    272      # looping
    273      for line in log:
    274
    275          # matching Ant logging
    276          if match_ant_target_end.match(line):
    277              current = stack.pop()
    278              continue
    279          elif match_ant_target_start.match(line):
    280              result = match_ant_target_start.match(line)
    281              if result != None:
    282                  stack.push(current)
    283                  task = doc.createElementNS("", "task")
    284                  task.setAttributeNS("", "name", result.group(1))
    285                  current.appendChild(task)
    286                  current = task
    287              continue
    288          # matching Symbian logging
    289          line = match_logger_component.sub(r'', line)
    290          line = line.strip()
    291          if line.startswith("++ Finished at"):
    292              current = stack.pop()
    293          elif line.startswith("==="):
    294              if match_finnished.match(line):
    295                  current = stack.pop()
    296              else:
    297                  result = match_component.match(line)
    298                  if result != None:
    299                      stack.push(current)
    300                      task = doc.createElementNS("", "task")
    301                      task.setAttributeNS("", "name", result.group(2))
    302                      current.appendChild(task)
    303                      current = task
    304                  # === cenrep_s60_32 started
    305                  result = match_started.match(line)
    306                  if result != None:
    307                      task = doc.createElementNS("", "task")
    308                      task.setAttributeNS("", "name", result.group(1))
    309                      stack.push(current)
    310                      current.appendChild(task)
    311                      current = task
    312          else:
    313              msg = doc.createElementNS("", "message")
    314              # Type?
    315              priority = find_priority(line, built_config)
    316              if (fulllogging or priority != 'stdout'):
    317                  msg.setAttributeNS("", "priority", priority)
    318                  msg.appendChild(doc.createCDATASection(to_cdata(line.decode("utf-8"))))
    319                  current.appendChild(msg)
    320
    321      file_object = codecs.open(outputfile, 'w', "utf_8")
    322      file_object.write(doc.toprettyxml())
    323      file_object.close()
    324
    325 -class ContentWriter(ContentHandler):
    326      """ SAX Content writer. Parse and write an XML file. """
    327 -    def __init__(self, os, indent=""):
    328          self.os = os
    329          self.indent = indent
    330          self.__content = u""
    331
    332 -    def startElement(self, name, attrs):
    333          self.os.write(self.indent + "<" + name)
    334          if attrs.getLength() > 0:
    335              self.os.write(" ")
    336          self.os.write(" ".join(map(lambda x: "%s=\"%s\"" % (x, attrs.getValue(x)), attrs.getNames())))
    337          self.os.write(">\n")
    338          self.indent += "\t"
    339          self.__content = ""
    340
    341 -    def endElement(self, name):
    342          if len(self.__content) > 0:
    343              self.os.write(self.indent + self.__content + "\n")
    344          self.indent = self.indent[:-1]
    345          self.os.write("%s</%s>\n" % (self.indent, name))
    346          self.__content = ""
    347
    348 -    def characters(self, content):
    349          self.__content += unicode(escape(content.strip()))
    350
    351 -class AppendSummary(ContentWriter):
    352      """ SAX content handler to add an XML log to the summary. """
    353 -    def __init__(self, output, xmllog):
    354          ContentWriter.__init__(self, output)
    355          self.xmllog = xmllog
    356
    357 -    def startDocument(self):
    358          self.os.write('<?xml version="1.0" encoding="utf-8"?>\n')
    359
    360 -    def startElement(self, name, attrs):
    361          ContentWriter.startElement(self, name, attrs)
    362          if name == "logSummary":
    363              parser = make_parser()
    364              parser.setContentHandler(ContentWriter(self.os, self.indent))
    365              parser.parse(open(self.xmllog, 'r'))
    366
    367
    368 -def append_summary(summary, xmllog, maxmb=80):
    369      """ Append content to the summary xml file. """
    370      if os.path.getsize(summary) + os.path.getsize(xmllog) > (maxmb*1024*1024):
    371          print 'Error: ' + summary + ' larger than ' + str(maxmb) + 'MB, not appending'
    372          return
    373
    374      outfile = codecs.open(summary + ".tmp", 'w', "utf8")
    375      parser = make_parser()
    376      parser.setContentHandler(AppendSummary(outfile, xmllog))
    377
    378      input = open(summary, 'r')
    379      parser.parse(input)
    380      input.close()
    381      outfile.close()
    382      # Updating the summary file.
    383      os.unlink(summary)
    384      os.rename(summary + ".tmp", summary)
    385
    386
    387 -def symbian_log_header(output, config, command, dir):
    388      output.log("===-------------------------------------------------")
    389      output.log("=== %s" % config)
    390      output.log("===-------------------------------------------------")
    391      output.log("=== %s started %s" % (config, datetime.datetime.now().ctime()))
    392      output.log("=== %s == %s" % (config, dir))
    393      output.log("-- %s" % command)
    394      output.log("++ Started at %s" % datetime.datetime.now().ctime())
    395      output.log("+++ HiRes Start %f" % time.time())
    396      output.log("Chdir %s" % dir)
    397
    398
    402
    403
    404  if __name__ == "__main__":
    405      convert(sys.argv[1], "%s.xml" % sys.argv[1], fulllogging=False)
    406      """ An empty summary:
    407          <?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>
    408      """
    409      #s = open(r"z:\summary.xml", "w")
    410      #s.write("""<?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>""")
    411      #s.close()
    412      #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1.mc_5132_2_build.log2.xml')
    413      #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1_BOM.xml')
    414