#============================================================================ 
#Name        : log2xml.py 
#Part of     : Helium 

#Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
#All rights reserved.
#This component and the accompanying materials are made available
#under the terms of the License "Eclipse Public License v1.0"
#which accompanies this distribution, and is available
#at the URL "http://www.eclipse.org/legal/epl-v10.html".
#
#Initial Contributors:
#Nokia Corporation - initial contribution.
#
#Contributors:
#
#Description:
#===============================================================================

""" Symbian log converter."""

import xml.dom.minidom
import sys
import os
import re
import codecs
import time
import datetime
from xml.sax import make_parser 
from xml.sax.handler import ContentHandler 
from xml.sax.saxutils import escape


DEFAULT_CONFIGURATION = {"FATAL": [r"mingw_make.exe"],
                         "ERROR": [r'^(?:(?:\s*\d+\)\s*)|(?:\s*\*\*\*\s*))ERROR:',
                                   r"^MISSING:",
                                   r"Error:\s+",
                                   r"^Error:",
                                   r"'.+' is not recognized as an internal or external command",
                                   r"FLEXlm error:",
                                   r"(ABLD|BLDMAKE) ERROR:",
                                   r"FATAL ERROR\(S\):",
                                   r"fatal error U1077",
                                   r"warning U4010",
                                   r"^make(?:\[\d+\])?\: \*\*\*",
                                   r"^make(?:\[\d+\])?:\s+.*\s+not\s+remade",
                                   r"\"(.*)\", line (\d+): (Error: +(.\d+.*?):.*)$",
                                   r"error: ((Internal fault):.*)$",
                                   r"Exception: [A-Z0-9_]+",
                                   r"target .* given more than once in the same rule",
                                   r"^ERROR:",
                                   r"^ERROR EC\d+:",
                                   r"^ERROR\t",
                                   r"syntax error at line",],
                         "CRITICAL": [r"[Ww]arning:?\s+(#111-D|#1166-D|#117-D|#128-D|#1293-D|#1441-D|#170-D|#174-D|#175-D|#185-D|#186-D|#223-D|#231-D|#257-D|#284-D|#368-D|#414-D|#430-D|#47-D|#514-D|#546-D|#68-D|#69-D|#830-D|#940-D|#836-D|A1495E|L6318W|C2874W|C4127|C4355|C4530|C4702|C4786|LNK4049)"],
                         "WARNING": [r'\): Missing file:',
                                      r'^(\d+\))?\s*WARNING:', r'^MAKEDEF WARNING:',
                                      r'line \d+: Warning:', r':\s+warning\s+\w+:',
                                      r"\\\\(.*?)\(\d+\)\s:\sWarning:\s\(\d+\)",
                                      r"^(BLDMAKE |MAKEDEF )?WARNING:",
                                      r"WARNING\(S\)",
                                      r"\(\d+\) : warning C",
                                      r"LINK : warning",
                                      r":\d+: warning:",
                                      r"\"(.*)\", line (\d+): (Warning: +(?!A1495E)(.\d+.*?):.*)$",
                                      r"Usage Warning:",
                                      r"mwld.exe:",
                                      r"^Command line warning",
                                      r"ERROR: bad relocation:",
                                      r"^(\d+) warning",
                                      r"EventType:\s+Error\s+Source:\s+SweepNT",
                                      r"^WARN\t",
                                      ],
                        "REMARK": [r"Command line warning D4025 : ",
                                   r"^REMARK: ",
                                   r"^EventType:\s+Error\s+Source:\s+GNU\s+Make",
                                   r":\d+: warning: cannot find matching deallocation function",
                                   r"((:\d+)*: note: )",
                                   ],
                        "INFO": [r"^INFO:"]
                        }


def find_priority(line, config):
    """find priority"""
    keys = config.keys()
    keys.reverse()    
    for category in keys:
        for rule in config[category]:
            if rule.search(line) != None:
                return category.lower()
    return "stdout"

class Stack:
    """ Bottomless stack. If empty just pop a default element. """
    
    def __init__(self, default):
        self.__default = default
        self.__stack = []
  
    def pop(self):
        """pop off the stack"""
        result = None
        try:
            result = self.__stack.pop()
        except IndexError:
            result = self.__default
        return result
  
    def push(self, item):
        """push on the stack"""
        self.__stack.append(item)
  
    def __len__(self):
        return len(self.__stack)

def to_cdata(text):
    """ Cleanup string to match CDATA requiements.
        These are the only allowed characters: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF].
    """
    result = ""
    for char in list(text):
        v_char = ord(char)
        if v_char == 0x9 or v_char == 0xa or v_char == 0xd:
            result += char
        elif v_char >= 0x20 and v_char <= 0xd7ff:
            result += char
        elif v_char >= 0xe000 and v_char <= 0xfffd:
            result += char
        elif v_char >= 0x10000 and v_char <= 0x10ffff:
            result += char
        else:
            result += " " 
    return result

class LogWriter(object):
    """ XML Log writer. """
    
    def __init__(self, stream, filename):
        self.__stream = stream
        self.__stream.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
        self.__stream.write("<log filename=\"%s\">\n" % filename)
        self.__stream.write("\t<build>\n")
        self.__indent = "\t"
        self.__intask = 0

    def close(self):
        """ closing open tasks"""
        while self.__intask > 0:
            self.close_task()
        self.__stream.write("\t</build>\n")
        self.__stream.write("</log>\n")
        self.__stream.close()

    def open_task(self, name):
        """open taks"""
        self.__indent += "\t"
        self.__intask += 1
        self.__stream.write("%s<task name=\"%s\">\n" % (self.__indent, name))

    def close_task(self):
        """close task"""
        if self.__intask > 0:
            self.__intask -= 1
            self.__stream.write("%s</task>\n" % (self.__indent))
            self.__indent = self.__indent[:-1]
    
    def message(self, priority, msg):
        """message"""
        try:
            acdata = to_cdata(msg.decode('utf-8', 'ignore'))
            self.__stream.write("%s<message priority=\"%s\"><![CDATA[%s]]></message>\n" % (self.__indent+"\t", priority, acdata))
        except UnicodeDecodeError, exc:
            print exc


def convert(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
    """ Convert an input log into an XML log and write an outputfile. """
    
    # Compiling the regexp  
    built_config = {}
    for category in configuration.keys():
        built_config[category] = []
        for rule in configuration[category]:
            built_config[category].append(re.compile(rule))
    
    # Generating the XML log
    log = open(inputfile, 'r')
    olog = codecs.open(outputfile, 'w+', 'utf-8', errors='ignore')
    xmllog = LogWriter(olog, inputfile)

    
    match_finnished = re.compile(r"^===\s+.+\s+finished") 
    match_started = re.compile(r"^===\s+(.+)\s+started") 
    match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
    match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
    #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started')
    #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished')
    match_ant_target_start = re.compile(r'^([^\s=\[\]]+):$')
    match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
    symbian = False
    ant_has_open_task = False
    # looping
    for line in log:
                
        # matching Ant logging
        if not symbian and match_ant_target_end.match(line):
            xmllog.close_task()
            ant_has_open_task = False
            continue
        elif not symbian and match_ant_target_start.match(line):
            result = match_ant_target_start.match(line)
            if result != None:
                if ant_has_open_task:
                    xmllog.close_task()
                    ant_has_open_task = False
                xmllog.open_task(result.group(1))
                ant_has_open_task = True
            continue
        # matching Symbian logging
        line = match_logger_component.sub(r'', line)
        line = line.strip()
        if line.startswith("++ Finished at"):
            xmllog.close_task()
        elif line.startswith("=== "):
            if match_finnished.match(line):
                xmllog.close_task()
            else:
                # This is a symbian log
                symbian = True
                result = match_component.match(line)
                if result != None:
                    xmllog.open_task(result.group(2))
                # === cenrep_s60_32 started
                result = match_started.match(line)
                if result != None:
                    xmllog.open_task(result.group(1))
        else:
            # Type?
            priority = find_priority(line, built_config)
            if (fulllogging or priority != 'stdout'):
                xmllog.message(priority, line)
    # end file
    xmllog.close()
    
def convert_old(inputfile, outputfile, fulllogging=True, configuration=DEFAULT_CONFIGURATION):
    """ Convert an input log into an XML log and write an outputfile. """
    
    # Compiling the regexp  
    built_config = {}
    for category in configuration.keys():
        built_config[category] = []
        for rule in configuration[category]:
            built_config[category].append(re.compile(rule))
    
    # Generating the XML log
    log = open (inputfile, 'r')
    doc = xml.dom.minidom.Document()
    root = doc.createElementNS("", "log")
    root.setAttributeNS("", "name", inputfile)
    doc.appendChild(root)
    build = doc.createElementNS("", "build")
    root.appendChild(build)
    # current group/task
    current = build
    # bottomless stask, if losing sync all message will be at top level.
    stack = Stack(build)
    
    match_finnished = re.compile(r"^===\s+.+\s+finished") 
    match_started = re.compile(r"===\s+(.+)\s+started") 
    match_component = re.compile(r"^===\s+(.+?)\s+==\s+(.+)")
    match_logger_component = re.compile(r'^\s*\[.+?\]\s*')
    #match_ant_target_start = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+started')
    #match_ant_target_end = re.compile(r'.*INFO\s+-\s+Target\s+####\s+(.+)\s+####\s+has\s+finnished')
    match_ant_target_start = re.compile(r'^([^\s=]+):$')
    match_ant_target_end = re.compile(r'^([^\s=]+):\s+duration')
    # looping
    for line in log:
                
        # matching Ant logging
        if match_ant_target_end.match(line):
            current = stack.pop()
            continue
        elif match_ant_target_start.match(line):
            result = match_ant_target_start.match(line)
            if result != None:
                stack.push(current)
                task = doc.createElementNS("", "task")
                task.setAttributeNS("", "name", result.group(1))
                current.appendChild(task)
                current = task
            continue
        # matching Symbian logging
        line = match_logger_component.sub(r'', line)
        line = line.strip()
        if line.startswith("++ Finished at"):
            current = stack.pop()
        elif line.startswith("==="):
            if match_finnished.match(line):
                current = stack.pop()
            else:
                result = match_component.match(line)
                if result != None:
                    stack.push(current)
                    task = doc.createElementNS("", "task")
                    task.setAttributeNS("", "name", result.group(2))
                    current.appendChild(task)
                    current = task
                # === cenrep_s60_32 started
                result = match_started.match(line)
                if result != None:
                    task = doc.createElementNS("", "task")
                    task.setAttributeNS("", "name", result.group(1))
                    stack.push(current)
                    current.appendChild(task)
                    current = task
        else:
            msg = doc.createElementNS("", "message")
            # Type?
            priority = find_priority(line, built_config)
            if (fulllogging or priority != 'stdout'):
                msg.setAttributeNS("", "priority", priority)
                msg.appendChild(doc.createCDATASection(to_cdata(line.decode("utf-8"))))
                current.appendChild(msg)
        
    file_object = codecs.open(outputfile, 'w', "utf_8")
    file_object.write(doc.toprettyxml())
    file_object.close()

class ContentWriter(ContentHandler):
    """ SAX Content writer. Parse and write an XML file. """
    def __init__(self, op_sys, indent=""):
        ContentHandler.__init__(self)
        self.os = op_sys
        self.indent = indent
        self.__content = u""
    
    def startElement(self, name, attrs):
        """start Element"""
        self.os.write(self.indent + "<" + name)
        if attrs.getLength() > 0:
            self.os.write(" ")
        self.os.write(" ".join(["%s=\"%s\"" % (x, attrs.getValue(x)) for x in attrs.getNames()]))
        self.os.write(">\n")
        self.indent += "\t"
        self.__content = ""
    
    def endElement(self, name):
        """end element"""
        if len(self.__content) > 0:
            self.os.write(self.indent + self.__content + "\n")
        self.indent = self.indent[:-1]
        self.os.write("%s</%s>\n" % (self.indent, name))
        self.__content = ""
        
    def characters(self, content):
        """characters"""
        self.__content += unicode(escape(content.strip()))

class AppendSummary(ContentWriter):
    """ SAX content handler to add an XML log to the summary. """
    def __init__(self, output, xmllog):
        ContentWriter.__init__(self, output)
        self.xmllog = xmllog

    def startDocument(self):
        """start document"""
        self.os.write('<?xml version="1.0" encoding="utf-8"?>\n')
    
    def startElement(self, name, attrs):
        """start element"""
        ContentWriter.startElement(self, name, attrs)
        if name == "logSummary":
            parser = make_parser()
            parser.setContentHandler(ContentWriter(self.os, self.indent))
            parser.parse(open(self.xmllog, 'r'))
    

def append_summary(summary, xmllog, maxmb=80):
    """ Append content to the summary xml file. """
    if os.path.getsize(summary) + os.path.getsize(xmllog) > (maxmb*1024*1024):
        print 'Error: ' + summary + ' larger than ' + str(maxmb) + 'MB, not appending'
        return
    
    outfile = codecs.open(summary + ".tmp", 'w', "utf8")
    parser = make_parser()
    parser.setContentHandler(AppendSummary(outfile, xmllog))
    
    input_ = open(summary, 'r')
    parser.parse(input_)
    input_.close()
    outfile.close()
    # Updating the summary file.
    os.unlink(summary)
    os.rename(summary + ".tmp", summary)
    

def symbian_log_header(output, config, command, dir_):
    """symbian log header"""
    output.log("===-------------------------------------------------")
    output.log("=== %s" % config)
    output.log("===-------------------------------------------------")
    output.log("=== %s started %s" % (config, datetime.datetime.now().ctime()))
    output.log("=== %s == %s" % (config, dir_))
    output.log("-- %s" % command)
    output.log("++ Started at %s" % datetime.datetime.now().ctime())
    output.log("+++ HiRes Start %f" % time.time())
    output.log("Chdir %s" % dir_)


def symbian_log_footer(output):
    """symbian log footer"""
    output.log("+++ HiRes End %f" % time.time())
    output.log("++ Finished at %s" % datetime.datetime.now().ctime())


if __name__ == "__main__":
    convert(sys.argv[1], "%s.xml" % sys.argv[1], fulllogging=False)
    # An empty summary:
    #   <?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>
    #
    
    #s = open(r"z:\summary.xml", "w")
    #s.write("""<?xml version=\"1.0\" encoding=\"UTF-8\"?><logSummary/>""")
    #s.close()
    #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1.mc_5132_2_build.log2.xml')
    #append_summary(r'Z:\summary.xml', r'Z:\output\logs\test_0.0.1_BOM.xml')