buildframework/helium/tools/common/python/scripts/sbsscanlogmetadata.py
author wbernard
Tue, 27 Apr 2010 08:33:08 +0300
changeset 587 85df38eb4012
parent 217 0f5e3a7fb6af
permissions -rw-r--r--
helium_9.0-a7879c935424
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     1
#============================================================================ 
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
     2
#Name        : sbsscanlogmetadata.py
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     3
#Part of     : Helium 
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     4
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     5
#Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     6
#All rights reserved.
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     7
#This component and the accompanying materials are made available
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     8
#under the terms of the License "Eclipse Public License v1.0"
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
     9
#which accompanies this distribution, and is available
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    10
#at the URL "http://www.eclipse.org/legal/epl-v10.html".
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    11
#
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    12
#Initial Contributors:
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    13
#Nokia Corporation - initial contribution.
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    14
#
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    15
#Contributors:
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    16
#
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    17
#Description:
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    18
#===============================================================================
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    19
"""parses the raptor metadata logs and separates the info out into HTML and XML
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    20
   logs for writing to diamonds and other logs"""
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    21
    
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    22
import os
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    23
import sys
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    24
import re
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    25
import datetime
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    26
from optparse import OptionParser
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    27
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    28
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    29
IGNORE_TEXT_REG_EX = "warning: no newline at end of file"
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    30
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    31
STREAM_REGEX =  {   "clean" : [r'<clean', r'</clean'],
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    32
                    "what" : [r'<whatlog', r'</whatlog'],
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    33
                    "warning" : [r'<warning', r'</warning']
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    34
                }
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    35
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    36
class SBSScanlogMetadata(object):
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    37
    """parses the raptor metadata logs and separates the info out into HTML and 
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    38
    XML logs for writing to diamonds and other logs"""
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    39
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    40
    def initializeLogPath(self):
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    41
        """retrieve the log file names from the environment variables """
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    42
        index = self.logFileName.rfind(".")
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    43
        if index < 0:
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    44
            index = len(self.logFileName)
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    45
        for stream in STREAM_REGEX.keys():
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    46
            self.stream_path[stream] = self.logFileName[:index] + "." + stream + \
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    47
                        self.logFileName[index:]
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    48
        if os.environ.has_key('SBS_CLEAN_LOG_FILE'):
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    49
            self.stream_path['clean'] = os.environ['SBS_CLEAN_LOG_FILE']
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    50
        if os.environ.has_key('SBS_WHAT_LOG_FILE'):
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    51
            self.stream_path['what'] = os.environ['SBS_WHAT_LOG_FILE']
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    52
            
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    53
    def initialize(self, logFile):
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    54
        """Initialize helium log filter"""
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    55
        self.ignoreTextCompileObject = re.compile(IGNORE_TEXT_REG_EX);
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    56
        self.logFileName = str(logFile)
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    57
        self.streamStatus = {}
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    58
        self.streams = {}
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    59
        self.stream_path = {}
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    60
        self.start_time = datetime.datetime.now()
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    61
        self.loggerout = open(self.logFileName,"w")
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    62
        self.compiled_stream_object = {}
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    63
        print "logName: %s\n" % self.logFileName
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    64
        self.initializeLogPath()
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    65
        for stream in STREAM_REGEX.keys():
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    66
            self.compiled_stream_object[stream] = []
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    67
            self.streams[stream] = open(self.stream_path[stream], "w")
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    68
            self.streamStatus[stream] = False
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    69
            self.streams[stream].write(\
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    70
"""<?xml version="1.0" encoding="ISO-8859-1" ?>
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
    71
<buildlog sbs_version="" xmlns="http://symbian.com/xml/build/log" xmlns:progress="http://symbian.com/xml/build/log/progress" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symbian.com/xml/build/log http://symbian.com/xml/build/log/1_0.xsd">""")
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    72
            for  searchString in STREAM_REGEX[stream]:
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    73
                self.compiled_stream_object[stream].append(re.compile(searchString))
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    74
        return True
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    75
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    76
    def open(self, logFile):
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
    77
        """ open the log file"""
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    78
        self.logFileName = str(logFile)
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    79
        return self.initialize(logFile)
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    80
        
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    81
        
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    82
    def write(self, text):
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    83
        """ callback function which is to process the logs"""
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    84
        stream_list = STREAM_REGEX.keys()
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    85
        for textLine in text.splitlines():
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    86
            textLine = textLine + '\n'
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    87
            if textLine.startswith("<?xml ") or textLine.startswith("<buildlog ") \
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    88
                or textLine.startswith("</buildlog"):
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    89
                self.loggerout.write(textLine)
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    90
                continue
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    91
            if(self.ignoreTextCompileObject.search(textLine)):
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
    92
                continue
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    93
            for stream in stream_list:
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    94
                if( (not self.streamStatus[stream]) and self.compiled_stream_object[stream][0].search(textLine)!= None):
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    95
                    self.streamStatus[stream] = True
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    96
                if (self.streamStatus[stream] and self.compiled_stream_object[stream][1].search(textLine)!= None):
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    97
                    self.streams[stream].write(textLine)
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    98
                    self.streamStatus[stream] = False
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
    99
                    break
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   100
    
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   101
                if(self.streamStatus[stream]):
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
   102
                    if textLine.startswith("<?xml ") or textLine.startswith("<buildlog ") \
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
   103
                        or textLine.startswith("</buildlog"):
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
   104
                        continue
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   105
                    self.streams[stream].write(textLine)
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   106
                    break
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   107
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   108
            self.loggerout.write(textLine)
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   109
        return True
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   110
        
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   111
    def summary(self):
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   112
        """Write Summary"""
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   113
        sys.stdout.write("sbs: build log in %s\n" % str(self.logFileName))
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   114
        return False
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   115
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   116
    def close(self):
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   117
        """Close the log file"""
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   118
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   119
        try:
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   120
            self.loggerout.close()
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   121
            for stream in self.streams.keys():
217
0f5e3a7fb6af helium_8.0-r15308
wbernard
parents: 179
diff changeset
   122
                self.streams[stream].write('</buildlog>')
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   123
                self.streams[stream].close()
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   124
            return True
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   125
        except:
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   126
            self.loggerout = None
179
d8ac696cc51f helium_7.0-r14027
wbernard
parents: 1
diff changeset
   127
            self.streams = None
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   128
        return False
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   129
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   130
if __name__ == "__main__":
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   131
    # standalone app
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   132
    _cli = OptionParser(usage="%prog [options]")
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   133
    _cli.add_option("--log", help="Raptor log file")
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   134
    _cli.add_option("--output", help="Raptor log file")
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   135
                   
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   136
    _opts, _ = _cli.parse_args()
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   137
    if not _opts.log:
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   138
        _cli.print_help()
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   139
        sys.exit(-1)
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   140
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   141
    _sbsFilter = SBSScanlogMetadata()
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   142
    _sbsFilter.open(_opts.output)
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   143
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   144
    _logFile = open(_opts.log, 'r')
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   145
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   146
    for line in _logFile:
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   147
        _sbsFilter.write(line)
1
be27ed110b50 Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff changeset
   148
587
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   149
    _sbsFilter.summary()
85df38eb4012 helium_9.0-a7879c935424
wbernard
parents: 217
diff changeset
   150
    _sbsFilter.close()