author | Daniel Jacobs <daniel.jacobs@nokia.com> |
Thu, 04 Mar 2010 13:55:58 +0000 | |
branch | wip |
changeset 346 | d677c6f132ae |
permissions | -rw-r--r-- |
346
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
1 |
# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
2 |
# All rights reserved. |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
3 |
# This component and the accompanying materials are made available |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
4 |
# under the terms of the License "Eclipse Public License v1.0" |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
5 |
# which accompanies this distribution, and is available |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
6 |
# at the URL "http://www.eclipse.org/legal/epl-v10.html". |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
7 |
# |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
8 |
# Initial Contributors: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
9 |
# Nokia Corporation - initial contribution. |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
10 |
# |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
11 |
# Contributors: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
12 |
# |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
13 |
# Description: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
14 |
# Compress the full Raptor log file using the BZip2 algorithm, maximum compression. |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
15 |
# |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
16 |
# |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
17 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
18 |
import os |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
19 |
import sys |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
20 |
import raptor |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
21 |
import filter_interface |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
22 |
import bz2 |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
23 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
24 |
class StringListCompressor(object): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
25 |
def __init__(self, complevel=5, filename="file.log.bz2"): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
26 |
self.compressor = bz2.BZ2Compressor(complevel) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
27 |
self.stringlist = [] |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
28 |
self.outputopenedok = False |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
29 |
self.filename = filename |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
30 |
try: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
31 |
self.fh = open(self.filename, "wb") |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
32 |
self.outputopenedok = True |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
33 |
except: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
34 |
self.outputopenedok = False |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
35 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
36 |
def write(self, data): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
37 |
if self.outputopenedok: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
38 |
compresseddata = self.compressor.compress(data) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
39 |
self.fh.write(compresseddata) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
40 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
41 |
def __del__(self): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
42 |
if self.outputopenedok: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
43 |
compresseddata = self.compressor.flush() |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
44 |
self.fh.write(compresseddata) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
45 |
self.fh.close() |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
46 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
47 |
class Bz2log(filter_interface.Filter): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
48 |
def __init__(self): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
49 |
self.__inRecipe = False |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
50 |
self.compressor = None |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
51 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
52 |
def open(self, raptor_instance): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
53 |
"""Open a log file for the various I/O methods to write to.""" |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
54 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
55 |
if raptor_instance.logFileName == None: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
56 |
self.out = sys.stdout # Default to stdout if no log file is given |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
57 |
else: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
58 |
logname = str(raptor_instance.logFileName.path.replace("%TIME", raptor_instance.timestring)) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
59 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
60 |
# Ensure that filename has the right extension; append ".bz2" if required |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
61 |
if not logname.lower().endswith(".bz2"): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
62 |
logname += ".bz2" |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
63 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
64 |
try: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
65 |
dirname = str(raptor_instance.logFileName.Dir()) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
66 |
if dirname and not os.path.isdir(dirname): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
67 |
os.makedirs(dirname) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
68 |
except: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
69 |
self.formatError("cannot create directory %s", dirname) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
70 |
return False |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
71 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
72 |
# Use highest compression level 9 which corresponds to a 900KB dictionary |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
73 |
self.compressor = StringListCompressor(9, logname) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
74 |
if not self.compressor.outputopenedok: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
75 |
self.out = None |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
76 |
self.formatError("failed to initialise compression routines." ) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
77 |
return False |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
78 |
return True |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
79 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
80 |
def write(self, data): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
81 |
"""Write data compressed log""" |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
82 |
if self.compressor: |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
83 |
self.compressor.write(data) |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
84 |
return True |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
85 |
|
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
86 |
def close(self): |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
87 |
"""Close the log file""" |
d677c6f132ae
Add Bzip2 log filter/compressor to Raptor.
Daniel Jacobs <daniel.jacobs@nokia.com>
parents:
diff
changeset
|
88 |
return True |