author | wbernard |
Wed, 23 Dec 2009 19:29:07 +0200 | |
changeset 179 | d8ac696cc51f |
parent 1 | be27ed110b50 |
permissions | -rw-r--r-- |
1
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
1 |
#============================================================================ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
2 |
#Name : mappers.py |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
3 |
#Part of : Helium |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
4 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
5 |
#Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
6 |
#All rights reserved. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
7 |
#This component and the accompanying materials are made available |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
8 |
#under the terms of the License "Eclipse Public License v1.0" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
9 |
#which accompanies this distribution, and is available |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
10 |
#at the URL "http://www.eclipse.org/legal/epl-v10.html". |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
11 |
# |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
12 |
#Initial Contributors: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
13 |
#Nokia Corporation - initial contribution. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
14 |
# |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
15 |
#Contributors: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
16 |
# |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
17 |
#Description: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
18 |
#=============================================================================== |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
19 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
20 |
""" Archive mappers that map how the input files are divided into archives. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
21 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
22 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
23 |
""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
24 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
25 |
import buildtools |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
26 |
import os |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
27 |
import codecs |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
28 |
import fileutils |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
29 |
import logging |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
30 |
import symrec |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
31 |
import re |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
32 |
import csv |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
33 |
import shutil |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
34 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
35 |
_logger = logging.getLogger('logger.mappers') |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
36 |
_logger.setLevel(logging.INFO) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
37 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
38 |
# Default value for missing/invalid policy files. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
39 |
MISSING_POLICY = "9999" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
40 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
41 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
42 |
class Mapper(object): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
43 |
""" Mapper Abstract class. Any custom implementation must derive it!. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
44 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
45 |
It handles metadata creation. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
46 |
""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
47 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
48 |
def __init__(self, config, tool): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
49 |
self._tool = tool |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
50 |
self._config = config |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
51 |
self._metadata = None |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
52 |
if not os.path.exists(self._config['archives.dir']): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
53 |
os.makedirs(self._config['archives.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
54 |
if self._config.has_key("grace.metadata") and self._config.get_boolean("grace.metadata", False): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
55 |
if self._config.has_key("grace.template") and os.path.exists(self._config["grace.template"]) and \ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
56 |
not os.path.exists(os.path.join(self._config['archives.dir'], self._config['name'] + ".metadata.xml")): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
57 |
shutil.copy(config["grace.template"], os.path.join(self._config['archives.dir'], self._config['name'] + ".metadata.xml")) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
58 |
self._metadata = symrec.ReleaseMetadata(os.path.join(self._config['archives.dir'], self._config['name']+ ".metadata.xml"), |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
59 |
service=self._config['grace.service'], |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
60 |
product=self._config['grace.product'], |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
61 |
release=self._config['grace.release']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
62 |
self._metadata.save() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
63 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
64 |
def declare_package(self, filename, extract="single"): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
65 |
""" Add a package to the metadata file. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
66 |
if self._metadata is None: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
67 |
return |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
68 |
self._metadata.add_package(os.path.basename(filename), extract=extract, filters=self._config.get_list('grace.filters', None), default=self._config.get_boolean('grace.default', True)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
69 |
self._metadata.save() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
70 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
71 |
def create_commands(self, manifest): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
72 |
""" Return a list of command list. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
73 |
return [[self._tool.create_command(self._config['name'], manifests=[manifest])]] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
74 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
75 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
76 |
class DefaultMapper(Mapper): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
77 |
""" The default mapper. It splits the content based on size characteristics. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
78 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
79 |
'the max.files.per.archive' and 'max.uncompressed.size' properties define how the input files |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
80 |
are split between a number of part zips. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
81 |
""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
82 |
def __init__(self, config, archiver): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
83 |
""" Initialization. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
84 |
Mapper.__init__(self, config, archiver) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
85 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
86 |
def create_commands(self, manifest): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
87 |
""" Return a list of command lists. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
88 |
result = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
89 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
90 |
_logger.info(" * Input manifest: " + manifest) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
91 |
manifests = self._split_manifest_file(self._config['name'], manifest) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
92 |
if not os.path.exists(self._config['archives.dir']): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
93 |
_logger.info(" * Mkdir " + self._config['archives.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
94 |
os.makedirs(self._config['archives.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
95 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
96 |
for manifest in manifests: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
97 |
_logger.info(" * Creating command for manifest: " + manifest) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
98 |
filename = os.path.join(self._config['archives.dir'], os.path.splitext(os.path.basename(manifest))[0]) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
99 |
if len(manifests) == 1: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
100 |
filename = os.path.join(self._config['archives.dir'], self._config['name']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
101 |
_logger.info(" * " + filename + self._tool.extension()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
102 |
self.declare_package(filename + self._tool.extension(), self._config.get('grace.extract', 'single')) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
103 |
result.extend(self._tool.create_command(self._config.get('zip.root.dir', self._config['root.dir']), filename, manifests=[manifest])) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
104 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
105 |
return [result] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
106 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
107 |
def _split_manifest_file(self, name, manifest_file_path): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
108 |
""" This method return a list of files that contain the content of the zip parts to create. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
109 |
filenames = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
110 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
111 |
if (self._config.has_key('max.files.per.archive') or self._config.has_key('max.uncompressed.size')): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
112 |
size = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
113 |
files = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
114 |
part = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
115 |
filename = "" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
116 |
output = None |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
117 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
118 |
if os.path.exists(self._config['root.dir']) and os.path.isdir(self._config['root.dir']): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
119 |
curdir = os.path.abspath(os.curdir) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
120 |
os.chdir(self._config.get('zip.root.dir', self._config['root.dir'])) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
121 |
maxfiles = self._config.get('max.files.per.archive', 100000000) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
122 |
_logger.info("Max number of files per archive: " + str(maxfiles)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
123 |
max_uncompressed_size = self._config.get('max.uncompressed.size', 100000000) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
124 |
_logger.info("Max uncompressed size per archive: " + str(max_uncompressed_size)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
125 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
126 |
file_handle = codecs.open(manifest_file_path, "r", "utf-8" ) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
127 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
128 |
for line in file_handle.readlines(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
129 |
line = line.rstrip() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
130 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
131 |
if(os.path.isfile(line)): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
132 |
if part == 0 or files == int(maxfiles) or size + os.path.getsize(line) >= int(max_uncompressed_size): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
133 |
if output != None: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
134 |
output.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
135 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
136 |
size = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
137 |
files = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
138 |
part += 1 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
139 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
140 |
filename = "%s_part%02d" % (name, part) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
141 |
filenames.append(os.path.join(self._config['temp.build.dir'], filename + ".txt")) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
142 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
143 |
output = codecs.open(os.path.join(self._config['temp.build.dir'], filename + ".txt"), 'w', "utf-8" ) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
144 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
145 |
files += 1 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
146 |
size += os.path.getsize(line) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
147 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
148 |
output.write(u"".join([line, u'\n'])) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
149 |
elif(os.path.isdir(line)): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
150 |
if (len(os.listdir(line)) == 0): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
151 |
if part == 0 or files == int(maxfiles): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
152 |
if output != None: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
153 |
output.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
154 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
155 |
size = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
156 |
files = 0 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
157 |
part += 1 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
158 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
159 |
filename = "%s_part%02d" % (name, part) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
160 |
filenames.append(os.path.join(self._config['temp.build.dir'], filename + ".txt")) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
161 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
162 |
output = open(os.path.abspath(os.path.join(self._config['temp.build.dir'], filename + ".txt")), 'w') |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
163 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
164 |
files += 1 |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
165 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
166 |
output.write(u"".join([line, u'\n'])) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
167 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
168 |
_logger.warning('Not recognized as file or directory: %s' % line) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
169 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
170 |
if output != None: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
171 |
output.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
172 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
173 |
file_handle.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
174 |
os.chdir(curdir) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
175 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
176 |
filenames.append(manifest_file_path) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
177 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
178 |
return filenames |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
179 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
180 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
181 |
class PolicyMapper(Mapper): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
182 |
""" Implements a policy content mapper. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
183 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
184 |
It transforms a list of files into a list of commands with their inputs. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
185 |
All files with policy 0 will be under the main archive. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
186 |
All other files will get backed up by policy and then store into an second archive. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
187 |
""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
188 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
189 |
def __init__(self, config, archiver): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
190 |
""" Initialization. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
191 |
Mapper.__init__(self, config, archiver) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
192 |
self._policies = {} |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
193 |
self._policy_cache = {} |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
194 |
self._binary = {} |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
195 |
# Load csv |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
196 |
if self._config.has_key('policy.csv'): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
197 |
if os.path.exists(self._config['policy.csv']): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
198 |
self.load_policy_binary(self._config['policy.csv']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
199 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
200 |
_logger.error("POLICY_ERROR: File not found '%s'." % self._config['policy.csv']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
201 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
202 |
def load_policy_binary(self, csvfile, column=1): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
203 |
""" Loads the binary IDs from the CSV file. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
204 |
_logger.info("POLICY_INFO: Loading policy definition '%s'." % csvfile) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
205 |
reader = csv.reader(open(csvfile, "rU")) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
206 |
for row in reader: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
207 |
if re.match(r"^((?:\d+)|(?:0842[0-9a-zA-Z]{3}))$", row[0].strip()): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
208 |
_logger.info("POLICY_INFO: Adding policy: '%s' => '%s'" % (row[0].strip(), row[column].strip().lower())) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
209 |
self._binary[row[0].strip()] = row[column].strip().lower() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
210 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
211 |
_logger.warning("POLICY_WARNING: Discarding policy: '%s'." % row[0].strip()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
212 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
213 |
def zip2zip(self): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
214 |
""" Should the non public zip be zipped up under a specific zip. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
215 |
return self._config.get_boolean('policy.zip2zip', False) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
216 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
217 |
def create_commands(self, manifest): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
218 |
""" Generates a list of build commands. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
219 |
result = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
220 |
stages = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
221 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
222 |
# Create the archive output directory |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
223 |
if not os.path.exists(self._config['archives.dir']): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
224 |
_logger.info(" * Mkdir " + self._config['archives.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
225 |
os.makedirs(self._config['archives.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
226 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
227 |
# Sort the manifest content, splitting it by policy |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
228 |
file_handle = codecs.open(manifest, "r", "utf-8") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
229 |
for line in file_handle.readlines(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
230 |
line = line.rstrip() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
231 |
self._sort_by_policy(line) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
232 |
file_handle.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
233 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
234 |
# Generating sublists. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
235 |
for key in self._policies.keys(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
236 |
self._policies[key].close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
237 |
manifest = os.path.join(self._config['temp.build.dir'], self._config['name'] + "_%s" % key + ".txt") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
238 |
filename = os.path.join(self._config['archives.dir'], self._config['name'] + "_%s" % key) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
239 |
_logger.info(" * " + filename + self._tool.extension()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
240 |
result.extend(self._tool.create_command(self._config.get('zip.root.dir', self._config['root.dir']), filename, manifests=[manifest])) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
241 |
stages.append(result) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
242 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
243 |
# See if any internal archives need to be created |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
244 |
content = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
245 |
for key in self._policies.keys(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
246 |
if not self.zip2zip(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
247 |
self.declare_package(self._config['name'] + "_%s" % key + self._tool.extension()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
248 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
249 |
if key != "0": |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
250 |
content.append(os.path.join(self._config['archives.dir'], self._config['name'] + "_%s" % key + self._tool.extension())) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
251 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
252 |
self.declare_package(self._config['name'] + "_%s" % key + self._tool.extension()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
253 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
254 |
# Creating zip that contains each policy zips. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
255 |
if self.zip2zip() and len(content) > 0: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
256 |
manifest = os.path.join(self._config['temp.build.dir'], self._config['name'] + ".internal.txt") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
257 |
file_handle = codecs.open( manifest, "w+", "utf-8" ) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
258 |
file_handle.write(u"\n".join(content)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
259 |
file_handle.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
260 |
internal = "internal" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
261 |
if self._config.has_key('policy.internal.name'): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
262 |
internal = self._config['policy.internal.name'] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
263 |
filename = os.path.join(self._config['archives.dir'], self._config['name'] + "_" + internal) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
264 |
_logger.info(" * " + filename + self._tool.extension()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
265 |
self.declare_package(filename + self._tool.extension(), "double") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
266 |
stages.append(self._tool.create_command(self._config['archives.dir'], filename, manifests=[manifest])) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
267 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
268 |
cmds = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
269 |
for filename in content: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
270 |
cmds.append(buildtools.Delete(filename=filename)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
271 |
stages.append(cmds) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
272 |
return stages |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
273 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
274 |
def get_dir_policy(self, dirname): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
275 |
""" Get policy value for a specific directory. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
276 |
dirname = os.path.normpath(dirname) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
277 |
if not self._policy_cache.has_key(dirname): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
278 |
policyfile = None |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
279 |
for name in self.get_policy_filenames(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
280 |
if os.sep != '\\': |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
281 |
for filename in os.listdir(dirname): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
282 |
if filename.lower() == name.lower(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
283 |
policyfile = os.path.join(dirname, filename) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
284 |
break |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
285 |
elif os.path.exists(os.path.join(dirname, name)): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
286 |
policyfile = os.path.join(dirname, name) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
287 |
break |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
288 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
289 |
value = self._config.get('policy.default.value', MISSING_POLICY) |
179 | 290 |
if policyfile != None: #policy file present |
1
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
291 |
try: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
292 |
value = fileutils.read_policy_content(policyfile) |
179 | 293 |
if value not in self._binary.keys(): #check policy file is valid |
1
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
294 |
_logger.error("POLICY_ERROR: policy file found %s but policy %s value not exists in csv" % (policyfile, value)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
295 |
except Exception, exc: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
296 |
_logger.error("POLICY_ERROR: %s" % exc) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
297 |
value = self._config.get('policy.default.value', MISSING_POLICY) |
179 | 298 |
else: #no policy file present |
299 |
filePresent = False |
|
300 |
dirPresent = False |
|
301 |
for ftype in os.listdir(dirname): #see if files or directories are present |
|
302 |
if os.path.isdir(os.path.join(dirname, ftype)): |
|
303 |
dirPresent = True |
|
304 |
if os.path.isfile(os.path.join(dirname, ftype)): |
|
305 |
filePresent = True |
|
306 |
||
307 |
if filePresent: #files present : error |
|
308 |
_logger.error("POLICY_ERROR: could not find a policy file under: '%s'" % dirname) |
|
309 |
elif dirPresent and not filePresent: #directories only : warning |
|
310 |
_logger.error("POLICY_WARNING: no policy file, no files present, but sub-folder present in : '%s'" % dirname) |
|
311 |
else: #no files no dirs : warning |
|
312 |
_logger.error("POLICY_WARNING: empty directory at : '%s'" % dirname) |
|
313 |
||
1
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
314 |
# saving the policy value for that directory. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
315 |
self._policy_cache[dirname] = value |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
316 |
return self._policy_cache[dirname] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
317 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
318 |
def get_policy_filenames(self): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
319 |
""" Returns the list of potential policy filenames. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
320 |
return self._config.get_list('policy.filenames', ['Distribution.policy.s60']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
321 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
322 |
def _sort_by_policy(self, filename): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
323 |
""" Store the input file sorted by its policy number. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
324 |
path = os.path.join(self._config['root.dir'], filename) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
325 |
parentdir = os.path.dirname(path) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
326 |
if os.path.isdir(path): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
327 |
parentdir = path |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
328 |
value = self.get_dir_policy(parentdir) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
329 |
if not value in self._policies: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
330 |
self._policies[value] = codecs.open( os.path.join(self._config['temp.build.dir'], self._config['name'] + "_%s" % value + ".txt"), "w+", "utf-8" ) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
331 |
self._policies[value].write(u"%s\n" % filename) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
332 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
333 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
334 |
class PolicyRemoverMapper(PolicyMapper): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
335 |
""" This class implements a variant of the policy mapper. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
336 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
337 |
It removes the internal source. Only binary flagged content is kept. |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
338 |
""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
339 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
340 |
def __init__(self, config, archiver): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
341 |
""" Initialization. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
342 |
PolicyMapper.__init__(self, config, archiver) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
343 |
self._rm_policy_cache = {} |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
344 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
345 |
def get_policy_root_dir(self): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
346 |
""" Return the policy.root.dir or root.dir if not set or not under root.dir.""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
347 |
if not self._config.has_key("policy.root.dir"): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
348 |
return os.path.normpath(self._config['root.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
349 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
350 |
if fileutils.destinsrc(self._config['root.dir'], self._config['policy.root.dir']): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
351 |
return os.path.normpath(self._config['policy.root.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
352 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
353 |
return os.path.normpath(self._config['root.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
354 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
355 |
def get_rmdir_policy(self, dirname): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
356 |
""" check if the directory should be dropped or not""" |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
357 |
dirname = os.path.normpath(dirname) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
358 |
# check if parent is banned... |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
359 |
prootdir = os.path.normpath(self.get_policy_root_dir()) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
360 |
rootdir = os.path.normpath(self._config['root.dir']) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
361 |
if os.sep == '\\': |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
362 |
dirname = dirname.lower() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
363 |
prootdir = prootdir.lower() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
364 |
rootdir = rootdir.lower() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
365 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
366 |
# else get real value... |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
367 |
if not self._rm_policy_cache.has_key(dirname): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
368 |
self._rm_policy_cache[dirname] = self.get_dir_policy(dirname) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
369 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
370 |
return self._rm_policy_cache[dirname] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
371 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
372 |
def create_commands(self, manifest): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
373 |
""" Generates a list of build commands. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
374 |
stages = PolicyMapper.create_commands(self, manifest) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
375 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
376 |
if not self._config.has_key('policy.csv'): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
377 |
_logger.error("POLICY_ERROR: Property 'policy.csv' not defined everything will get removed.") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
378 |
cmds = [] |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
379 |
file_handle = codecs.open( manifest, "r", "utf-8" ) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
380 |
for line in file_handle.readlines(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
381 |
line = line.rstrip() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
382 |
filepath = os.path.normpath(os.path.join(self._config.get('zip.root.dir', self._config['root.dir']), line)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
383 |
value = self.get_rmdir_policy(os.path.dirname(filepath)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
384 |
delete = True |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
385 |
if value in self._binary.keys(): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
386 |
if self._binary[value] == "yes": |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
387 |
_logger.info("POLICY_INFO: Keeping %s (%s=>yes)!" % (filepath, value)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
388 |
delete = False |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
389 |
elif self._binary[value] == "bin": |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
390 |
_logger.info("POLICY_INFO: Keeping %s (%s=>bin)!" % (filepath, value)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
391 |
delete = False |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
392 |
else: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
393 |
_logger.error("POLICY_ERROR: %s value for %s not in csv file. Will be removed!!" % (value, filepath)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
394 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
395 |
if delete: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
396 |
_logger.info("POLICY_INFO: File %s will be removed!" % filepath) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
397 |
cmds.append(buildtools.Delete(filename=filepath)) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
398 |
file_handle.close() |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
399 |
if len(cmds) > 0: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
400 |
stages.append(cmds) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
401 |
return stages |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
402 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
403 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
404 |
class SFPolicyRemoverMapper(PolicyRemoverMapper): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
405 |
""" Implement an SFL column based policy remover. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
406 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
407 |
def __init__(self, config, archiver): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
408 |
""" Initialization. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
409 |
PolicyRemoverMapper.__init__(self, config, archiver) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
410 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
411 |
def load_policy_binary(self, csvfile): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
412 |
""" Loading the policy using the 3rd column. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
413 |
_logger.info("POLICY_INFO: Loading actions from the 3rd column") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
414 |
PolicyRemoverMapper.load_policy_binary(self, csvfile, column=3) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
415 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
416 |
class EPLPolicyRemoverMapper(PolicyRemoverMapper): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
417 |
""" Implement an EPL column based policy remover. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
418 |
def __init__(self, config, archiver): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
419 |
""" Initialization. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
420 |
PolicyRemoverMapper.__init__(self, config, archiver) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
421 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
422 |
def load_policy_binary(self, csvfile): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
423 |
""" Loading the policy using the 4th column. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
424 |
_logger.info("POLICY_INFO: Loading actions from the 4th column") |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
425 |
PolicyRemoverMapper.load_policy_binary(self, csvfile, column=4) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
426 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
427 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
428 |
MAPPERS = {'default': DefaultMapper, |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
429 |
'policy': PolicyMapper, |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
430 |
'policy.remover': PolicyRemoverMapper, |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
431 |
'sfl.policy.remover': SFPolicyRemoverMapper, |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
432 |
'epl.policy.remover': EPLPolicyRemoverMapper,} |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
433 |
|
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
434 |
def get_mapper(name, config, archiver): |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
435 |
""" Get mapper instance from its string id. """ |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
436 |
if name in MAPPERS: |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
437 |
return MAPPERS[name](config, archiver) |
be27ed110b50
Bringing in Helium, imaker and cmaker
Alex Gilkes <alex.gilkes@nokia.com>
parents:
diff
changeset
|
438 |
raise Exception("ERROR: Could not find mapper '%s'." % name) |