0
|
1 |
#
|
|
2 |
# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
|
|
3 |
# All rights reserved.
|
|
4 |
# This component and the accompanying materials are made available
|
|
5 |
# under the terms of "Eclipse Public License v1.0"
|
|
6 |
# which accompanies this distribution, and is available
|
|
7 |
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
|
|
8 |
#
|
|
9 |
# Initial Contributors:
|
|
10 |
# Nokia Corporation - initial contribution.
|
|
11 |
#
|
|
12 |
# Contributors:
|
|
13 |
#
|
|
14 |
# Description:
|
|
15 |
#
|
|
16 |
'''
|
|
17 |
A plugin implementation for content selection from ConfigurationLayers.
|
|
18 |
'''
|
|
19 |
|
|
20 |
|
|
21 |
import re
|
|
22 |
import os
|
|
23 |
import sys
|
|
24 |
import logging
|
|
25 |
import shutil
|
3
|
26 |
import pkg_resources
|
0
|
27 |
|
|
28 |
import __init__
|
|
29 |
|
|
30 |
from cone.public import exceptions,plugin,utils,api,container
|
|
31 |
from contentplugin import contentmlparser
|
|
32 |
|
|
33 |
class ContentImpl(plugin.ImplBase):
|
|
34 |
"""
|
|
35 |
ContentImpl plugin finds all content resources from each layer and copies
|
|
36 |
them to the output correctly. It follows the Configuration project override
|
|
37 |
rules, so that the topmost layer files override files on the previous layers.
|
|
38 |
"""
|
|
39 |
|
|
40 |
IMPL_TYPE_ID = 'content'
|
|
41 |
|
|
42 |
def __init__(self,ref,configuration):
|
|
43 |
"""
|
|
44 |
Overloading the default constructor
|
|
45 |
"""
|
|
46 |
plugin.ImplBase.__init__(self,ref,configuration)
|
|
47 |
self.desc = ""
|
|
48 |
self.logger = logging.getLogger('cone.content(%s)' % self.ref)
|
|
49 |
self.errors = False
|
3
|
50 |
self.context = plugin.GenerationContext(configuration=configuration)
|
|
51 |
|
0
|
52 |
|
|
53 |
def list_output_files(self):
|
|
54 |
"""
|
|
55 |
Return a list of output files as an array.
|
|
56 |
"""
|
|
57 |
if not self.errors:
|
|
58 |
copylist = self.get_full_copy_list()
|
|
59 |
outputfiles = [entry[1] for entry in copylist]
|
|
60 |
return outputfiles
|
|
61 |
else:
|
|
62 |
return []
|
|
63 |
|
|
64 |
def get_refs(self):
|
|
65 |
refs = []
|
|
66 |
for output in self.outputs:
|
|
67 |
refs.extend(output.get_refs())
|
|
68 |
if refs:
|
|
69 |
return refs
|
|
70 |
else:
|
|
71 |
return None
|
|
72 |
|
|
73 |
def get_full_copy_list(self, print_info=False):
|
|
74 |
fullcopylist = []
|
|
75 |
for output in self.outputs:
|
|
76 |
for input in output.inputs:
|
|
77 |
copylist = []
|
|
78 |
if print_info:
|
|
79 |
self.logger.info('Content copy items from %s to %s' % (input.dir,os.path.join(self.output,output.dir)))
|
|
80 |
|
|
81 |
if input.__class__.__name__ == "ContentInput":
|
3
|
82 |
copylist = self.create_copy_list(content=self.configuration.layered_content(empty_folders=True),
|
0
|
83 |
input=input.dir,
|
|
84 |
output=os.path.join(self.output,output.dir),
|
|
85 |
include_pattern=input.get_include_pattern(),
|
|
86 |
exclude_pattern=input.get_exclude_pattern(),
|
|
87 |
files=input.get_filelist(),
|
|
88 |
flatten=output.flatten,
|
|
89 |
output_file=output.file)
|
|
90 |
elif input.__class__.__name__ == "ExternalContentInput":
|
|
91 |
#Handling external inputs separately
|
|
92 |
if input.dir != None:
|
|
93 |
fulldir = os.path.abspath(os.path.join(self.configuration.get_project().get_storage().get_path(),input.dir))
|
|
94 |
else:
|
|
95 |
fulldir = self.configuration.get_project().get_storage().get_path()
|
|
96 |
|
3
|
97 |
data = container.DataContainer()
|
0
|
98 |
for root, dirs, files in os.walk(fulldir):
|
3
|
99 |
for f in files:
|
0
|
100 |
filepath = utils.resourceref.norm(os.path.join(root, f))
|
|
101 |
key = utils.resourceref.replace_dir(filepath,fulldir,"")
|
|
102 |
data.add_value(key,filepath)
|
3
|
103 |
|
|
104 |
# If the root contains no directories and no files, it is
|
|
105 |
# an empty directory and needs to be added
|
|
106 |
if not dirs and not files:
|
|
107 |
filepath = utils.resourceref.norm(root)
|
|
108 |
key = utils.resourceref.replace_dir(filepath,fulldir,"")
|
|
109 |
data.add_value(key,filepath)
|
|
110 |
|
0
|
111 |
copylist = self.create_copy_list(content=data,
|
|
112 |
input=input.dir,
|
|
113 |
output=os.path.join(self.output,output.dir),
|
|
114 |
include_pattern=input.get_include_pattern(),
|
|
115 |
exclude_pattern=input.get_exclude_pattern(),
|
|
116 |
files=input.get_filelist(),
|
|
117 |
flatten=output.flatten,
|
|
118 |
output_file=output.file,
|
|
119 |
external=True)
|
|
120 |
else:
|
|
121 |
logging.getLogger('cone.content').warning("Unknown input %s" % (input.__class__.__name__))
|
|
122 |
|
|
123 |
fullcopylist += copylist
|
3
|
124 |
|
|
125 |
# Sort to make automated testing easier (list always in same order)
|
|
126 |
fullcopylist.sort()
|
|
127 |
|
0
|
128 |
return fullcopylist
|
|
129 |
|
|
130 |
def generate(self, context=None):
|
|
131 |
"""
|
|
132 |
Generate the given implementation.
|
|
133 |
"""
|
3
|
134 |
#assert context, "No Context given for generation!"
|
|
135 |
self.context = context
|
0
|
136 |
self.logger.info('Generating')
|
|
137 |
self.create_output()
|
|
138 |
return
|
|
139 |
|
|
140 |
def create_output(self,layers=None):
|
|
141 |
"""
|
|
142 |
Create the output directory from the content folder files
|
|
143 |
"""
|
|
144 |
if not self.errors:
|
|
145 |
copylist = self.get_full_copy_list(True)
|
|
146 |
for copy_item in copylist:
|
3
|
147 |
source_path = copy_item[0]
|
|
148 |
target_path = copy_item[1]
|
|
149 |
external = copy_item[2]
|
|
150 |
|
|
151 |
self.logger.info('Copy from %s to %s' % (source_path,target_path))
|
0
|
152 |
|
3
|
153 |
# Open file resource if the source is a file
|
|
154 |
file_res = None
|
|
155 |
if not external and not self.configuration.get_storage().is_folder(source_path):
|
|
156 |
file_res = self.configuration.get_storage().open_resource(source_path, "rb")
|
|
157 |
elif external and os.path.isfile(source_path):
|
|
158 |
file_res = open(source_path, 'rb')
|
|
159 |
|
|
160 |
# Copy file or create empty directory
|
|
161 |
if file_res:
|
|
162 |
try: self._copy_file(file_res, target_path)
|
|
163 |
finally: file_res.close()
|
0
|
164 |
else:
|
3
|
165 |
path = os.path.join(self.context.output, target_path)
|
|
166 |
if not os.path.exists(path): os.makedirs(path)
|
0
|
167 |
return
|
|
168 |
else:
|
3
|
169 |
self.logger.error('Plugin had errors! Bailing out!')
|
|
170 |
|
|
171 |
def _copy_file(self, source_file, target_file_path):
|
|
172 |
outfile = self.context.create_file(target_file_path, implementation=self)
|
|
173 |
try:
|
|
174 |
# Copy data in chunks of max 2 MB to avoid
|
|
175 |
# memory errors with very large files
|
|
176 |
while True:
|
|
177 |
data = source_file.read(2 * 1024 * 1024)
|
|
178 |
if data: outfile.write(data)
|
|
179 |
else: break
|
|
180 |
finally:
|
|
181 |
outfile.close()
|
0
|
182 |
|
|
183 |
def create_copy_list(self, **kwargs):
|
|
184 |
"""
|
3
|
185 |
Return a list copy list where each element is a (from, to, is_external) tuple
|
0
|
186 |
"""
|
|
187 |
datacontainer = kwargs.get('content',None)
|
|
188 |
input_dir = kwargs.get('input','')
|
|
189 |
output_dir = kwargs.get('output','')
|
|
190 |
output_file = kwargs.get('output_file','')
|
|
191 |
include_filter= kwargs.get('include_pattern','')
|
|
192 |
exclude_filter= kwargs.get('exclude_pattern','')
|
|
193 |
files = kwargs.get('files',[])
|
|
194 |
flatten = kwargs.get('flatten',False)
|
|
195 |
external = kwargs.get('external',False)
|
|
196 |
copylist = []
|
|
197 |
contentfiles = datacontainer.list_keys()
|
|
198 |
"""
|
|
199 |
First get only the files list from content files.
|
|
200 |
Then apply the possible filters.
|
|
201 |
"""
|
|
202 |
if input_dir == None:
|
3
|
203 |
self.logger.warning("Input dir is none!")
|
0
|
204 |
|
|
205 |
|
|
206 |
if files != []:
|
|
207 |
for f in files:
|
|
208 |
if f in contentfiles:
|
|
209 |
pass
|
|
210 |
elif f not in contentfiles:
|
|
211 |
self.logger.info("File: %s not found in content" % f)
|
|
212 |
|
|
213 |
|
|
214 |
if files != []:
|
|
215 |
filesfunc = lambda x: x.lower() in [f.lower() for f in files]
|
|
216 |
contentfiles = filter(filesfunc, contentfiles)
|
|
217 |
if include_filter != "":
|
|
218 |
filter_regexp = include_filter
|
|
219 |
filter_regexp = filter_regexp.replace('.','\.')
|
|
220 |
filter_regexp = filter_regexp.replace('*','.*')
|
|
221 |
self.logger.info("filtering with include %s" % filter_regexp)
|
|
222 |
contentfiles = utils.resourceref.filter_resources(contentfiles,filter_regexp)
|
|
223 |
if exclude_filter != "":
|
|
224 |
filter_regexp = exclude_filter
|
|
225 |
filter_regexp = filter_regexp.replace('.','\.')
|
|
226 |
filter_regexp = filter_regexp.replace('*','.*')
|
|
227 |
self.logger.info("filtering with exclude %s" % filter_regexp)
|
|
228 |
contentfiles = utils.resourceref.neg_filter_resources(contentfiles,filter_regexp)
|
|
229 |
for outfile in contentfiles:
|
|
230 |
sourcefile = ""
|
|
231 |
targetfile = ""
|
|
232 |
|
3
|
233 |
# For the startswith() check, make sure that input dir has a trailing slash
|
|
234 |
if input_dir and input_dir[-1] != '/': input_dir_check = input_dir + '/'
|
|
235 |
else: input_dir_check = input_dir
|
|
236 |
|
|
237 |
if input_dir != None and (input_dir == outfile or outfile.startswith(input_dir_check)):
|
0
|
238 |
sourcefile = datacontainer.get_value(outfile)
|
|
239 |
if flatten:
|
|
240 |
targetfile = utils.resourceref.join_refs([output_dir, os.path.basename(outfile)])
|
|
241 |
targetfile = utils.resourceref.norm(targetfile)
|
|
242 |
else:
|
|
243 |
targetfile = utils.resourceref.replace_dir(outfile,input_dir,output_dir)
|
|
244 |
elif external:
|
|
245 |
#External inputs
|
|
246 |
sourcefile = utils.resourceref.norm(datacontainer.get_value(outfile))
|
|
247 |
|
|
248 |
if flatten:
|
|
249 |
targetfile = utils.resourceref.join_refs([output_dir, os.path.basename(sourcefile)])
|
|
250 |
targetfile = utils.resourceref.norm(targetfile)
|
|
251 |
else:
|
|
252 |
fulldir = os.path.abspath(os.path.join(self.configuration.get_project().get_storage().get_path(),input_dir))
|
|
253 |
targetfile = utils.resourceref.replace_dir(sourcefile,fulldir,output_dir)
|
|
254 |
|
|
255 |
if output_file:
|
|
256 |
#Renaming output if defined
|
|
257 |
targetfile = targetfile.replace(os.path.basename(targetfile), output_file)
|
3
|
258 |
|
|
259 |
if sourcefile and targetfile:
|
|
260 |
copylist.append((sourcefile,targetfile,external))
|
0
|
261 |
return copylist
|
3
|
262 |
|
|
263 |
def uses_layer(self, layer):
|
|
264 |
layered_content = layer.layered_content().list_keys()
|
|
265 |
for f in self.get_full_copy_list():
|
|
266 |
for file in layered_content:
|
|
267 |
if utils.resourceref.norm(os.path.join(utils.resourceref.get_path(layer.get_path()), 'content', file)) == f[0]:
|
|
268 |
return True
|
|
269 |
return False
|
|
270 |
|
|
271 |
def uses_layers(self, layers, context):
|
|
272 |
# Use the base implementation to check with refs first
|
|
273 |
if super(ContentImpl, self).uses_layers(layers, context):
|
|
274 |
return True
|
|
275 |
|
|
276 |
# Then check if any of the files in the copy list come from the layers
|
|
277 |
copy_list = self.get_full_copy_list()
|
|
278 |
for layer in layers:
|
|
279 |
layered_content = layer.layered_content().list_keys()
|
|
280 |
for f in copy_list:
|
|
281 |
for file in layered_content:
|
|
282 |
if utils.resourceref.norm(os.path.join(utils.resourceref.get_path(layer.get_path()), 'content', file)) == f[0]:
|
|
283 |
return True
|
|
284 |
return False
|
|
285 |
|
|
286 |
|
0
|
287 |
class ContentImplReaderBase(object):
|
|
288 |
FILE_EXTENSIONS = ['content', 'contentml']
|
|
289 |
|
|
290 |
@classmethod
|
|
291 |
def read_impl(cls, resource_ref, configuration, etree):
|
|
292 |
parser = cls.parser_class()
|
|
293 |
|
|
294 |
desc = parser.parse_desc(etree)
|
|
295 |
outputs = parser.parse_outputs(etree)
|
|
296 |
phase = parser.parse_phase(etree)
|
|
297 |
tags = parser.parse_tags(etree)
|
|
298 |
|
|
299 |
impl = ContentImpl(resource_ref, configuration)
|
|
300 |
impl.desc = desc
|
|
301 |
impl.outputs = outputs
|
|
302 |
if tags:
|
|
303 |
impl.set_tags(tags)
|
|
304 |
for output in impl.outputs:
|
|
305 |
output.set_configuration(configuration)
|
|
306 |
if phase != None:
|
|
307 |
impl.set_invocation_phase(phase)
|
|
308 |
|
|
309 |
return impl
|
|
310 |
|
|
311 |
class ContentImplReader1(ContentImplReaderBase, plugin.ReaderBase):
|
|
312 |
NAMESPACE = 'http://www.s60.com/xml/content/1'
|
3
|
313 |
NAMESPACE_ID = 'contentml1'
|
|
314 |
ROOT_ELEMENT_NAME = 'content'
|
0
|
315 |
parser_class = contentmlparser.Content1Parser
|
3
|
316 |
|
|
317 |
@classmethod
|
|
318 |
def get_schema_data(cls):
|
|
319 |
return pkg_resources.resource_string('contentplugin', 'xsd/contentml.xsd')
|
0
|
320 |
|
|
321 |
class ContentImplReader2(ContentImplReaderBase, plugin.ReaderBase):
|
|
322 |
NAMESPACE = 'http://www.s60.com/xml/content/2'
|
3
|
323 |
NAMESPACE_ID = 'contentml2'
|
|
324 |
ROOT_ELEMENT_NAME = 'content'
|
0
|
325 |
parser_class = contentmlparser.Content2Parser
|
3
|
326 |
|
|
327 |
@classmethod
|
|
328 |
def get_schema_data(cls):
|
|
329 |
return pkg_resources.resource_string('contentplugin', 'xsd/contentml2.xsd')
|