|
1 # |
|
2 # Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
3 # All rights reserved. |
|
4 # This component and the accompanying materials are made available |
|
5 # under the terms of "Eclipse Public License v1.0" |
|
6 # which accompanies this distribution, and is available |
|
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Nokia Corporation - initial contribution. |
|
11 # |
|
12 # Contributors: |
|
13 # |
|
14 # Description: |
|
15 # |
|
16 |
|
17 |
|
18 import os |
|
19 import re |
|
20 import StringIO |
|
21 import tokenize |
|
22 import inspect |
|
23 import traceback |
|
24 import logging |
|
25 import imghdr |
|
26 from token import ENDMARKER, NAME, NUMBER, STRING |
|
27 import api |
|
28 import mimetypes |
|
29 import exceptions |
|
30 |
|
31 import _etree_wrapper |
|
32 etree = _etree_wrapper.ElementTreeWrapper() |
|
33 |
|
34 class resourceref(object): |
|
35 """ |
|
36 Class container for set of resource reference related functions |
|
37 """ |
|
38 @classmethod |
|
39 def filter_resources(cls, resources, regexp): |
|
40 """ |
|
41 Filter out all resources that do not match the given regexp |
|
42 @return a array of resources that match the given resource |
|
43 """ |
|
44 test = re.compile(regexp, re.IGNORECASE) |
|
45 return [r for r in resources if test.search(r)] |
|
46 |
|
47 @classmethod |
|
48 def neg_filter_resources(cls, resources, regexp): |
|
49 """ |
|
50 Filter out all resources that do match the given regexp |
|
51 @return a array of resources that dont match the given resource |
|
52 """ |
|
53 test = re.compile(regexp, re.IGNORECASE) |
|
54 return [r for r in resources if not test.search(r)] |
|
55 |
|
56 @classmethod |
|
57 def insert_begin_slash(cls, ref): |
|
58 if not ref.startswith('/'): |
|
59 return '/' + ref |
|
60 return ref |
|
61 |
|
62 @classmethod |
|
63 def remove_begin_slash(cls, ref): |
|
64 if ref.startswith('/'): |
|
65 return ref.replace('/', '', 1) |
|
66 return ref |
|
67 |
|
68 @classmethod |
|
69 def remove_end(self, path, str): |
|
70 try: |
|
71 (ret, sep, rest) = path.partition(str) |
|
72 return ret |
|
73 except ValueError: |
|
74 return path |
|
75 |
|
76 @classmethod |
|
77 def add_end_slash(cls, ref): |
|
78 if not ref.endswith('/'): |
|
79 return ref+'/' |
|
80 return ref |
|
81 |
|
82 @classmethod |
|
83 def remove_end_slash(cls, ref): |
|
84 if ref.endswith('/'): |
|
85 return ref[:-1] |
|
86 return ref |
|
87 |
|
88 @classmethod |
|
89 def norm(cls, ref): |
|
90 """ |
|
91 Normalize the reference to common cone form. |
|
92 1. Always with forward slashes |
|
93 2. no beginning slash |
|
94 3. no end slash |
|
95 @return: A normalized reference string |
|
96 """ |
|
97 |
|
98 # Do not modify emtpy string at all |
|
99 if not ref == '': |
|
100 normref = os.path.normpath(ref) |
|
101 normref = normref.replace('\\','/').replace('"','').replace('//','/') |
|
102 else: |
|
103 normref = ref |
|
104 return normref |
|
105 |
|
106 @classmethod |
|
107 def replace_dir(cls, ref, frompart, topart): |
|
108 """ |
|
109 Replace a part of directory beginning from ref. |
|
110 @param ref: the resource reference |
|
111 @param frompart: the part of directory name to be replaced |
|
112 @param topart: the partial name which replaces the frompart |
|
113 @return: a refenence with forward slashes |
|
114 """ |
|
115 # Normalize all paths and replace the name with string replace |
|
116 # |
|
117 normref = cls.norm(ref) |
|
118 normfrom = cls.norm(frompart) |
|
119 normto = cls.norm(topart) |
|
120 # Add the end slash to from and to as it should be a dir (if not empty) |
|
121 if normto != "": normto = cls.add_end_slash(normto) |
|
122 if normfrom != "": normfrom = cls.add_end_slash(normfrom) |
|
123 if normref != "": normref = cls.add_end_slash(normref) |
|
124 retref = cls.norm(normref.replace(normfrom, normto, 1)) |
|
125 if retref != "": retref = cls.remove_end_slash(retref) |
|
126 return retref |
|
127 |
|
128 @classmethod |
|
129 def join_refs(cls, refs): |
|
130 """ |
|
131 join a list of dotted references together with dots |
|
132 1. ignore empty refs |
|
133 2. no dot include begin dot |
|
134 3. no dot include end dot |
|
135 @param refs: a list of references |
|
136 @return: A normalized dotted reference |
|
137 """ |
|
138 # Create a copy of references without any empty strings |
|
139 import posixpath |
|
140 paramdict = {} |
|
141 retref = posixpath.join(*refs) |
|
142 #retref = "/".join([ref for ref in refs if ref != '']) |
|
143 #subs = re.sub('/+', '/', retref) |
|
144 return retref |
|
145 |
|
146 @classmethod |
|
147 def split_ref(cls, ref): |
|
148 """ |
|
149 Replace a part of directory beginning from ref. |
|
150 @param ref: the resource reference |
|
151 @return: a list of path elems |
|
152 """ |
|
153 return [r for r in ref.split('/') if r] |
|
154 |
|
155 @classmethod |
|
156 def psplit_ref(cls, ref): |
|
157 """ |
|
158 pop split that splits the last element of the array |
|
159 1. empty ref returns an empty list |
|
160 @param ref: a resource references string (e.g. aaa/bbb/ccc.txt) |
|
161 @return: A tuple of references (with given param ('aaa/bbb','ccc.txt') |
|
162 """ |
|
163 refs = ref.rsplit('/', 1) |
|
164 return ("".join(refs[0:-1]), refs[-1]) |
|
165 |
|
166 @classmethod |
|
167 def remove_ext(cls, ref): |
|
168 """ |
|
169 Remove file extension from ref |
|
170 1. remove file extension |
|
171 @return: a reference. E.g. (foo/test.confml) => foo/test |
|
172 """ |
|
173 filenameparts = cls.get_filename(ref).rsplit('.', 1) |
|
174 path = cls.get_path(ref) |
|
175 if len(filenameparts)==2 and filenameparts[0] != "": |
|
176 return cls.join_refs([path, filenameparts[0]]) |
|
177 else: |
|
178 return ref |
|
179 |
|
180 @classmethod |
|
181 def get_ext(cls, ref): |
|
182 """ |
|
183 get file extension from ref |
|
184 1. get file extension |
|
185 @return: a reference. E.g. (foo/test.confml) => confml |
|
186 """ |
|
187 if len(ref.rsplit('.', 1)) == 2: |
|
188 return ref.rsplit('.', 1)[1] |
|
189 else: |
|
190 return "" |
|
191 |
|
192 @classmethod |
|
193 def get_filename(cls, ref): |
|
194 """ |
|
195 get file name part from ref |
|
196 1. get file extension |
|
197 @return: a reference. E.g. (foo/test.confml) => confml |
|
198 """ |
|
199 return ref.rsplit('/', 1)[-1] |
|
200 |
|
201 @classmethod |
|
202 def get_path(cls, ref): |
|
203 """ |
|
204 get file name part from ref |
|
205 1. get file extension |
|
206 @return: a reference. E.g. (foo/test.confml) => confml |
|
207 """ |
|
208 if len(ref.rsplit('/', 1)) == 2: |
|
209 return ref.rsplit('/', 1)[0] |
|
210 else: |
|
211 return "" |
|
212 |
|
213 @classmethod |
|
214 def to_dottedref(cls, ref): |
|
215 """ |
|
216 Convert a resource ref to dotted ref |
|
217 1. remove file extension |
|
218 2. convert path delims to dots |
|
219 @return: a dotted reference. E.g. (foo/test.confml) => foo_test |
|
220 """ |
|
221 newref = cls.remove_ext(ref).replace('/', '_').replace(' ', '_') |
|
222 return dottedref.remove_begin_dot(newref) |
|
223 |
|
224 |
|
225 @classmethod |
|
226 def to_objref(cls, ref): |
|
227 """ |
|
228 Convert a resource ref to dotted ref |
|
229 1. remove file extension |
|
230 2. convert path delims to dots |
|
231 3. using double underscores for directory separation |
|
232 @return: a dotted reference. E.g. (foo/test.confml) => foo_test |
|
233 """ |
|
234 ref = ref.replace('/', '__') |
|
235 newref = '' |
|
236 first_token = True |
|
237 try: |
|
238 for toknum, tokval, spos, epos, _ in tokenize.generate_tokens(StringIO.StringIO(unicode(ref)).readline): |
|
239 if toknum == ENDMARKER: |
|
240 break |
|
241 elif toknum == NAME: |
|
242 newref += str(tokval) |
|
243 elif toknum == NUMBER: |
|
244 # Add a character before the number token if the first token is a number |
|
245 if first_token: |
|
246 newref += '_' |
|
247 # replace a possible dot in number .123 |
|
248 newref += str(tokval.replace('.','_')) |
|
249 elif toknum == STRING: |
|
250 newref += str(tokval.replace('"', '')) |
|
251 else: |
|
252 newref += '_' |
|
253 # After first round set the first token to false |
|
254 first_token = False |
|
255 except tokenize.TokenError: |
|
256 pass |
|
257 return newref |
|
258 |
|
259 @classmethod |
|
260 def to_dref(cls, ref): |
|
261 """ |
|
262 Convert a resource ref to dotted ref |
|
263 1. remove file extension |
|
264 2. convert path delims to dots |
|
265 @return: a dotted reference. E.g. (foo/test.confml) => foo.test |
|
266 """ |
|
267 return dottedref.remove_begin_dot(cls.remove_ext(ref).replace('/','.')) |
|
268 |
|
269 @classmethod |
|
270 def to_hash(cls, ref): |
|
271 """ |
|
272 Convert a resource ref to to hash 32 bit integer |
|
273 @return: |
|
274 """ |
|
275 return "%s" % hex(hash(ref)) |
|
276 |
|
277 class dottedref(object): |
|
278 """ |
|
279 Class container for set of dotted reference related functions |
|
280 """ |
|
281 @classmethod |
|
282 def join_refs(cls, refs): |
|
283 """ |
|
284 join a list of dotted references together with dots |
|
285 1. ignore empty refs |
|
286 2. no dot include begin dot |
|
287 3. no dot include end dot |
|
288 @param refs: a list of references |
|
289 @return: A normalized dotted reference |
|
290 """ |
|
291 # Create a dotted reference without any empty strings |
|
292 return '.'.join([ref for ref in refs if ref.strip()]) |
|
293 |
|
294 @classmethod |
|
295 def split_ref(cls, ref): |
|
296 """ |
|
297 split a dotted references string to a list of ref elements |
|
298 1. empty ref returns an empty list |
|
299 @param ref: a dotted references string (e.g. aaa.bbb.ccc) |
|
300 @return: A list of references (with given param ['aaa','bbb','ccc'] |
|
301 """ |
|
302 # list of reference parts without any empty strings |
|
303 return [r for r in ref.split('.') if r] |
|
304 |
|
305 @classmethod |
|
306 def psplit_ref(cls, ref): |
|
307 """ |
|
308 pop split that splits the last element of the array |
|
309 1. empty ref returns an empty list |
|
310 @param ref: a dotted references string (e.g. aaa.bbb.ccc) |
|
311 @return: A tuple of references (with given param ('aaa.bbb','ccc') |
|
312 """ |
|
313 refs = ref.rsplit('.', 1) |
|
314 return ("".join(refs[0:-1]), refs[-1]) |
|
315 |
|
316 @classmethod |
|
317 def remove_last(cls, ref): |
|
318 """ |
|
319 removes the last element of the ref |
|
320 1. empty ref returns an empty list |
|
321 @param ref: a dotted references string (e.g. aaa.bbb.ccc) |
|
322 @return: A reference (with given param ('aaa.bbb') |
|
323 """ |
|
324 return ref.rsplit('.', 1)[0] |
|
325 |
|
326 @classmethod |
|
327 def get_last(cls, ref): |
|
328 """ |
|
329 returns the last element of the ref |
|
330 1. empty ref returns an empty string |
|
331 @param ref: a dotted references string (e.g. aaa.bbb.ccc) |
|
332 @return: A reference (with given param ('ccc') |
|
333 """ |
|
334 return ref.rsplit('.', 1)[-1] |
|
335 |
|
336 @classmethod |
|
337 def get_name(cls, ref): |
|
338 """ |
|
339 returns the last element of the ref |
|
340 1. empty ref returns an empty string |
|
341 @param ref: a dotted references string (e.g. aaa.bbb.ccc) |
|
342 @return: A reference (with given param ('ccc') |
|
343 """ |
|
344 if re.match('^(.*)\[.*\]$', ref): |
|
345 return re.match('^(.*)\[.*\]$', ref).group(1) |
|
346 else: |
|
347 return ref |
|
348 |
|
349 @classmethod |
|
350 def get_index(cls, ref): |
|
351 """ |
|
352 returns the last element of the ref |
|
353 1. empty ref returns an empty string |
|
354 @param ref: a dotted references string (e.g. aaa.bbb.ccc) |
|
355 @return: A reference (with given param ('ccc') |
|
356 """ |
|
357 if re.match('^.*\[(\d+)\]$', ref): |
|
358 return int( re.match('^.*\[(\d+)\]$', ref).group(1) ) |
|
359 else: |
|
360 return None |
|
361 |
|
362 @classmethod |
|
363 def remove_begin_dot(cls, ref): |
|
364 """ |
|
365 removes all the dots from the begin of the ref |
|
366 @param ref: a dotted references string (e.g. .aaa.bbb.ccc) |
|
367 @return: A reference (with given param ('aaa.bbb.ccc') |
|
368 """ |
|
369 return ref.lstrip('.') |
|
370 |
|
371 @classmethod |
|
372 def ref2filter(cls, ref): |
|
373 elems = [] |
|
374 for refelem in dottedref.split_ref(ref): |
|
375 if refelem == "**": |
|
376 elems.append(".*") |
|
377 else: |
|
378 elems.append(refelem.replace("*","[^\.]*")) |
|
379 return "\\.".join(elems)+"$" |
|
380 |
|
381 def extract_delimited_tokens(string, delimiters=('${', '}')): |
|
382 """ |
|
383 Return a list of all tokens delimited by the given strings in the given string. |
|
384 This function returns basically the first row of the result of |
|
385 extract_delimited_token_tuples(), with duplicates removed. |
|
386 |
|
387 >>> dottedref.extract_refs("test1 ${my.ref1} test2 ${ my.ref1 } ${my.ref2}") |
|
388 ['my.ref1', 'my.ref2'] |
|
389 """ |
|
390 ref_tuples = extract_delimited_token_tuples(string, delimiters) |
|
391 return distinct_array([ref for ref, raw_ref in ref_tuples]) |
|
392 |
|
393 def extract_delimited_token_tuples(string, delimiters=('${', '}')): |
|
394 """ |
|
395 Extract a list of (token, raw_token) tuples from the given string. |
|
396 'token' is the the token extracted from the string and trimmed (surrounding |
|
397 whitespace removed), and raw_token is the unmodified match from the |
|
398 string, which can be used for replacing. |
|
399 |
|
400 >>> dottedref.extract_ref_tuples("test1 ${my.ref1} test2 ${ my.ref1 } ${my.ref2}") |
|
401 [('my.ref1', '${my.ref1}'), ('my.ref1', '${ my.ref1 }'), ('my.ref2', '${my.ref2}')] |
|
402 """ |
|
403 pattern = '%s.*?%s' % (re.escape(delimiters[0]), re.escape(delimiters[1])) |
|
404 matches = distinct_array(re.findall(pattern, string, re.DOTALL)) |
|
405 |
|
406 result = [] |
|
407 for match in matches: |
|
408 ref = match[len(delimiters[0]):-len(delimiters[1])].strip() |
|
409 result.append((ref, match)) |
|
410 return result |
|
411 |
|
412 def expand_delimited_tokens(string, expander_func, delimiters=('${', '}')): |
|
413 """ |
|
414 Expand all tokens in the given string using the given expander function. |
|
415 |
|
416 @param string: The string to expand. |
|
417 @param expander_func: The function used for expanding. Should take two parameters: |
|
418 1 - The token to expand. |
|
419 2 - The index of the token in the string. |
|
420 @param delimiters: Tuple specifying the delimiters for tokens. |
|
421 @return: The expanded string. |
|
422 """ |
|
423 # Collect a dictionary of token-entry pairs |
|
424 class Entry(object): |
|
425 pass |
|
426 tokens = {} |
|
427 for index, (token, raw_token) in enumerate(extract_delimited_token_tuples(string, delimiters)): |
|
428 if token not in tokens: |
|
429 entry = Entry() |
|
430 entry.index = index |
|
431 entry.raw_tokens = [] |
|
432 entry.value = unicode(expander_func(token, index)) |
|
433 tokens[token] = entry |
|
434 else: |
|
435 entry = tokens[token] |
|
436 |
|
437 entry.raw_tokens.append(raw_token) |
|
438 |
|
439 # Replace all tokens with the expanded values |
|
440 result = string |
|
441 for entry in tokens.itervalues(): |
|
442 for raw_token in entry.raw_tokens: |
|
443 result = result.replace(raw_token, entry.value) |
|
444 return result |
|
445 |
|
446 def expand_refs_by_default_view(string, default_view, delimiters=('${', '}'), default_value_for_missing=''): |
|
447 """ |
|
448 Convenience function for expanding the refs in a string using setting values. |
|
449 @param default_value_for_missing: The default value used if a setting for |
|
450 a reference cannot be found. |
|
451 @return: The expanded string. |
|
452 """ |
|
453 def expand(ref, index): |
|
454 try: |
|
455 return default_view.get_feature(ref).get_original_value() |
|
456 except exceptions.NotFound: |
|
457 logging.getLogger('cone').error("Feature '%s' not found" % ref) |
|
458 return default_value_for_missing |
|
459 return expand_delimited_tokens(string, expand, delimiters) |
|
460 |
|
461 def distinct_array(arr): |
|
462 newarray = [] |
|
463 for val in arr: |
|
464 try: |
|
465 # test to see whether the value already is in thearray |
|
466 newarray.index(val) |
|
467 except ValueError: |
|
468 newarray.append(val) |
|
469 return newarray |
|
470 |
|
471 |
|
472 def list_files(path): |
|
473 """ |
|
474 Get an array of files in a folder |
|
475 """ |
|
476 retarray = [] |
|
477 # Walk through all files in the layer |
|
478 path = os.path.abspath(path) |
|
479 for root, dirs, files in os.walk(path): |
|
480 for name in files: |
|
481 entry = os.path.join(root, name) |
|
482 entry = os.path.normpath(os.path.abspath(entry)) |
|
483 if os.path.isfile(entry): |
|
484 retarray.append(entry) |
|
485 return retarray |
|
486 |
|
487 def all_subclasses(classname): |
|
488 """ |
|
489 @return: A list of all subclasses of classname |
|
490 """ |
|
491 subclasses = classname.__subclasses__() |
|
492 # Create copy of the subclasses list for the iteration, |
|
493 # so that added items are not recursed again |
|
494 for subclass in classname.__subclasses__(): |
|
495 subclasses += all_subclasses(subclass) |
|
496 return subclasses |
|
497 |
|
498 def pathmatch(pattern, refpath): |
|
499 """ |
|
500 Check for matching pattern for a ref path |
|
501 """ |
|
502 filter = dottedref.ref2filter(pattern) |
|
503 return re.match(filter, refpath) != None |
|
504 |
|
505 def filter(obj, filters): |
|
506 for filter in filters: |
|
507 if not filter(obj): |
|
508 return False |
|
509 return True |
|
510 |
|
511 def get_list(elem): |
|
512 if not isinstance(elem, list): |
|
513 return [elem] |
|
514 else: |
|
515 return elem |
|
516 |
|
517 def add_list(elem, add): |
|
518 retlist = get_list(elem) |
|
519 retlist.append(add) |
|
520 return retlist |
|
521 |
|
522 def prepend_list(elem, prepend): |
|
523 retlist = get_list(elem) |
|
524 retlist.insert(0, prepend) |
|
525 return retlist |
|
526 |
|
527 def is_list(elem): |
|
528 return isinstance(elem, list) |
|
529 |
|
530 def get_class(modelinstance, classinstance): |
|
531 """ |
|
532 Get the actual model specific implementation class for a classinstance |
|
533 """ |
|
534 for attr in dir(modelinstance): |
|
535 modelclass = getattr(modelinstance, attr) |
|
536 if inspect.isclass(modelclass): |
|
537 if issubclass(modelclass, classinstance): |
|
538 return modelclass |
|
539 return classinstance |
|
540 |
|
541 class DataMapRef(object): |
|
542 """ |
|
543 Utility class for handling map attributes in data section |
|
544 """ |
|
545 @classmethod |
|
546 def get_feature_ref(cls, map): |
|
547 index = map.find("@") |
|
548 if index != -1: |
|
549 parts = map.split("@") |
|
550 return parts[0][:-1] |
|
551 else: |
|
552 return None |
|
553 |
|
554 @classmethod |
|
555 def get_key_value(cls, map): |
|
556 index = map.find("@") |
|
557 if index != -1: |
|
558 parts = map.split("@") |
|
559 key = parts[1][:-1] |
|
560 keys = key.split("=") |
|
561 value = keys[1].strip() |
|
562 return value[1:-1] |
|
563 else: |
|
564 return None |
|
565 |
|
566 |
|
567 class xml(object): |
|
568 """ |
|
569 Class container for set of XML-related helper functions. |
|
570 """ |
|
571 |
|
572 @classmethod |
|
573 def split_tag_namespace(cls, xml_tag): |
|
574 """ |
|
575 Split the given XML tag into a (namespace, tag) tuple. |
|
576 |
|
577 >>> ReaderBase._split_tag_namespace("test") |
|
578 (None, 'test') |
|
579 >>> ReaderBase._split_tag_namespace("{http://www.test.com/xml/1}test") |
|
580 ('http://www.test.com/xml/1', 'test') |
|
581 """ |
|
582 if xml_tag.startswith('{'): |
|
583 parts = xml_tag[1:].split('}') |
|
584 return (parts[0], parts[1]) |
|
585 else: |
|
586 return (None, xml_tag) |
|
587 |
|
588 def log_exception(logger, msg, msg_level=logging.ERROR, traceback_level=logging.DEBUG): |
|
589 """ |
|
590 Log an exception so that the given message and the exception's |
|
591 traceback are logged separately with the given log levels. |
|
592 |
|
593 The purpose is to print minimal information to the user when running |
|
594 the CLI (default level for STDOUT logging is WARNING), but the traceback |
|
595 should still be available in the log file (which uses the level DEBUG |
|
596 by default). |
|
597 |
|
598 Note that this function should be only used in an exception handler. |
|
599 """ |
|
600 logger.log(msg_level, msg) |
|
601 logger.log(traceback_level, traceback.format_exc()) |
|
602 |
|
603 def make_content_info(resource, data): |
|
604 """ |
|
605 Factory for ContentInfo |
|
606 """ |
|
607 cnt_inf = None |
|
608 |
|
609 if resource != None: |
|
610 guessed_type = mimetypes.guess_type(resource.get_path()) |
|
611 mimetype = None |
|
612 mimesubtype = None |
|
613 |
|
614 if guessed_type != None: |
|
615 mimetype, mimesubtype = guessed_type[0].split('/') |
|
616 |
|
617 if mimetype == 'image' and mimesubtype == 'x-ms-bmp': |
|
618 cnt_inf = api.BmpImageContentInfo(resource, data) |
|
619 else: |
|
620 cnt_inf = api.ContentInfo(mimetype, mimesubtype) |
|
621 return cnt_inf |