|
1 # |
|
2 # Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
3 # All rights reserved. |
|
4 # This component and the accompanying materials are made available |
|
5 # under the terms of "Eclipse Public License v1.0" |
|
6 # which accompanies this distribution, and is available |
|
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Nokia Corporation - initial contribution. |
|
11 # |
|
12 # Contributors: |
|
13 # |
|
14 # Description: |
|
15 # |
|
16 |
|
17 import os |
|
18 import sys |
|
19 import logging |
|
20 from optparse import OptionParser, OptionGroup |
|
21 import cone_common |
|
22 import time |
|
23 from os import path |
|
24 from cone.public import api, plugin, utils, exceptions |
|
25 import generation_report |
|
26 ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) |
|
27 |
|
28 VERSION = '1.0' |
|
29 |
|
30 |
|
31 def main(): |
|
32 parser = OptionParser(version="%%prog %s" % VERSION) |
|
33 |
|
34 parser.add_options(cone_common.COMMON_OPTIONS) |
|
35 |
|
36 parser.add_option("-c", "--configuration",\ |
|
37 dest="configuration",\ |
|
38 help="defines the name of the configuration for the action",\ |
|
39 metavar="CONFIG") |
|
40 |
|
41 parser.add_option("-p", "--project",\ |
|
42 dest="project",\ |
|
43 help="defines the location of current project. Default is the current working directory.",\ |
|
44 default=".",\ |
|
45 metavar="STORAGE") |
|
46 |
|
47 gen_group = OptionGroup(parser, 'Generate options', |
|
48 'The generate function will create target files from a specific configuration.'\ |
|
49 'The generate will always work with read-only mode of the project, so no changes are saved to project') |
|
50 |
|
51 gen_group.add_option("-o", "--output",\ |
|
52 dest="output",\ |
|
53 help="defines the target folder where the files are is generated or copied",\ |
|
54 metavar="FOLDER",\ |
|
55 default="output") |
|
56 |
|
57 gen_group.add_option("-l", "--layer",\ |
|
58 dest="layers",\ |
|
59 type="int", |
|
60 action="append", |
|
61 help="define layers of the configuration that are included to the output. "\ |
|
62 "The layer operation can be used several times in a single command."\ |
|
63 "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2", |
|
64 metavar="LAYER",\ |
|
65 default=None) |
|
66 |
|
67 gen_group.add_option("--all-layers", |
|
68 dest="all_layers", |
|
69 action="store_true", |
|
70 help="Include all layers in generation. This switch overrides all other layer "\ |
|
71 "configurations (iMaker API and using the --layer parameter)", |
|
72 default=False) |
|
73 |
|
74 gen_group.add_option("-i", "--impl",\ |
|
75 dest="impls",\ |
|
76 action="append", |
|
77 help=\ |
|
78 """Define a Python regular expression filter for actual ImplML plugin(s) that needs to be executed. The whole path to ImplML filename is used in the regexp matching. |
|
79 The impl operation can be used several times in a single command. |
|
80 |
|
81 Example1 --impl crml => matches for any ImplML file that has a CrML string in the path. |
|
82 Example2 --impl makeml$ => matches for ImplML file that has ends with MakeML string. |
|
83 """, |
|
84 metavar="IMPLS",\ |
|
85 default=None) |
|
86 |
|
87 gen_group.add_option("--impl-tag",\ |
|
88 dest="tags",\ |
|
89 type="string", |
|
90 action="append", |
|
91 help="define a tag for the implementations that are included to the output. "\ |
|
92 "A tag is name value pair and has the following format: name:value, e.g. target:rofs3."\ |
|
93 "Example --impl-tag=target:uda --impl-tag=target:content, which would include impls include both tags.", |
|
94 metavar="TAG",\ |
|
95 default=None) |
|
96 |
|
97 gen_group.add_option("--impl-tag-policy",\ |
|
98 dest="tags_policy",\ |
|
99 type="string", |
|
100 action="append", |
|
101 help="Policy for implementation tags. May have one of the following values: --impl-tag-policy=AND, --impl-tag-policy=OR. "\ |
|
102 "Default is OR.", |
|
103 metavar="TAGS_POLICY",\ |
|
104 default=None) |
|
105 |
|
106 gen_group.add_option("-s", "--set",\ |
|
107 dest="overrides",\ |
|
108 action="append", |
|
109 type="string", |
|
110 help="Override a ConfML reference in the execution."\ |
|
111 "The set operation can be used several times in a single command."\ |
|
112 "Example -s foo.bar=10 -s foo.fea='test'.", |
|
113 metavar="SET",\ |
|
114 default=None) |
|
115 |
|
116 gen_group.add_option("--add",\ |
|
117 dest="added",\ |
|
118 action="append", |
|
119 type="string", |
|
120 help="Add a given configuration to the given configuration as last element."\ |
|
121 "The add operation can be used several times in a single command."\ |
|
122 "Example --add foo/root.confml --add bar/root-confml.", |
|
123 metavar="CONF",\ |
|
124 default=None) |
|
125 |
|
126 gen_group.add_option("-r", "--report",\ |
|
127 dest="report",\ |
|
128 action="store", |
|
129 type="string", |
|
130 help="Generates a report about settings that are properly generated."\ |
|
131 "Example -r report.html.", |
|
132 metavar="FILE",\ |
|
133 default=None) |
|
134 |
|
135 gen_group.add_option("-t", "--template",\ |
|
136 dest="template",\ |
|
137 action="store", |
|
138 type="string", |
|
139 help="Template used in report generation."\ |
|
140 "Example -t report_template.html.", |
|
141 metavar="FILE",\ |
|
142 default=None) |
|
143 |
|
144 gen_group.add_option("--report-data-output",\ |
|
145 type="string", |
|
146 help="Specifies a file where intermediary report data is generated.", |
|
147 metavar="FILE",\ |
|
148 default=None) |
|
149 |
|
150 gen_group.add_option("-n", "--dryrun",\ |
|
151 dest="dryrun",\ |
|
152 action="store_true", |
|
153 help="Executes generation without generation output.", |
|
154 default=False) |
|
155 |
|
156 gen_group.add_option("--add-setting-file",\ |
|
157 dest="settings",\ |
|
158 action="append", |
|
159 type="string", |
|
160 help="Generate specific settings in ini format."\ |
|
161 "Example -o my_generate_settings.cfg.", |
|
162 metavar="FILE",\ |
|
163 default=None) |
|
164 |
|
165 layers = None |
|
166 current = None |
|
167 remote = None |
|
168 |
|
169 start_time = time.time() |
|
170 |
|
171 parser.add_option_group(gen_group) |
|
172 (options, args) = parser.parse_args() |
|
173 |
|
174 settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')] |
|
175 if options.settings: |
|
176 for setting_file in options.settings: |
|
177 settinglist.append(os.path.normpath(os.path.join(ROOT_PATH, setting_file))) |
|
178 gset = cone_common.get_settings(settinglist) |
|
179 |
|
180 cone_common.handle_common_options(options, settings=gset) |
|
181 |
|
182 current = api.Project(api.Storage.open(options.project,"r")) |
|
183 active_root = current.get_storage().get_active_configuration() |
|
184 if not options.configuration: |
|
185 if active_root == "": |
|
186 parser.error("configuration must be given") |
|
187 else: |
|
188 logging.getLogger('cone').info('No configuration given! Using active root configuration %s' % active_root) |
|
189 options.configuration = active_root |
|
190 try: |
|
191 config = current.get_configuration(options.configuration) |
|
192 except exceptions.NotFound: |
|
193 parser.error("No such configuration: %s" % options.configuration) |
|
194 reffilters = None |
|
195 implfilters = None |
|
196 impltags = None |
|
197 |
|
198 # Include possible additional configurations |
|
199 if options.added: |
|
200 for configname in options.added: |
|
201 logging.getLogger('cone').info('Adding configuration %s' % configname) |
|
202 config.include_configuration(utils.resourceref.norm(configname)) |
|
203 |
|
204 # Get defs from configuration |
|
205 try: |
|
206 layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',') |
|
207 # Make sure that empty layers definitions are ignored |
|
208 layer_str_list = utils.distinct_array(layer_str_list) |
|
209 if '' in layer_str_list: |
|
210 layer_str_list.remove('') |
|
211 # converting layrs identifiers from strings to int |
|
212 layerdefs = [] |
|
213 for layerstr in layer_str_list: |
|
214 try: |
|
215 layerdefs.append(int(layerstr)) |
|
216 except ValueError, e: |
|
217 logging.getLogger('cone').error('Invalid layer filter %s' % layerstr) |
|
218 implfilters = (config.get_default_view().get_feature('imakerapi.cone_impls').get_value() or '').split(',') |
|
219 except exceptions.NotFound: |
|
220 layerdefs = [] |
|
221 implfilters = [] |
|
222 pass |
|
223 |
|
224 # Get filters from command line if they exist => cmd overrides configuration |
|
225 if options.layers: |
|
226 layerdefs = options.layers |
|
227 if options.impls: |
|
228 implfilters = options.impls |
|
229 if options.tags and len(options.tags) > 0: |
|
230 impltags = {} |
|
231 for tag in options.tags: |
|
232 (name,value) = tag.split(':',2) |
|
233 existingvalue = impltags.get(name,[]) |
|
234 existingvalue.append(value) |
|
235 impltags[name] = existingvalue |
|
236 logging.getLogger('cone').info('Tag filter %s' % impltags) |
|
237 else: |
|
238 impltags = None |
|
239 |
|
240 tags_policy = 'OR' |
|
241 if options.tags_policy: |
|
242 tags_policy = options.tags_policy[0] |
|
243 |
|
244 # Finally, --all-layers overrides all other layer settings |
|
245 if options.all_layers: |
|
246 layerdefs = [] |
|
247 |
|
248 logging.getLogger('cone').info('Layer filter %s' % layerdefs) |
|
249 |
|
250 # Add reffilters only if the given layerids are somehow reasonable |
|
251 if len(layerdefs) > 0: |
|
252 # get the data references from given layers |
|
253 logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs) |
|
254 reffilters = [] |
|
255 for layerid in utils.distinct_array(layerdefs): |
|
256 logging.getLogger('cone').info('Searching layer %s' % layerid) |
|
257 layer = config.get_configuration_by_index(layerid) |
|
258 refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
|
259 logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs))) |
|
260 reffilters += refs |
|
261 |
|
262 |
|
263 if options.overrides: |
|
264 config.add_configuration(api.Configuration('tempdata.confml')) |
|
265 for override in options.overrides: |
|
266 (ref,value) = override.split('=',1) |
|
267 config.get_default_view().get_feature(ref).set_value(value) |
|
268 |
|
269 # Make sure that the output folder exists |
|
270 if not os.path.exists(options.output): |
|
271 os.makedirs(options.output) |
|
272 |
|
273 impls = plugin.filtered_impl_set(config,implfilters) |
|
274 impls.output = options.output |
|
275 |
|
276 logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions()) |
|
277 |
|
278 # logging.getLogger('cone').debug('Loaded implementations:') |
|
279 # for impl in impls: |
|
280 # msg = "File '%s', impl. type '%s', class '%s', phase '%s'" % \ |
|
281 # (impl.ref, impl.IMPL_TYPE_ID, type(impl).__name__, impl.invocation_phase()) |
|
282 # logging.getLogger('cone').debug(msg) |
|
283 |
|
284 |
|
285 # Create temporary variables |
|
286 temp_feature_refs = impls.create_temp_features(config) |
|
287 if reffilters is not None: |
|
288 reffilters.extend(temp_feature_refs) |
|
289 logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs)) |
|
290 |
|
291 |
|
292 |
|
293 # --------------- |
|
294 # Generate output |
|
295 # --------------- |
|
296 |
|
297 rule_exec_results = [] |
|
298 |
|
299 # Create an implementation container with all the relevant implementations |
|
300 all_impls = impls.filter_implementations(tags=impltags, policy=tags_policy) |
|
301 |
|
302 # Implementations taking part in output generation |
|
303 gen_impls = plugin.ImplSet() |
|
304 context = plugin.GenerationContext() |
|
305 context.configuration = config |
|
306 log = logging.getLogger('cone') |
|
307 for phase in plugin.ImplSet.INVOCATION_PHASES: |
|
308 phase_impls = all_impls.filter_implementations(phase=phase) |
|
309 log.info("Generating phase '%s', %d implementation(s)" % (phase, len(phase_impls))) |
|
310 |
|
311 context.phase = phase |
|
312 # No use going any further if there are no implementations |
|
313 # for the phase at all |
|
314 if len(phase_impls) == 0: |
|
315 continue |
|
316 |
|
317 # Load and execute rules for this phase |
|
318 # ------------------------------------- |
|
319 # relation_container = phase_impls.get_relation_container() |
|
320 # log.info("%d rule(s) for phase '%s'" % (relation_container.get_relation_count(), phase)) |
|
321 # if relation_container.get_relation_count() > 0: |
|
322 # log.info("Executing rules...") |
|
323 # results = relation_container.execute() |
|
324 # log.info("Got %d execution result(s)" % len(results)) |
|
325 # rule_exec_results.extend(results) |
|
326 |
|
327 |
|
328 # Create an implementation container for the phase with |
|
329 # the new reffilters and generate output with it |
|
330 # ----------------------------------------------------- |
|
331 impls = phase_impls.filter_implementations(refs=reffilters) |
|
332 log.info("%d implementation(s) after filtering for phase '%s'" % (len(impls), phase)) |
|
333 if len(impls) > 0: |
|
334 if impltags != None: |
|
335 context.tags = impltags |
|
336 context.tags_policy = tags_policy |
|
337 impls.output = options.output |
|
338 log.info("Generating output...") |
|
339 impls.generate(context) |
|
340 impls.post_generate(context) |
|
341 |
|
342 # Add new refs after generation |
|
343 # if reffilters != None and len(reffilters) > 0: |
|
344 # layer = config.get_configuration_by_index(-1) |
|
345 # new_refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
|
346 # log.info('Added %d ref(s) after generation:\n%s' % (len(new_refs), '\n'.join(new_refs))) |
|
347 # reffilters += new_refs |
|
348 |
|
349 # Add new references after each phase execution |
|
350 # --------------------------------------- |
|
351 if reffilters != None and len(reffilters) > 0: |
|
352 layer = config.get_configuration_by_index(-1) |
|
353 new_refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
|
354 log.info('Added %d ref(s) after phase %s execution:\n%s' % (len(new_refs), phase, '\n'.join(new_refs))) |
|
355 reffilters += new_refs |
|
356 |
|
357 # Add the implementations to the set of implementations participating |
|
358 # in output generation (used in the report) |
|
359 for impl in impls: |
|
360 for actual_impl in impl.get_all_implementations(): |
|
361 logging.getLogger('cone').info('Adding impl %s' % impl) |
|
362 gen_impls.add(actual_impl) |
|
363 |
|
364 rule_exec_results = context.results |
|
365 print "Generated %s to %s!" % (options.configuration, impls.output) |
|
366 |
|
367 |
|
368 # --------------- |
|
369 # Generate report |
|
370 # --------------- |
|
371 |
|
372 # If reporting is enabled collect data for report |
|
373 if options.report != None or options.report_data_output != None: |
|
374 logging.getLogger('cone').info('Collecting data for report.') |
|
375 all_refs = reffilters or utils.distinct_array(config.get_configuration_by_index(-1).list_leaf_datas()) |
|
376 logging.getLogger('cone').info('Collecting data found refs %s' % all_refs) |
|
377 logging.getLogger('cone').info('Collecting data found gen_impls %s' % gen_impls) |
|
378 rep_data = generation_report.collect_report_data(config, options, all_refs, gen_impls, rule_exec_results) |
|
379 logging.getLogger('cone').info('Collecting data found rep_data %s' % rep_data) |
|
380 |
|
381 duration = str("%.3f" % (time.time() - start_time) ) |
|
382 rep_data.set_duration( duration ) |
|
383 |
|
384 # Save intermediary report data file if necessary |
|
385 if options.report_data_output != None: |
|
386 logging.getLogger('cone').info('Dumping report data to %s' % options.report_data_output) |
|
387 print "Dumping report data to '%s'" % options.report_data_output |
|
388 generation_report.save_report_data(rep_data, options.report_data_output) |
|
389 |
|
390 # Generate the report if necessary |
|
391 if options.report != None: |
|
392 generation_report.generate_report(rep_data, options.report, options.template) |
|
393 print_summary(rep_data) |
|
394 |
|
395 if current: current.close() |
|
396 |
|
397 def _get_new_refs(old_refs, new_refs): |
|
398 """ |
|
399 Return a distinct array of refs in ``new_refs`` that are not present in ``old_refs``. |
|
400 """ |
|
401 result = [] |
|
402 for ref in new_refs: |
|
403 if ref not in old_refs and ref not in result: |
|
404 result.append(ref) |
|
405 return result |
|
406 |
|
407 def print_summary(rep_data): |
|
408 """ Prints generation summary to logger and console. """ |
|
409 print "\nGENERATION SUMMARY:" |
|
410 print "--------------------" |
|
411 print "Refs in files: %s" % rep_data.nbr_of_refs |
|
412 print "Refs with no implementation: %s" % rep_data.nbr_of_refs_noimpl |
|
413 print "Generation duration: %s" % rep_data.duration |
|
414 print "\n\n" |
|
415 |
|
416 |
|
417 if __name__ == "__main__": |
|
418 main() |