159 type="string", |
154 type="string", |
160 help="Generate specific settings in ini format."\ |
155 help="Generate specific settings in ini format."\ |
161 "Example -o my_generate_settings.cfg.", |
156 "Example -o my_generate_settings.cfg.", |
162 metavar="FILE",\ |
157 metavar="FILE",\ |
163 default=None) |
158 default=None) |
164 |
159 gen_group.add_option("--dump-autodata",\ |
165 layers = None |
160 dest="dump_autodata",\ |
166 current = None |
161 action="store", |
167 remote = None |
162 type="string", |
|
163 metavar="FILE", |
|
164 help="Specifies a confml file for storing autodata.confml permanently.", |
|
165 default=None) |
|
166 gen_group.add_option("-w", "--what",\ |
|
167 dest="what",\ |
|
168 action="store", |
|
169 type="string", |
|
170 metavar="FILE", |
|
171 help="List output files to a txt file", |
|
172 default=None) |
|
173 |
|
174 lf_group = OptionGroup(parser, 'Layer filtering options', |
|
175 'Layer filtering options define configuration layers to be used for filtering '\ |
|
176 'the implementations that are used to generate output. Filtering by a layer means that '\ |
|
177 'only implementations that generate their output based on settings changed on that layer '\ |
|
178 'are included in the generation.') |
|
179 |
|
180 lf_group.add_option("-l", "--layer",\ |
|
181 dest="layers",\ |
|
182 type="int", |
|
183 action="append", |
|
184 help="Define a layer by giving its index in the root configuration. "\ |
|
185 "0 is first, 1 the second, -1 the last, -2 the second to last and so on. "\ |
|
186 "The layer operation can be used several times in a single command. "\ |
|
187 "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2", |
|
188 metavar="LAYER",\ |
|
189 default=None) |
|
190 |
|
191 lf_group.add_option("--layer-regex", |
|
192 dest="layer_regexes", |
|
193 action="append", |
|
194 help="Define a regular expression for including layers into the generation process, "\ |
|
195 "e.g. --layer-regex layer[0-9]/root.confml. The pattern is matched against the layer root "\ |
|
196 "path, which could be e.g. 'assets/layer1/root.confml'.", |
|
197 metavar="REGEX",) |
|
198 |
|
199 lf_group.add_option("--layer-wildcard", |
|
200 dest="layer_wildcards", |
|
201 action="append", |
|
202 help="Define a wildcard for including layers into the generation process, e.g "\ |
|
203 "--layer-wildcard layer*", |
|
204 metavar="WILDCARD",) |
|
205 |
|
206 lf_group.add_option("--all-layers", |
|
207 dest="all_layers", |
|
208 action="store_true", |
|
209 help="Include all layers in generation. This switch overrides all other layer "\ |
|
210 "configurations (iMaker API and using the --layer, --layer-regex and --layer-wildcard parameters)", |
|
211 default=False) |
|
212 |
168 |
213 |
169 start_time = time.time() |
214 start_time = time.time() |
170 |
215 |
171 parser.add_option_group(gen_group) |
216 parser.add_option_group(gen_group) |
172 (options, args) = parser.parse_args() |
217 parser.add_option_group(lf_group) |
|
218 (options, _) = parser.parse_args() |
173 |
219 |
174 settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')] |
220 settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')] |
175 if options.settings: |
221 if options.settings: |
176 for setting_file in options.settings: |
222 for setting_file in options.settings: |
177 settinglist.append(os.path.normpath(os.path.join(ROOT_PATH, setting_file))) |
223 settinglist.append(os.path.normpath(os.path.join(ROOT_PATH, setting_file))) |
239 |
269 |
240 tags_policy = 'OR' |
270 tags_policy = 'OR' |
241 if options.tags_policy: |
271 if options.tags_policy: |
242 tags_policy = options.tags_policy[0] |
272 tags_policy = options.tags_policy[0] |
243 |
273 |
244 # Finally, --all-layers overrides all other layer settings |
274 |
245 if options.all_layers: |
275 layerdefs = _get_included_layers(config, options, parser) |
246 layerdefs = [] |
276 filter_by_refs = _filter_by_refs(config, options, parser) |
247 |
277 |
248 logging.getLogger('cone').info('Layer filter %s' % layerdefs) |
278 if layerdefs: |
249 |
279 logging.getLogger('cone').info('Included layers:\n%s' % '\n'.join(layerdefs)) |
250 # Add reffilters only if the given layerids are somehow reasonable |
280 else: |
|
281 logging.getLogger('cone').info('Including all layers') |
|
282 |
|
283 dview = config.get_default_view() |
|
284 # Add data references if included layers are defined |
251 if len(layerdefs) > 0: |
285 if len(layerdefs) > 0: |
252 # get the data references from given layers |
286 # get the data references from given layers |
253 logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs) |
287 logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs) |
254 reffilters = [] |
288 reffilters = [] |
255 for layerid in utils.distinct_array(layerdefs): |
289 for layer_path in utils.distinct_array(layerdefs): |
256 logging.getLogger('cone').info('Searching layer %s' % layerid) |
290 logging.getLogger('cone').info('Searching layer %s' % layer_path) |
257 layer = config.get_configuration_by_index(layerid) |
291 layer = config.get_configuration(layer_path) |
258 refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
292 refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
259 logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs))) |
293 # reduce the refs of sequences to single reference of the sequence feature |
|
294 layerrefs = set() |
|
295 for fea in dview.get_features(refs): |
|
296 layerrefs.add(fea.fqr) |
|
297 if fea.is_sequence(): |
|
298 layerrefs.add(fea.get_sequence_parent().fqr) |
|
299 |
|
300 refs = sorted(list(layerrefs)) |
|
301 #logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs))) |
260 reffilters += refs |
302 reffilters += refs |
261 |
303 |
262 |
|
263 if options.overrides: |
|
264 config.add_configuration(api.Configuration('tempdata.confml')) |
|
265 for override in options.overrides: |
|
266 (ref,value) = override.split('=',1) |
|
267 config.get_default_view().get_feature(ref).set_value(value) |
|
268 |
|
269 # Make sure that the output folder exists |
304 # Make sure that the output folder exists |
270 if not os.path.exists(options.output): |
305 if not os.path.exists(options.output): |
271 os.makedirs(options.output) |
306 os.makedirs(options.output) |
272 |
307 |
273 impls = plugin.filtered_impl_set(config,implfilters) |
308 impls = plugin.filtered_impl_set(config,implfilters) |
274 impls.output = options.output |
309 impls.output = options.output |
|
310 |
|
311 log.info("Parsed %s implementation(s)" % len(impls)) |
275 |
312 |
276 logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions()) |
313 logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions()) |
277 |
314 |
278 # logging.getLogger('cone').debug('Loaded implementations:') |
315 # logging.getLogger('cone').debug('Loaded implementations:') |
279 # for impl in impls: |
316 # for impl in impls: |
282 # logging.getLogger('cone').debug(msg) |
319 # logging.getLogger('cone').debug(msg) |
283 |
320 |
284 |
321 |
285 # Create temporary variables |
322 # Create temporary variables |
286 temp_feature_refs = impls.create_temp_features(config) |
323 temp_feature_refs = impls.create_temp_features(config) |
|
324 |
287 if reffilters is not None: |
325 if reffilters is not None: |
288 reffilters.extend(temp_feature_refs) |
326 reffilters.extend(temp_feature_refs) |
289 logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs)) |
327 logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs)) |
290 |
328 |
|
329 # Set overrides only after temp variables are created, so that |
|
330 # they can also be modified from the command line |
|
331 if options.overrides: |
|
332 # Make sure that the last layer is the autodata layer |
|
333 plugin.get_autoconfig(config) |
|
334 for override in options.overrides: |
|
335 (ref,value) = override.split('=',1) |
|
336 config.get_default_view().get_feature(ref).set_value(value) |
291 |
337 |
292 |
338 |
293 # --------------- |
339 # --------------- |
294 # Generate output |
340 # Generate output |
295 # --------------- |
341 # --------------- |
296 |
342 |
297 rule_exec_results = [] |
343 context = plugin.GenerationContext(configuration = config, |
298 |
344 tags = impltags or {}, |
299 # Create an implementation container with all the relevant implementations |
345 tags_policy = tags_policy, |
300 all_impls = impls.filter_implementations(tags=impltags, policy=tags_policy) |
346 output = options.output, |
301 |
347 impl_set = impls, |
302 # Implementations taking part in output generation |
348 temp_features = temp_feature_refs, |
303 gen_impls = plugin.ImplSet() |
349 filter_by_refs = filter_by_refs) |
304 context = plugin.GenerationContext() |
350 context.changed_refs = reffilters |
305 context.configuration = config |
351 context.output = options.output |
306 log = logging.getLogger('cone') |
352 |
307 for phase in plugin.ImplSet.INVOCATION_PHASES: |
353 impls.output = options.output |
308 phase_impls = all_impls.filter_implementations(phase=phase) |
354 for phase in impls.INVOCATION_PHASES: |
309 log.info("Generating phase '%s', %d implementation(s)" % (phase, len(phase_impls))) |
355 log.info("Generating phase '%s'" % phase) |
310 |
|
311 context.phase = phase |
356 context.phase = phase |
312 # No use going any further if there are no implementations |
357 impls.generate(context) |
313 # for the phase at all |
358 impls.post_generate(context) |
314 if len(phase_impls) == 0: |
359 |
315 continue |
360 if options.what: |
316 |
361 log.info("Write output files to '%s'" % options.what) |
317 # Load and execute rules for this phase |
362 output_files = [] |
318 # ------------------------------------- |
363 for op in context.get_output(): |
319 # relation_container = phase_impls.get_relation_container() |
364 # Only append once |
320 # log.info("%d rule(s) for phase '%s'" % (relation_container.get_relation_count(), phase)) |
365 if op.type == 'file' and output_files.count(op.abspath) < 1: |
321 # if relation_container.get_relation_count() > 0: |
366 output_files.append(op.abspath) |
322 # log.info("Executing rules...") |
367 try: |
323 # results = relation_container.execute() |
368 mkpath(os.path.dirname(os.path.abspath(options.what))) |
324 # log.info("Got %d execution result(s)" % len(results)) |
369 what_fh = open(os.path.abspath(options.what), 'w') |
325 # rule_exec_results.extend(results) |
370 try: |
326 |
371 [what_fh.write('%s\n' % ofile) for ofile in output_files] |
327 |
372 print "Wrote output file list to '%s'" % options.what |
328 # Create an implementation container for the phase with |
373 finally: |
329 # the new reffilters and generate output with it |
374 what_fh.close() |
330 # ----------------------------------------------------- |
375 except Exception: |
331 impls = phase_impls.filter_implementations(refs=reffilters) |
376 log.info("Could not create directory for '%s'" % options.what) |
332 log.info("%d implementation(s) after filtering for phase '%s'" % (len(impls), phase)) |
377 |
333 if len(impls) > 0: |
|
334 if impltags != None: |
|
335 context.tags = impltags |
|
336 context.tags_policy = tags_policy |
|
337 impls.output = options.output |
|
338 log.info("Generating output...") |
|
339 impls.generate(context) |
|
340 impls.post_generate(context) |
|
341 |
|
342 # Add new refs after generation |
|
343 # if reffilters != None and len(reffilters) > 0: |
|
344 # layer = config.get_configuration_by_index(-1) |
|
345 # new_refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
|
346 # log.info('Added %d ref(s) after generation:\n%s' % (len(new_refs), '\n'.join(new_refs))) |
|
347 # reffilters += new_refs |
|
348 |
|
349 # Add new references after each phase execution |
|
350 # --------------------------------------- |
|
351 if reffilters != None and len(reffilters) > 0: |
|
352 layer = config.get_configuration_by_index(-1) |
|
353 new_refs = _get_new_refs(reffilters, layer.list_leaf_datas()) |
|
354 log.info('Added %d ref(s) after phase %s execution:\n%s' % (len(new_refs), phase, '\n'.join(new_refs))) |
|
355 reffilters += new_refs |
|
356 |
|
357 # Add the implementations to the set of implementations participating |
|
358 # in output generation (used in the report) |
|
359 for impl in impls: |
|
360 for actual_impl in impl.get_all_implementations(): |
|
361 logging.getLogger('cone').info('Adding impl %s' % impl) |
|
362 gen_impls.add(actual_impl) |
|
363 |
|
364 rule_exec_results = context.results |
|
365 print "Generated %s to %s!" % (options.configuration, impls.output) |
378 print "Generated %s to %s!" % (options.configuration, impls.output) |
366 |
379 |
|
380 # Store temporary rule execution outputs to a new configuration |
|
381 if options.dump_autodata: |
|
382 # Make sure autodata layer is the one we're dealing with |
|
383 plugin.get_autoconfig(config) |
|
384 lastconfig = config.get_last_configuration() |
|
385 lastconfig.set_name(utils.resourceref.to_objref(utils.resourceref.get_filename(utils.resourceref.norm(options.dump_autodata)))) |
|
386 data = persistentconfml.dumps(lastconfig) |
|
387 try: |
|
388 mkpath(utils.resourceref.get_path(utils.resourceref.norm(options.dump_autodata))) |
|
389 fh = open(options.dump_autodata, 'w') |
|
390 try: fh.write(data) |
|
391 finally: fh.close() |
|
392 print 'Saved autodata to %s' % options.dump_autodata |
|
393 except DistutilsFileError: |
|
394 log.info('Unable to dump autodata') |
|
395 |
367 |
396 |
368 # --------------- |
397 # --------------- |
369 # Generate report |
398 # Generate report |
370 # --------------- |
399 # --------------- |
371 |
400 |
372 # If reporting is enabled collect data for report |
401 # If reporting is enabled collect data for report |
373 if options.report != None or options.report_data_output != None: |
402 if options.report != None or options.report_data_output != None: |
374 logging.getLogger('cone').info('Collecting data for report.') |
403 logging.getLogger('cone').info('Collecting data for report.') |
375 all_refs = reffilters or utils.distinct_array(config.get_configuration_by_index(-1).list_leaf_datas()) |
404 |
376 logging.getLogger('cone').info('Collecting data found refs %s' % all_refs) |
405 rep_data = generation_report.ReportData() |
377 logging.getLogger('cone').info('Collecting data found gen_impls %s' % gen_impls) |
406 rep_data.context = context |
378 rep_data = generation_report.collect_report_data(config, options, all_refs, gen_impls, rule_exec_results) |
407 rep_data.context.log_file = os.path.abspath(options.log_file) |
|
408 rep_data.context.log = _read_log(options.log_file) |
|
409 rep_data.project_dir = options.project |
379 logging.getLogger('cone').info('Collecting data found rep_data %s' % rep_data) |
410 logging.getLogger('cone').info('Collecting data found rep_data %s' % rep_data) |
380 |
411 |
381 duration = str("%.3f" % (time.time() - start_time) ) |
412 duration = str("%.3f" % (time.time() - start_time) ) |
382 rep_data.set_duration( duration ) |
413 rep_data.set_duration( duration ) |
|
414 rep_data.options = options |
383 |
415 |
384 # Save intermediary report data file if necessary |
416 # Save intermediary report data file if necessary |
385 if options.report_data_output != None: |
417 if options.report_data_output != None: |
386 logging.getLogger('cone').info('Dumping report data to %s' % options.report_data_output) |
418 logging.getLogger('cone').info('Dumping report data to %s' % options.report_data_output) |
387 print "Dumping report data to '%s'" % options.report_data_output |
419 print "Dumping report data to '%s'" % options.report_data_output |
388 generation_report.save_report_data(rep_data, options.report_data_output) |
420 generation_report.save_report_data(rep_data, options.report_data_output) |
389 |
421 |
390 # Generate the report if necessary |
422 # Generate the report if necessary |
391 if options.report != None: |
423 if options.report != None: |
392 generation_report.generate_report(rep_data, options.report, options.template) |
424 generation_report.generate_report([rep_data], options.report, options.template, [ROOT_PATH], options.report_option) |
393 print_summary(rep_data) |
425 print_summary(rep_data) |
394 |
426 |
395 if current: current.close() |
427 if current: current.close() |
396 |
428 |
|
429 def _read_log(log_file): |
|
430 logf = open(log_file) |
|
431 # strip endlines |
|
432 return [line.strip('\n') for line in logf.readlines()] |
|
433 |
397 def _get_new_refs(old_refs, new_refs): |
434 def _get_new_refs(old_refs, new_refs): |
398 """ |
435 """ |
399 Return a distinct array of refs in ``new_refs`` that are not present in ``old_refs``. |
436 Return a distinct array of refs in ``new_refs`` that are not present in ``old_refs``. |
400 """ |
437 """ |
401 result = [] |
438 result = [] |
402 for ref in new_refs: |
439 for ref in new_refs: |
403 if ref not in old_refs and ref not in result: |
440 if ref not in old_refs and ref not in result: |
404 result.append(ref) |
441 result.append(ref) |
405 return result |
442 return result |
406 |
443 |
|
444 |
|
445 def _filter_by_refs(config, options, parser): |
|
446 """ |
|
447 """ |
|
448 filter_by_refs = True |
|
449 if options.all_layers: |
|
450 filter_by_refs = False |
|
451 elif not options.layers and not options.layer_regexes and not options.layer_wildcards: |
|
452 filter_by_refs = False |
|
453 return filter_by_refs |
|
454 |
|
455 def _get_included_layers(config, options, parser): |
|
456 """ |
|
457 Collect a list of included layer root paths from the config based on the |
|
458 given parameters in options. |
|
459 @return: A list of layer configuration paths (empty if all layers |
|
460 should be generated). |
|
461 """ |
|
462 # --all-layers overrides all other definitions |
|
463 if options.all_layers: |
|
464 options.layers = [i for i in range(len(config.list_configurations()))] |
|
465 elif not options.layers and not options.layer_regexes and not options.layer_wildcards: |
|
466 options.layers = [i for i in range(len(config.list_configurations()))] |
|
467 |
|
468 # Command line definitions override others |
|
469 if options.layers or options.layer_regexes or options.layer_wildcards: |
|
470 layer_paths = [] |
|
471 all_layers = config.list_configurations() |
|
472 |
|
473 for layer_index in options.layers or []: |
|
474 try: |
|
475 layer_paths.append(all_layers[int(layer_index)]) |
|
476 except (IndexError, ValueError): |
|
477 parser.error("Invalid layer index: %s" % layer_index) |
|
478 |
|
479 for regex in options.layer_regexes or []: |
|
480 for layer_path in all_layers: |
|
481 if re.search(regex, layer_path): |
|
482 layer_paths.append(layer_path) |
|
483 |
|
484 for wildcard in options.layer_wildcards or []: |
|
485 for layer_path in all_layers: |
|
486 if fnmatch.fnmatch(layer_path, wildcard): |
|
487 layer_paths.append(layer_path) |
|
488 |
|
489 if not layer_paths: |
|
490 parser.error('No layers matched by layer patterns') |
|
491 |
|
492 return utils.distinct_array(layer_paths) |
|
493 |
|
494 # Use iMaker API definitions if no others have been specified |
|
495 return _get_included_layers_from_imaker_api(config, parser) |
|
496 |
|
497 def _get_included_layers_from_imaker_api(config, parser): |
|
498 try: |
|
499 layer_str_list = (config.get_default_view().get_feature('imakerapi.cone_layers').get_value() or '').split(',') |
|
500 # Make sure that empty layers definitions are ignored |
|
501 layer_str_list = utils.distinct_array(layer_str_list) |
|
502 if '' in layer_str_list: |
|
503 layer_str_list.remove('') |
|
504 |
|
505 all_layers = config.list_configurations() |
|
506 layerdefs = [] |
|
507 for layerstr in layer_str_list: |
|
508 try: |
|
509 layerdefs.append(all_layers[int(layerstr)]) |
|
510 except (ValueError, IndexError): |
|
511 parser.error("Invalid layer index from iMaker API: %s" % layerstr) |
|
512 return layerdefs |
|
513 except exceptions.NotFound: |
|
514 return [] |
|
515 |
407 def print_summary(rep_data): |
516 def print_summary(rep_data): |
408 """ Prints generation summary to logger and console. """ |
517 """ Prints generation summary to logger and console. """ |
409 print "\nGENERATION SUMMARY:" |
518 print "\nGENERATION SUMMARY:" |
410 print "--------------------" |
519 print "--------------------" |
411 print "Refs in files: %s" % rep_data.nbr_of_refs |
520 print "Refs in files: %s" % len(rep_data.context.changed_refs) |
412 print "Refs with no implementation: %s" % rep_data.nbr_of_refs_noimpl |
521 print "Refs with no implementation: %s" % len(rep_data.context.get_refs_with_no_output()) |
413 print "Generation duration: %s" % rep_data.duration |
522 print "Generation duration: %s" % rep_data.duration |
414 print "\n\n" |
523 print "\n\n" |
415 |
524 |
416 |
525 |
417 if __name__ == "__main__": |
526 if __name__ == "__main__": |