|
1 # |
|
2 # Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
3 # All rights reserved. |
|
4 # This component and the accompanying materials are made available |
|
5 # under the terms of "Eclipse Public License v1.0" |
|
6 # which accompanies this distribution, and is available |
|
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Nokia Corporation - initial contribution. |
|
11 # |
|
12 # Contributors: |
|
13 # |
|
14 # Description: |
|
15 # |
|
16 |
|
17 import sys |
|
18 import logging |
|
19 from optparse import OptionParser, OptionGroup |
|
20 import cone_common |
|
21 |
|
22 from cone.public import api, utils, exceptions |
|
23 |
|
24 VERSION = '1.0' |
|
25 |
|
26 logger = logging.getLogger('cone') |
|
27 |
|
28 class MergeFailedException(Exception): |
|
29 """ |
|
30 Exception raised if the merge failed for some reason. |
|
31 """ |
|
32 pass |
|
33 |
|
34 class MergePolicy(object): |
|
35 """ |
|
36 Merge policy constants. |
|
37 """ |
|
38 #: Replace/add files from the source layer into the |
|
39 #: target layer, preserving files in the target layer |
|
40 #: that do not exist in the source layer |
|
41 REPLACE_ADD = 'replace-add' |
|
42 |
|
43 #: Overwrite the entire target layer, so that it contains |
|
44 #: only the contents of the source layer |
|
45 OVERWRITE_LAYER = 'overwrite-layer' |
|
46 |
|
47 ALL = (REPLACE_ADD, OVERWRITE_LAYER) |
|
48 |
|
49 @classmethod |
|
50 def is_valid(cls, policy): |
|
51 return policy in cls.ALL |
|
52 |
|
53 def get_new_layer_name(oldname,rootconfig): |
|
54 newpath = utils.resourceref.get_path(oldname) |
|
55 newpath+= "_" + utils.resourceref.remove_ext(utils.resourceref.get_filename(rootconfig)) |
|
56 newpath+= "/" + utils.resourceref.get_filename(oldname) |
|
57 return newpath |
|
58 |
|
59 def merge_configuration_layer(sourceconfig, targetconfig, merge_policy): |
|
60 """ |
|
61 Merge the contents of the source layer into the target layer. |
|
62 @param sourceconfig: Source layer root configuration object. |
|
63 @param targetconfig: Target layer root configuration object. |
|
64 @param merge_policy: The used layer merge policy. |
|
65 """ |
|
66 # If policy tells to entirely overwrite the layer, remove all |
|
67 # resources from the layer first |
|
68 if merge_policy == MergePolicy.OVERWRITE_LAYER: |
|
69 # Remove configurations from the layer root |
|
70 confs = targetconfig.list_configurations() |
|
71 for conf in confs: |
|
72 targetconfig.remove_configuration(conf) |
|
73 |
|
74 # Remove all related resources |
|
75 layerobj = targetconfig.get_layer() |
|
76 # Round one: remove all files |
|
77 resources = layerobj.list_all_related(empty_folders=False) |
|
78 for res in resources: layerobj.delete_resource(res) |
|
79 # Round two: remove any remaining empty directories |
|
80 resources = layerobj.list_all_related(empty_folders=True) |
|
81 for res in resources: layerobj.delete_folder(res) |
|
82 |
|
83 # Find all ConfML files |
|
84 confml_resources = sourceconfig.list_configurations() |
|
85 |
|
86 # Find all other related files and folders (content/, doc/ etc.) |
|
87 layerobj = sourceconfig.get_layer() |
|
88 targetobj = targetconfig.get_layer() |
|
89 other_resources = layerobj.list_all_related(empty_folders=True) |
|
90 |
|
91 # Copy the resources to the target configuration |
|
92 for res_path in confml_resources + other_resources: |
|
93 try: |
|
94 rres = layerobj.open_resource(res_path,"rb") |
|
95 wres = targetobj.open_resource(res_path,"wb") |
|
96 print "Copying %s" % rres.path |
|
97 logger.info('Copying layer resource %s to %s' % (rres.path, wres.path)) |
|
98 wres.write(rres.read()) |
|
99 wres.close() |
|
100 rres.close() |
|
101 except exceptions.NotResource: |
|
102 # If it isn't a resource (file), it's a folder |
|
103 targetobj.create_folder(res_path) |
|
104 |
|
105 # Remove all configurations from the target layer root |
|
106 already_included = targetconfig.list_configurations() |
|
107 for confml_path in already_included: |
|
108 targetconfig.remove_configuration(confml_path) |
|
109 |
|
110 # Include them back |
|
111 for confml_path in already_included: |
|
112 targetconfig.include_configuration(confml_path) |
|
113 |
|
114 # Include all added configurations |
|
115 for confml_path in confml_resources: |
|
116 if confml_path not in already_included: |
|
117 print "Including %s in layer root %s" % (confml_path, targetconfig.path) |
|
118 targetconfig.include_configuration(confml_path) |
|
119 |
|
120 def find_layers_to_merge(layer_indices, rename, sourceconfig, targetconfig): |
|
121 """ |
|
122 Return a list of layers to merge. |
|
123 |
|
124 @param layer_indices: List of layer indices to merge, can also be |
|
125 None to indicate that all layers are to be merged. |
|
126 @param rename: True if the layers should be renamed in the target |
|
127 config, False if not. |
|
128 @return: A list of tuples (layer_root, target_layer_root), where |
|
129 layer_root is the path to the layer root in the source |
|
130 configuration and target_layer_root the one in the target |
|
131 configuration. |
|
132 """ |
|
133 # Get a list of all configurations to merge |
|
134 if layer_indices is None: |
|
135 mergeconfigs = sourceconfig.list_configurations() |
|
136 else: |
|
137 mergeconfigs = sort_mergeconfigs(layer_indices, sourceconfig.list_configurations()) |
|
138 |
|
139 result = [] |
|
140 if rename: |
|
141 for source_path in mergeconfigs: |
|
142 target_path = get_new_layer_name(source_path, targetconfig.path) |
|
143 result.append((source_path, target_path)) |
|
144 else: |
|
145 for source_path in mergeconfigs: |
|
146 result.append((source_path, source_path)) |
|
147 |
|
148 return result |
|
149 |
|
150 def sort_mergeconfigs(layers, sourceconfigs): |
|
151 """ |
|
152 Return a correctly sorted list of source configuration layers. |
|
153 @param layers: List of the indices of the layers to merg. Can be None, in |
|
154 which case all layers are returned. |
|
155 @param sourceconfigs: List of all configuration layer root paths in the |
|
156 source project. |
|
157 @return: List of configuration layer root paths. |
|
158 """ |
|
159 sorted_configs = [None for _ in xrange(len(sourceconfigs))] |
|
160 for layer in layers: |
|
161 sorted_configs[layer]=sourceconfigs[layer] |
|
162 sorted_configs = filter(lambda x: x != None, sorted_configs) |
|
163 return sorted_configs |
|
164 |
|
165 def merge_config_root_to_config_root(source_project, target_project, |
|
166 source_config, target_config, |
|
167 layer_indices, rename, |
|
168 merge_policy): |
|
169 """ |
|
170 Merge the source configuration root to the target configuration root. |
|
171 @param layer_indices: List of layer indices to specify the layers |
|
172 to merge, can be None. |
|
173 @param rename: If True, the merged layers are renamed based on the |
|
174 name of the target configuration root. |
|
175 @param merge_policy: The used merge policy. |
|
176 """ |
|
177 |
|
178 def get_active_root_if_necessary(project, configuration, name): |
|
179 if configuration: |
|
180 return configuration |
|
181 else: |
|
182 active_root = project.get_storage().get_active_configuration() |
|
183 if active_root == "": |
|
184 raise MergeFailedException("No %s configuration given and the project does not have an active root" % name) |
|
185 else: |
|
186 return active_root |
|
187 |
|
188 target_root = get_active_root_if_necessary(target_project, target_config, 'target') |
|
189 source_root = get_active_root_if_necessary(source_project, source_config, 'source') |
|
190 |
|
191 print "Target config: %s" % target_root |
|
192 print "Source config: %s" % source_root |
|
193 |
|
194 try: |
|
195 source_config = source_project.get_configuration(source_root) |
|
196 except exceptions.NotFound: |
|
197 raise MergeFailedException("Configuration root '%s' not found in source project" % source_root) |
|
198 |
|
199 |
|
200 # Create or get the target configuration root |
|
201 try: |
|
202 target_config = target_project.get_configuration(target_root) |
|
203 except exceptions.NotFound: |
|
204 logger.info('Creating new root configuration %s' % (target_config)) |
|
205 target_config = target_project.create_configuration(target_config) |
|
206 for sourcelayer_path in source_config.list_configurations(): |
|
207 sourcelayer = source_config.get_configuration(sourcelayer_path) |
|
208 sourcelayer_path = sourcelayer.path |
|
209 if target_config.get_storage().is_resource(sourcelayer.path): |
|
210 logger.info('Including layer %s to root %s' % (sourcelayer_path, target_config.path)) |
|
211 target_config.include_configuration(sourcelayer_path) |
|
212 else: |
|
213 logger.info('Creating new layer %s to root %s' % (sourcelayer_path, target_config.path)) |
|
214 target_config.create_configuration(sourcelayer_path) |
|
215 |
|
216 # Collect a correctly sorted list of all layer paths to merge |
|
217 layers_to_merge = find_layers_to_merge( |
|
218 layer_indices = layer_indices, |
|
219 rename = rename, |
|
220 sourceconfig = source_config, |
|
221 targetconfig = target_config) |
|
222 |
|
223 print "Merging %d layer(s)..." % len(layers_to_merge) |
|
224 |
|
225 # Merge the layers |
|
226 for source_path, target_path in layers_to_merge: |
|
227 print "Merging %s -> %s" % (source_path, target_path) |
|
228 |
|
229 source_layer = source_project.get_configuration(source_path) |
|
230 |
|
231 if source_path != target_path: |
|
232 try: |
|
233 target_config.remove_configuration(source_path) |
|
234 except exceptions.NotFound: |
|
235 pass |
|
236 target_config.create_configuration(target_path) |
|
237 |
|
238 # Get or create the target configuration layer |
|
239 try: |
|
240 target_layer = target_config.get_configuration(target_path) |
|
241 except exceptions.NotFound: |
|
242 logger.info('Creating new layer configuration %s' % (target_path)) |
|
243 target_layer = target_config.create_configuration(target_path) |
|
244 |
|
245 merge_configuration_layer(source_layer, target_layer, merge_policy) |
|
246 |
|
247 |
|
248 |
|
249 def main(argv=sys.argv): |
|
250 parser = OptionParser(version="%%prog %s" % VERSION) |
|
251 |
|
252 parser.add_options(cone_common.COMMON_OPTIONS) |
|
253 |
|
254 parser.add_option("-c", "--configuration",\ |
|
255 dest="configuration",\ |
|
256 help="defines the name of the target configuration for the action",\ |
|
257 metavar="CONFIG") |
|
258 |
|
259 parser.add_option("-p", "--project",\ |
|
260 dest="project",\ |
|
261 help="defines the location of current project. Default is the current working directory.",\ |
|
262 default=".",\ |
|
263 metavar="STORAGE") |
|
264 |
|
265 group = OptionGroup(parser, 'Merge options', |
|
266 'The merge functionality is meant to merge configurations/layers ' |
|
267 'from a remote project (defined with -r) to the current project (defined with -p). ' |
|
268 'Default value for the current project is the currently working directory. ' |
|
269 'A project can be either a folder or a cpf/zip file. There are two ways to ' |
|
270 'use merge: merge configuration roots (multiple layers), or specific layers. ' |
|
271 'See the ConE documentation for details and examples.') |
|
272 |
|
273 group.add_option("-r", "--remote",\ |
|
274 dest="remote",\ |
|
275 help="defines the location of remote storage",\ |
|
276 metavar="STORAGE") |
|
277 |
|
278 group.add_option("-s", "--sourceconfiguration",\ |
|
279 dest="sourceconfiguration",\ |
|
280 help="defines the name of the remote configuration inside the remote storage for the merge action. "\ |
|
281 "Default is the active root of the remote project.",\ |
|
282 metavar="CONFIG") |
|
283 |
|
284 group.add_option("--sourcelayer", |
|
285 help="Defines a specific layer to use as the layer to merge "\ |
|
286 "from the remote project. Must be the layer root (ConfML file)."\ |
|
287 "For example: --sourcelayer assets/somelayer/root.confml", |
|
288 metavar="LAYER_ROOT", |
|
289 default=None) |
|
290 |
|
291 group.add_option("--targetlayer", |
|
292 help="Defines a specific layer (root) to use as the layer to merge "\ |
|
293 "into the target project. Must be the layer root (ConfML file)."\ |
|
294 "For example: --targetlayer assets/somelayer/root.confml", |
|
295 metavar="LAYER_ROOT", |
|
296 default=None) |
|
297 |
|
298 group.add_option("--rename",\ |
|
299 action="store_true", |
|
300 dest="rename",\ |
|
301 help="defines that the merged layers need to be renamed", |
|
302 default=False) |
|
303 |
|
304 group.add_option("--all",\ |
|
305 action="store_true", |
|
306 dest="all",\ |
|
307 help="Defines that the entire configuration (all layers) needs to be merged. "\ |
|
308 "This has no effect when merging layers directly using --sourcelayer and --targetlayer.", |
|
309 default=False) |
|
310 |
|
311 group.add_option("-l", "--layer",\ |
|
312 dest="layers",\ |
|
313 type="int", |
|
314 action="append", |
|
315 help="Define the layers of the source configuration that are included to merge action. "\ |
|
316 "The layer operation can be used several times in a single command. "\ |
|
317 "Note that this can only be used when merging configuration roots, not "\ |
|
318 "specific layers using --sourcelayer and --targetlayer. "\ |
|
319 "Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2", |
|
320 metavar="LAYERS",\ |
|
321 default=None) |
|
322 |
|
323 group.add_option("--merge-policy", |
|
324 help="Specifies the merge policy to use when merging layers. "\ |
|
325 "Possible values: "\ |
|
326 "replace-add - Add/replace files from source layer, but leave other files in the target as they are. "\ |
|
327 " "\ |
|
328 "overwrite-layer - Overwrite the entire layer (remove all previous content).", |
|
329 default=MergePolicy.REPLACE_ADD) |
|
330 |
|
331 parser.add_option_group(group) |
|
332 (options, _) = parser.parse_args(argv) |
|
333 |
|
334 cone_common.handle_common_options(options) |
|
335 |
|
336 # Check the passed options |
|
337 if not MergePolicy.is_valid(options.merge_policy): |
|
338 parser.error("Invalid merge policy: %s\nMust be one of %s" % (options.merge_policy, '\n'.join(MergePolicy.ALL))) |
|
339 if not options.remote: parser.error("Remote project must be given") |
|
340 if options.layers and (options.sourcelayer or options.targetlayer): |
|
341 parser.error("Specifying layer indices using --layer is not supported when using --sourcelayer or --targetlayer!") |
|
342 if options.sourcelayer and not options.targetlayer: |
|
343 parser.error("Merging a layer into a configuration is not supported at the moment!") |
|
344 if options.sourcelayer and not options.sourcelayer.lower().endswith('.confml'): |
|
345 parser.error("Source layer root should be a .confml file") |
|
346 if options.targetlayer and not options.targetlayer.lower().endswith('.confml'): |
|
347 parser.error("Target layer root should be a .confml file") |
|
348 if not options.sourcelayer and options.targetlayer: |
|
349 parser.error("Cannot merge a configuration into a layer!") |
|
350 |
|
351 # If layers for configuration root merging are not specifically given, |
|
352 # the default is the last layer |
|
353 if options.layers is None: |
|
354 options.layers = [-1] |
|
355 |
|
356 target_project = api.Project(api.Storage.open(options.project,"a", username=options.username, password=options.password)) |
|
357 source_project = api.Project(api.Storage.open(options.remote,"r", username=options.username, password=options.password)) |
|
358 |
|
359 print "Target project: %s" % options.project |
|
360 print "Source project: %s" % options.remote |
|
361 |
|
362 target_config = None |
|
363 try: |
|
364 if options.sourcelayer and options.targetlayer: |
|
365 print "Target layer: %s" % options.targetlayer |
|
366 print "Source layer: %s" % options.sourcelayer |
|
367 |
|
368 try: |
|
369 source_config = source_project.get_configuration(options.sourcelayer) |
|
370 except exceptions.NotFound: |
|
371 raise MergeFailedException("Layer root '%s' not found in source project" % options.sourcelayer) |
|
372 |
|
373 try: |
|
374 target_config = target_project.get_configuration(options.targetlayer) |
|
375 except exceptions.NotFound: |
|
376 logger.info('Creating new layer %s' % (options.targetlayer)) |
|
377 target_config = target_project.create_configuration(options.targetlayer) |
|
378 |
|
379 print "Merging layers..." |
|
380 merge_configuration_layer(source_config, target_config, options.merge_policy) |
|
381 else: |
|
382 # Merging a configuration root into a configuration root |
|
383 |
|
384 if options.all: layer_indices = None |
|
385 else: layer_indices = utils.distinct_array(options.layers) |
|
386 |
|
387 merge_config_root_to_config_root( |
|
388 source_project = source_project, |
|
389 target_project = target_project, |
|
390 source_config = options.sourceconfiguration, |
|
391 target_config = options.configuration, |
|
392 layer_indices = layer_indices, |
|
393 rename = options.rename, |
|
394 merge_policy = options.merge_policy) |
|
395 except MergeFailedException, e: |
|
396 print "Could not merge: %s" % e |
|
397 sys.exit(2) |
|
398 else: |
|
399 # Merge successful, so save the target configuration and project |
|
400 # to persist the changes |
|
401 if target_config: target_config.save() |
|
402 target_project.save() |
|
403 |
|
404 target_project.close() |
|
405 source_project.close() |
|
406 |
|
407 |
|
408 if __name__ == "__main__": |
|
409 main() |
|
410 |
|
411 |
|
412 |