0
|
1 |
#
|
|
2 |
# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
|
|
3 |
# All rights reserved.
|
|
4 |
# This component and the accompanying materials are made available
|
|
5 |
# under the terms of "Eclipse Public License v1.0"
|
|
6 |
# which accompanies this distribution, and is available
|
|
7 |
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
|
|
8 |
#
|
|
9 |
# Initial Contributors:
|
|
10 |
# Nokia Corporation - initial contribution.
|
|
11 |
#
|
|
12 |
# Contributors:
|
|
13 |
#
|
|
14 |
# Description:
|
|
15 |
#
|
|
16 |
|
|
17 |
import sys
|
|
18 |
import logging
|
|
19 |
from optparse import OptionParser, OptionGroup
|
|
20 |
import cone_common
|
|
21 |
|
|
22 |
from cone.public import api, utils, exceptions
|
|
23 |
|
|
24 |
VERSION = '1.0'
|
|
25 |
|
|
26 |
logger = logging.getLogger('cone')
|
|
27 |
|
|
28 |
class MergeFailedException(Exception):
|
|
29 |
"""
|
|
30 |
Exception raised if the merge failed for some reason.
|
|
31 |
"""
|
|
32 |
pass
|
|
33 |
|
|
34 |
class MergePolicy(object):
|
|
35 |
"""
|
|
36 |
Merge policy constants.
|
|
37 |
"""
|
|
38 |
#: Replace/add files from the source layer into the
|
|
39 |
#: target layer, preserving files in the target layer
|
|
40 |
#: that do not exist in the source layer
|
|
41 |
REPLACE_ADD = 'replace-add'
|
|
42 |
|
|
43 |
#: Overwrite the entire target layer, so that it contains
|
|
44 |
#: only the contents of the source layer
|
|
45 |
OVERWRITE_LAYER = 'overwrite-layer'
|
|
46 |
|
|
47 |
ALL = (REPLACE_ADD, OVERWRITE_LAYER)
|
|
48 |
|
|
49 |
@classmethod
|
|
50 |
def is_valid(cls, policy):
|
|
51 |
return policy in cls.ALL
|
|
52 |
|
|
53 |
def get_new_layer_name(oldname,rootconfig):
|
|
54 |
newpath = utils.resourceref.get_path(oldname)
|
|
55 |
newpath+= "_" + utils.resourceref.remove_ext(utils.resourceref.get_filename(rootconfig))
|
|
56 |
newpath+= "/" + utils.resourceref.get_filename(oldname)
|
|
57 |
return newpath
|
|
58 |
|
|
59 |
def merge_configuration_layer(sourceconfig, targetconfig, merge_policy):
|
|
60 |
"""
|
|
61 |
Merge the contents of the source layer into the target layer.
|
|
62 |
@param sourceconfig: Source layer root configuration object.
|
|
63 |
@param targetconfig: Target layer root configuration object.
|
|
64 |
@param merge_policy: The used layer merge policy.
|
|
65 |
"""
|
|
66 |
# If policy tells to entirely overwrite the layer, remove all
|
|
67 |
# resources from the layer first
|
|
68 |
if merge_policy == MergePolicy.OVERWRITE_LAYER:
|
|
69 |
# Remove configurations from the layer root
|
|
70 |
confs = targetconfig.list_configurations()
|
|
71 |
for conf in confs:
|
|
72 |
targetconfig.remove_configuration(conf)
|
|
73 |
|
|
74 |
# Remove all related resources
|
|
75 |
layerobj = targetconfig.get_layer()
|
|
76 |
# Round one: remove all files
|
|
77 |
resources = layerobj.list_all_related(empty_folders=False)
|
|
78 |
for res in resources: layerobj.delete_resource(res)
|
|
79 |
# Round two: remove any remaining empty directories
|
|
80 |
resources = layerobj.list_all_related(empty_folders=True)
|
|
81 |
for res in resources: layerobj.delete_folder(res)
|
|
82 |
|
|
83 |
# Find all ConfML files
|
|
84 |
confml_resources = sourceconfig.list_configurations()
|
|
85 |
|
|
86 |
# Find all other related files and folders (content/, doc/ etc.)
|
|
87 |
layerobj = sourceconfig.get_layer()
|
|
88 |
targetobj = targetconfig.get_layer()
|
|
89 |
other_resources = layerobj.list_all_related(empty_folders=True)
|
|
90 |
|
|
91 |
# Copy the resources to the target configuration
|
|
92 |
for res_path in confml_resources + other_resources:
|
|
93 |
try:
|
|
94 |
rres = layerobj.open_resource(res_path,"rb")
|
|
95 |
wres = targetobj.open_resource(res_path,"wb")
|
|
96 |
print "Copying %s" % rres.path
|
|
97 |
logger.info('Copying layer resource %s to %s' % (rres.path, wres.path))
|
|
98 |
wres.write(rres.read())
|
|
99 |
wres.close()
|
|
100 |
rres.close()
|
|
101 |
except exceptions.NotResource:
|
|
102 |
# If it isn't a resource (file), it's a folder
|
|
103 |
targetobj.create_folder(res_path)
|
|
104 |
|
|
105 |
# Remove all configurations from the target layer root
|
|
106 |
already_included = targetconfig.list_configurations()
|
|
107 |
for confml_path in already_included:
|
|
108 |
targetconfig.remove_configuration(confml_path)
|
|
109 |
|
|
110 |
# Include them back
|
|
111 |
for confml_path in already_included:
|
|
112 |
targetconfig.include_configuration(confml_path)
|
|
113 |
|
|
114 |
# Include all added configurations
|
|
115 |
for confml_path in confml_resources:
|
|
116 |
if confml_path not in already_included:
|
|
117 |
print "Including %s in layer root %s" % (confml_path, targetconfig.path)
|
|
118 |
targetconfig.include_configuration(confml_path)
|
|
119 |
|
|
120 |
def find_layers_to_merge(layer_indices, rename, sourceconfig, targetconfig):
|
|
121 |
"""
|
|
122 |
Return a list of layers to merge.
|
|
123 |
|
|
124 |
@param layer_indices: List of layer indices to merge, can also be
|
|
125 |
None to indicate that all layers are to be merged.
|
|
126 |
@param rename: True if the layers should be renamed in the target
|
|
127 |
config, False if not.
|
|
128 |
@return: A list of tuples (layer_root, target_layer_root), where
|
|
129 |
layer_root is the path to the layer root in the source
|
|
130 |
configuration and target_layer_root the one in the target
|
|
131 |
configuration.
|
|
132 |
"""
|
|
133 |
# Get a list of all configurations to merge
|
|
134 |
if layer_indices is None:
|
|
135 |
mergeconfigs = sourceconfig.list_configurations()
|
|
136 |
else:
|
|
137 |
mergeconfigs = sort_mergeconfigs(layer_indices, sourceconfig.list_configurations())
|
|
138 |
|
|
139 |
result = []
|
|
140 |
if rename:
|
|
141 |
for source_path in mergeconfigs:
|
|
142 |
target_path = get_new_layer_name(source_path, targetconfig.path)
|
|
143 |
result.append((source_path, target_path))
|
|
144 |
else:
|
|
145 |
for source_path in mergeconfigs:
|
|
146 |
result.append((source_path, source_path))
|
|
147 |
|
|
148 |
return result
|
|
149 |
|
|
150 |
def sort_mergeconfigs(layers, sourceconfigs):
|
|
151 |
"""
|
|
152 |
Return a correctly sorted list of source configuration layers.
|
|
153 |
@param layers: List of the indices of the layers to merg. Can be None, in
|
|
154 |
which case all layers are returned.
|
|
155 |
@param sourceconfigs: List of all configuration layer root paths in the
|
|
156 |
source project.
|
|
157 |
@return: List of configuration layer root paths.
|
|
158 |
"""
|
|
159 |
sorted_configs = [None for _ in xrange(len(sourceconfigs))]
|
|
160 |
for layer in layers:
|
|
161 |
sorted_configs[layer]=sourceconfigs[layer]
|
|
162 |
sorted_configs = filter(lambda x: x != None, sorted_configs)
|
|
163 |
return sorted_configs
|
|
164 |
|
|
165 |
def merge_config_root_to_config_root(source_project, target_project,
|
|
166 |
source_config, target_config,
|
|
167 |
layer_indices, rename,
|
|
168 |
merge_policy):
|
|
169 |
"""
|
|
170 |
Merge the source configuration root to the target configuration root.
|
|
171 |
@param layer_indices: List of layer indices to specify the layers
|
|
172 |
to merge, can be None.
|
|
173 |
@param rename: If True, the merged layers are renamed based on the
|
|
174 |
name of the target configuration root.
|
|
175 |
@param merge_policy: The used merge policy.
|
|
176 |
"""
|
|
177 |
|
|
178 |
def get_active_root_if_necessary(project, configuration, name):
|
|
179 |
if configuration:
|
|
180 |
return configuration
|
|
181 |
else:
|
|
182 |
active_root = project.get_storage().get_active_configuration()
|
|
183 |
if active_root == "":
|
|
184 |
raise MergeFailedException("No %s configuration given and the project does not have an active root" % name)
|
|
185 |
else:
|
|
186 |
return active_root
|
|
187 |
|
|
188 |
target_root = get_active_root_if_necessary(target_project, target_config, 'target')
|
|
189 |
source_root = get_active_root_if_necessary(source_project, source_config, 'source')
|
|
190 |
|
|
191 |
print "Target config: %s" % target_root
|
|
192 |
print "Source config: %s" % source_root
|
|
193 |
|
|
194 |
try:
|
|
195 |
source_config = source_project.get_configuration(source_root)
|
|
196 |
except exceptions.NotFound:
|
|
197 |
raise MergeFailedException("Configuration root '%s' not found in source project" % source_root)
|
|
198 |
|
|
199 |
|
|
200 |
# Create or get the target configuration root
|
|
201 |
try:
|
|
202 |
target_config = target_project.get_configuration(target_root)
|
|
203 |
except exceptions.NotFound:
|
|
204 |
logger.info('Creating new root configuration %s' % (target_config))
|
|
205 |
target_config = target_project.create_configuration(target_config)
|
|
206 |
for sourcelayer_path in source_config.list_configurations():
|
|
207 |
sourcelayer = source_config.get_configuration(sourcelayer_path)
|
|
208 |
sourcelayer_path = sourcelayer.path
|
|
209 |
if target_config.get_storage().is_resource(sourcelayer.path):
|
|
210 |
logger.info('Including layer %s to root %s' % (sourcelayer_path, target_config.path))
|
|
211 |
target_config.include_configuration(sourcelayer_path)
|
|
212 |
else:
|
|
213 |
logger.info('Creating new layer %s to root %s' % (sourcelayer_path, target_config.path))
|
|
214 |
target_config.create_configuration(sourcelayer_path)
|
|
215 |
|
|
216 |
# Collect a correctly sorted list of all layer paths to merge
|
|
217 |
layers_to_merge = find_layers_to_merge(
|
|
218 |
layer_indices = layer_indices,
|
|
219 |
rename = rename,
|
|
220 |
sourceconfig = source_config,
|
|
221 |
targetconfig = target_config)
|
|
222 |
|
|
223 |
print "Merging %d layer(s)..." % len(layers_to_merge)
|
|
224 |
|
|
225 |
# Merge the layers
|
|
226 |
for source_path, target_path in layers_to_merge:
|
|
227 |
print "Merging %s -> %s" % (source_path, target_path)
|
|
228 |
|
|
229 |
source_layer = source_project.get_configuration(source_path)
|
|
230 |
|
|
231 |
if source_path != target_path:
|
|
232 |
try:
|
|
233 |
target_config.remove_configuration(source_path)
|
|
234 |
except exceptions.NotFound:
|
|
235 |
pass
|
|
236 |
target_config.create_configuration(target_path)
|
|
237 |
|
|
238 |
# Get or create the target configuration layer
|
|
239 |
try:
|
|
240 |
target_layer = target_config.get_configuration(target_path)
|
|
241 |
except exceptions.NotFound:
|
|
242 |
logger.info('Creating new layer configuration %s' % (target_path))
|
|
243 |
target_layer = target_config.create_configuration(target_path)
|
|
244 |
|
|
245 |
merge_configuration_layer(source_layer, target_layer, merge_policy)
|
|
246 |
|
|
247 |
|
|
248 |
|
|
249 |
def main(argv=sys.argv):
|
|
250 |
parser = OptionParser(version="%%prog %s" % VERSION)
|
|
251 |
|
|
252 |
parser.add_options(cone_common.COMMON_OPTIONS)
|
|
253 |
|
|
254 |
parser.add_option("-c", "--configuration",\
|
|
255 |
dest="configuration",\
|
|
256 |
help="defines the name of the target configuration for the action",\
|
|
257 |
metavar="CONFIG")
|
|
258 |
|
|
259 |
parser.add_option("-p", "--project",\
|
|
260 |
dest="project",\
|
|
261 |
help="defines the location of current project. Default is the current working directory.",\
|
|
262 |
default=".",\
|
|
263 |
metavar="STORAGE")
|
|
264 |
|
|
265 |
group = OptionGroup(parser, 'Merge options',
|
|
266 |
'The merge functionality is meant to merge configurations/layers '
|
|
267 |
'from a remote project (defined with -r) to the current project (defined with -p). '
|
|
268 |
'Default value for the current project is the currently working directory. '
|
|
269 |
'A project can be either a folder or a cpf/zip file. There are two ways to '
|
|
270 |
'use merge: merge configuration roots (multiple layers), or specific layers. '
|
|
271 |
'See the ConE documentation for details and examples.')
|
|
272 |
|
|
273 |
group.add_option("-r", "--remote",\
|
|
274 |
dest="remote",\
|
|
275 |
help="defines the location of remote storage",\
|
|
276 |
metavar="STORAGE")
|
|
277 |
|
|
278 |
group.add_option("-s", "--sourceconfiguration",\
|
|
279 |
dest="sourceconfiguration",\
|
|
280 |
help="defines the name of the remote configuration inside the remote storage for the merge action. "\
|
|
281 |
"Default is the active root of the remote project.",\
|
|
282 |
metavar="CONFIG")
|
|
283 |
|
|
284 |
group.add_option("--sourcelayer",
|
|
285 |
help="Defines a specific layer to use as the layer to merge "\
|
|
286 |
"from the remote project. Must be the layer root (ConfML file)."\
|
|
287 |
"For example: --sourcelayer assets/somelayer/root.confml",
|
|
288 |
metavar="LAYER_ROOT",
|
|
289 |
default=None)
|
|
290 |
|
|
291 |
group.add_option("--targetlayer",
|
|
292 |
help="Defines a specific layer (root) to use as the layer to merge "\
|
|
293 |
"into the target project. Must be the layer root (ConfML file)."\
|
|
294 |
"For example: --targetlayer assets/somelayer/root.confml",
|
|
295 |
metavar="LAYER_ROOT",
|
|
296 |
default=None)
|
|
297 |
|
|
298 |
group.add_option("--rename",\
|
|
299 |
action="store_true",
|
|
300 |
dest="rename",\
|
|
301 |
help="defines that the merged layers need to be renamed",
|
|
302 |
default=False)
|
|
303 |
|
|
304 |
group.add_option("--all",\
|
|
305 |
action="store_true",
|
|
306 |
dest="all",\
|
|
307 |
help="Defines that the entire configuration (all layers) needs to be merged. "\
|
|
308 |
"This has no effect when merging layers directly using --sourcelayer and --targetlayer.",
|
|
309 |
default=False)
|
|
310 |
|
|
311 |
group.add_option("-l", "--layer",\
|
|
312 |
dest="layers",\
|
|
313 |
type="int",
|
|
314 |
action="append",
|
|
315 |
help="Define the layers of the source configuration that are included to merge action. "\
|
|
316 |
"The layer operation can be used several times in a single command. "\
|
|
317 |
"Note that this can only be used when merging configuration roots, not "\
|
|
318 |
"specific layers using --sourcelayer and --targetlayer. "\
|
|
319 |
"Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
|
|
320 |
metavar="LAYERS",\
|
|
321 |
default=None)
|
|
322 |
|
|
323 |
group.add_option("--merge-policy",
|
|
324 |
help="Specifies the merge policy to use when merging layers. "\
|
|
325 |
"Possible values: "\
|
|
326 |
"replace-add - Add/replace files from source layer, but leave other files in the target as they are. "\
|
|
327 |
" "\
|
|
328 |
"overwrite-layer - Overwrite the entire layer (remove all previous content).",
|
|
329 |
default=MergePolicy.REPLACE_ADD)
|
|
330 |
|
|
331 |
parser.add_option_group(group)
|
|
332 |
(options, _) = parser.parse_args(argv)
|
|
333 |
|
|
334 |
cone_common.handle_common_options(options)
|
|
335 |
|
|
336 |
# Check the passed options
|
|
337 |
if not MergePolicy.is_valid(options.merge_policy):
|
|
338 |
parser.error("Invalid merge policy: %s\nMust be one of %s" % (options.merge_policy, '\n'.join(MergePolicy.ALL)))
|
|
339 |
if not options.remote: parser.error("Remote project must be given")
|
|
340 |
if options.layers and (options.sourcelayer or options.targetlayer):
|
|
341 |
parser.error("Specifying layer indices using --layer is not supported when using --sourcelayer or --targetlayer!")
|
|
342 |
if options.sourcelayer and not options.targetlayer:
|
|
343 |
parser.error("Merging a layer into a configuration is not supported at the moment!")
|
|
344 |
if options.sourcelayer and not options.sourcelayer.lower().endswith('.confml'):
|
|
345 |
parser.error("Source layer root should be a .confml file")
|
|
346 |
if options.targetlayer and not options.targetlayer.lower().endswith('.confml'):
|
|
347 |
parser.error("Target layer root should be a .confml file")
|
|
348 |
if not options.sourcelayer and options.targetlayer:
|
|
349 |
parser.error("Cannot merge a configuration into a layer!")
|
|
350 |
|
|
351 |
# If layers for configuration root merging are not specifically given,
|
|
352 |
# the default is the last layer
|
|
353 |
if options.layers is None:
|
|
354 |
options.layers = [-1]
|
|
355 |
|
|
356 |
target_project = api.Project(api.Storage.open(options.project,"a", username=options.username, password=options.password))
|
|
357 |
source_project = api.Project(api.Storage.open(options.remote,"r", username=options.username, password=options.password))
|
|
358 |
|
|
359 |
print "Target project: %s" % options.project
|
|
360 |
print "Source project: %s" % options.remote
|
|
361 |
|
|
362 |
target_config = None
|
|
363 |
try:
|
|
364 |
if options.sourcelayer and options.targetlayer:
|
|
365 |
print "Target layer: %s" % options.targetlayer
|
|
366 |
print "Source layer: %s" % options.sourcelayer
|
|
367 |
|
|
368 |
try:
|
|
369 |
source_config = source_project.get_configuration(options.sourcelayer)
|
|
370 |
except exceptions.NotFound:
|
|
371 |
raise MergeFailedException("Layer root '%s' not found in source project" % options.sourcelayer)
|
|
372 |
|
|
373 |
try:
|
|
374 |
target_config = target_project.get_configuration(options.targetlayer)
|
|
375 |
except exceptions.NotFound:
|
|
376 |
logger.info('Creating new layer %s' % (options.targetlayer))
|
|
377 |
target_config = target_project.create_configuration(options.targetlayer)
|
|
378 |
|
|
379 |
print "Merging layers..."
|
|
380 |
merge_configuration_layer(source_config, target_config, options.merge_policy)
|
|
381 |
else:
|
|
382 |
# Merging a configuration root into a configuration root
|
|
383 |
|
|
384 |
if options.all: layer_indices = None
|
|
385 |
else: layer_indices = utils.distinct_array(options.layers)
|
|
386 |
|
|
387 |
merge_config_root_to_config_root(
|
|
388 |
source_project = source_project,
|
|
389 |
target_project = target_project,
|
|
390 |
source_config = options.sourceconfiguration,
|
|
391 |
target_config = options.configuration,
|
|
392 |
layer_indices = layer_indices,
|
|
393 |
rename = options.rename,
|
|
394 |
merge_policy = options.merge_policy)
|
|
395 |
except MergeFailedException, e:
|
|
396 |
print "Could not merge: %s" % e
|
|
397 |
sys.exit(2)
|
|
398 |
else:
|
|
399 |
# Merge successful, so save the target configuration and project
|
|
400 |
# to persist the changes
|
|
401 |
if target_config: target_config.save()
|
|
402 |
target_project.save()
|
|
403 |
|
|
404 |
target_project.close()
|
|
405 |
source_project.close()
|
|
406 |
|
|
407 |
|
|
408 |
if __name__ == "__main__":
|
|
409 |
main()
|
|
410 |
|
|
411 |
|
|
412 |
|