|
1 # -*- encoding: latin-1 -*- |
|
2 |
|
3 #============================================================================ |
|
4 #Name : matti.py |
|
5 #Part of : Helium |
|
6 |
|
7 #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
8 #All rights reserved. |
|
9 #This component and the accompanying materials are made available |
|
10 #under the terms of the License "Eclipse Public License v1.0" |
|
11 #which accompanies this distribution, and is available |
|
12 #at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
13 # |
|
14 #Initial Contributors: |
|
15 #Nokia Corporation - initial contribution. |
|
16 # |
|
17 #Contributors: |
|
18 # |
|
19 #Description: |
|
20 #=============================================================================== |
|
21 |
|
22 """MATTI test drop generation.""" |
|
23 |
|
24 # pylint: disable=R0201,R0903,R0902,W0142 |
|
25 #W0142 => * and ** were used |
|
26 #R* removed during refactoring |
|
27 |
|
28 from optparse import OptionParser |
|
29 from xml.etree import ElementTree as et |
|
30 import logging |
|
31 import os |
|
32 import re |
|
33 import tempfile |
|
34 import zipfile |
|
35 import pkg_resources # pylint: disable=F0401 |
|
36 from path import path # pylint: disable=F0401 |
|
37 import amara |
|
38 import ntpath as atspath |
|
39 import jinja2 # pylint: disable=F0401 |
|
40 import ats3.parsers as parser |
|
41 |
|
42 _logger = logging.getLogger('matti') |
|
43 |
|
44 # Shortcuts |
|
45 E = et.Element |
|
46 SE = et.SubElement |
|
47 |
|
48 class Configuration(object): |
|
49 """ |
|
50 MATTI drop generation configuration. |
|
51 """ |
|
52 |
|
53 def __init__(self, opts): |
|
54 """ |
|
55 Initialize from optparse configuration options. |
|
56 """ |
|
57 |
|
58 self._opts = opts |
|
59 # Customize some attributes from how optparse leaves them. |
|
60 self.build_drive = path(self._opts.build_drive) |
|
61 self.file_store = path(self._opts.file_store) |
|
62 self.flash_images = self.split_paths(self._opts.flash_images) |
|
63 self.matti_sis_files = self.split_paths(self._opts.matti_sis_files) |
|
64 self.test_assets = self.split_paths(self._opts.testasset_location) |
|
65 self.template_loc = path(self._opts.template_loc) |
|
66 |
|
67 |
|
68 def split_paths(self, arg, delim=","): |
|
69 """ |
|
70 Split the string by delim, removing extra whitespace. |
|
71 """ |
|
72 return [path(part.strip()) |
|
73 for part in arg.split(delim) if part.strip()] |
|
74 |
|
75 def __getattr__(self, attr): |
|
76 return getattr(self._opts, attr) |
|
77 |
|
78 def __str__(self): |
|
79 dump = "Configuration:\n" |
|
80 seen = set() |
|
81 for key, value in vars(self).items(): |
|
82 if not key.startswith("_"): |
|
83 dump += "\t%s = %s\n" % (key, value) |
|
84 seen.add(key) |
|
85 for key, value in vars(self._opts).items(): |
|
86 if key not in seen: |
|
87 dump += "\t%s = %s\n" % (key, value) |
|
88 seen.add(key) |
|
89 return dump |
|
90 |
|
91 |
|
92 class MattiTestPlan(object): |
|
93 """ |
|
94 Tells MATTI server what to test and how. |
|
95 |
|
96 The MATTI test plan from which the test.xml file can be written. The test |
|
97 plan requires TestAsset(s) to perform the tests |
|
98 """ |
|
99 |
|
100 def __init__(self, config): |
|
101 self.pkg_parser = parser.PkgFileParser() |
|
102 self.file_store = config.file_store |
|
103 self.matti_timeout = config.matti_timeout |
|
104 self.test_profiles = config.test_profiles |
|
105 self.sierra_enabled = to_bool(config.sierra_enabled) |
|
106 self.sierra_parameters = config.sierra_parameters |
|
107 self.test_profiles = config.test_profiles.strip().split(",") |
|
108 self.build_drive = config.build_drive |
|
109 self.matti_sis_files = "" |
|
110 self.install_files = [] |
|
111 self.matti_task_files = None |
|
112 |
|
113 def insert_execution_block(self, block_count=1, image_files=None, matti_sis_files=None, test_asset_path=None, matti_parameters=None): |
|
114 """ |
|
115 Insert Matti tasks and test data files into execution block |
|
116 """ |
|
117 self.matti_sis_files = matti_sis_files |
|
118 temp_sis_files = [] |
|
119 if self.matti_sis_files != None: |
|
120 for sis_file in self.matti_sis_files: |
|
121 temp_sis_files.append(sis_file.split("#")) |
|
122 |
|
123 test_asset_path = test_asset_path |
|
124 if image_files is None: |
|
125 image_files = [] |
|
126 |
|
127 exe_dict = dict(name="exe%d" % block_count, asset_path=test_asset_path, image_files=image_files, matti_sis_files=temp_sis_files) |
|
128 exe_dict = dict(exe_dict, test_timeout=self.matti_timeout) |
|
129 exe_dict = dict(exe_dict, matti_parameters=matti_parameters) |
|
130 exe_dict = dict(exe_dict, sierra_enabled=self.sierra_enabled.lower()) |
|
131 exe_dict = dict(exe_dict, sierra_parameters=self.sierra_parameters) |
|
132 |
|
133 |
|
134 self.matti_task_files = self.create_matti_task_files_list(self.sierra_enabled, test_asset_path) |
|
135 exe_dict = dict(exe_dict, matti_task_files=self.matti_task_files) |
|
136 |
|
137 self.install_files = self.create_install_files_list(test_asset_path) |
|
138 exe_dict = dict(exe_dict, install_files=self.install_files) |
|
139 return exe_dict |
|
140 |
|
141 def create_matti_task_files_list(self, enabler=None, asset_path=None): |
|
142 """ |
|
143 Creates list of files needed to include in MATTI execution tasks |
|
144 if sierra.enabled then |
|
145 profiles (.sip files) are included |
|
146 else |
|
147 all ruby (.rb) files are included |
|
148 """ |
|
149 |
|
150 profiles = [] |
|
151 rb_files = [] |
|
152 |
|
153 #If sierra engine is enabled (set to True) |
|
154 if self.sierra_enabled.lower() == "true": |
|
155 profile_path = path(os.path.join(asset_path, "profile")) |
|
156 if os.path.exists(profile_path): |
|
157 for profile_name in self.test_profiles: |
|
158 item = list(profile_path.walkfiles("%s.sip"%profile_name.lower().strip())) |
|
159 if len(item) > 0: |
|
160 #profiles.append(os.path.join(profile_path, item[0])) |
|
161 profiles.append(asset_path.rsplit(os.sep, 1)[1] + "/" + "profile" + "/" + item[0].rsplit(os.sep, 1)[1]) |
|
162 return profiles |
|
163 else: #If sierra engine is not enabled (set to False) |
|
164 if os.path.exists(asset_path): |
|
165 #returns list(asset_path.walkfiles("*.rb")): |
|
166 for item in list(asset_path.walkfiles("*.rb")): |
|
167 rb_files.append(asset_path.rsplit(os.sep, 1)[1] + "/" + item.rsplit(os.sep, 1)[1]) |
|
168 # Sorting the result, so we ensure they are always in similar order. |
|
169 rb_files.sort() |
|
170 return rb_files |
|
171 |
|
172 def create_install_files_list(self, asset_path=None): |
|
173 """ |
|
174 Collects all the .pkg files and extract data |
|
175 Creates a list of src, dst files. |
|
176 """ |
|
177 pkg_files = [] |
|
178 if os.path.exists(asset_path): |
|
179 pkg_files = list(asset_path.walkfiles("*.pkg")) |
|
180 return self.pkg_parser.get_data_files(pkg_files, self.build_drive) |
|
181 else: |
|
182 return None |
|
183 |
|
184 def __getitem__(self, key): |
|
185 return self.__dict__[key] |
|
186 |
|
187 |
|
188 |
|
189 class MattiTestDropGenerator(object): |
|
190 """ |
|
191 Generate test drop zip file for Matti. |
|
192 |
|
193 Generates drop zip files file from Test Assets. The main |
|
194 responsibility of this class is to create testdrop and test.xml |
|
195 file and build a zip file for the MATTI drop. |
|
196 |
|
197 """ |
|
198 |
|
199 def __init__(self): |
|
200 self.drop_path_root = path("MATTIDrop") |
|
201 self.drop_path = None |
|
202 self.defaults = {} |
|
203 |
|
204 def generate(self, xml_dict, output_file, template_loc=None): |
|
205 """Generate a test drop file.""" |
|
206 xml = self.generate_xml(xml_dict, template_loc) |
|
207 return self.generate_drop(xml_dict, xml, output_file) |
|
208 |
|
209 def generate_drop(self, xml_dict, xml, output_file): |
|
210 """Generate test drop zip file.""" |
|
211 |
|
212 zfile = zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) |
|
213 try: |
|
214 for drop_file, src_file in self.drop_files(xml_dict): |
|
215 |
|
216 _logger.info(" + Adding: %s" % src_file.strip()) |
|
217 try: |
|
218 zfile.write(src_file.strip(), drop_file.encode('utf-8')) |
|
219 except OSError, expr: |
|
220 _logger.error(expr) |
|
221 doc = amara.parse(et.tostring(xml.getroot())) |
|
222 _logger.debug("XML output: %s" % doc.xml(indent=u"yes", encoding="ISO-8859-1")) |
|
223 zfile.writestr("test.xml", doc.xml(indent="yes", encoding="ISO-8859-1")) |
|
224 finally: |
|
225 _logger.info("Matti testdrop created successfully!") |
|
226 zfile.close() |
|
227 |
|
228 def generate_xml(self, xml_dict, template_loc): |
|
229 """ generate an XML file""" |
|
230 template_loc = path(template_loc).normpath() |
|
231 loader = jinja2.ChoiceLoader([jinja2.PackageLoader(__name__, 'templates')]) |
|
232 env = jinja2.Environment(loader=loader) |
|
233 if template_loc is None or not ".xml" in template_loc.lower(): |
|
234 template = env.from_string(pkg_resources.resource_string(__name__, 'matti_template.xml'))# pylint: disable=E1101 |
|
235 else: |
|
236 template = env.from_string(open(template_loc).read())# pylint: disable=E1101 |
|
237 |
|
238 xmltext = template.render(xml_dict=xml_dict, os=os, atspath=atspath, atsself=self).encode('ISO-8859-1') |
|
239 #print xmltext |
|
240 return et.ElementTree(et.XML(xmltext)) |
|
241 |
|
242 |
|
243 def generate_testasset_zip(self, xml_dict, output_file=None): |
|
244 """Generate TestAsset.zip for the MATTI server""" |
|
245 filename = xml_dict["temp_directory"].joinpath(r"TestAsset.zip") |
|
246 |
|
247 if output_file != None: |
|
248 filename = output_file |
|
249 |
|
250 for exe_block in xml_dict["execution_blocks"]: |
|
251 testasset_location = path(exe_block["asset_path"]) |
|
252 |
|
253 zfile = zipfile.ZipFile(filename, "w", zipfile.ZIP_DEFLATED) |
|
254 try: |
|
255 for file_ in list(testasset_location.walkfiles()): |
|
256 file_mod = file_.replace(testasset_location, "") |
|
257 zfile.write(file_, file_mod.encode('utf-8')) |
|
258 finally: |
|
259 zfile.close() |
|
260 return filename |
|
261 |
|
262 def drop_files(self, xml_dict): |
|
263 """Yield a list of drop files.""" |
|
264 |
|
265 drop_set = set() |
|
266 drop_files = [] |
|
267 |
|
268 #Adding test asset, there's an execution block for every test asset |
|
269 for execution_block in xml_dict["execution_blocks"]: |
|
270 testasset_location = path(execution_block["asset_path"]) |
|
271 asset_files = list(testasset_location.walkfiles()) |
|
272 |
|
273 drop_path = path(execution_block["name"]) |
|
274 |
|
275 drop_files = ((drop_path.parent, "images", execution_block["image_files"]), |
|
276 (drop_path.parent, "sisfiles", execution_block["matti_sis_files"]), |
|
277 (drop_path.parent, "mattiparameters", execution_block["matti_parameters"]), |
|
278 (drop_path.parent, execution_block["name"], asset_files)) |
|
279 |
|
280 for drop_dir, sub_dir, files in drop_files: |
|
281 for file_path in files: |
|
282 if file_path != None: |
|
283 |
|
284 #Adding image files to the top level, |
|
285 #Also adding mattiparameters.xml file |
|
286 if sub_dir.lower() == "images" or sub_dir.lower() == "mattiparameters": |
|
287 drop_file = drop_dir.joinpath(sub_dir, file_path.name) |
|
288 |
|
289 #Adding sisfiles, installation of matti sisfiles is a bit different |
|
290 #than normal sisfiles |
|
291 elif sub_dir.lower() == "sisfiles": |
|
292 drop_file = drop_dir.joinpath(sub_dir, path(file_path[0]).name) |
|
293 file_path = path(file_path[0]) |
|
294 |
|
295 #Adding test asset files |
|
296 else: |
|
297 temp_file = file_path.rsplit(os.sep, 1)[0] |
|
298 replace_string = testasset_location.rsplit(os.sep, 1)[0] |
|
299 drop_file = drop_dir.joinpath(sub_dir + "\\" + temp_file.replace(replace_string, ""), file_path.name) |
|
300 |
|
301 drop_file = drop_file.normpath() |
|
302 if drop_file not in drop_set: |
|
303 drop_set.add(drop_file) |
|
304 yield (drop_file, file_path.normpath()) |
|
305 |
|
306 |
|
307 class MattiComponentParser(object): |
|
308 """ |
|
309 Add information to the XML dictionary |
|
310 """ |
|
311 def __init__(self, config): |
|
312 self.flash_images = [path(p) for p in config.flash_images] |
|
313 self.matti_parameters = [path(config.matti_parameters).normpath()] |
|
314 self.matti_sis_files = config.matti_sis_files |
|
315 self.build_drive = config.build_drive |
|
316 self.test_timeout = config.matti_timeout |
|
317 self.diamonds_build_url = config.diamonds_build_url |
|
318 self.testrun_name = config.testrun_name |
|
319 self.alias_name = config.alias_name |
|
320 self.device_type = config.device_type |
|
321 self.report_email = config.report_email |
|
322 self.email_format = config.email_format |
|
323 self.email_subject = config.email_subject |
|
324 |
|
325 self.xml_dict = {} |
|
326 |
|
327 |
|
328 def insert_pre_data(self): |
|
329 """ |
|
330 Creates a dictionary for the data before |
|
331 the <execution> block starts. |
|
332 """ |
|
333 self.xml_dict = dict(self.xml_dict, temp_directory=path(tempfile.mkdtemp())) |
|
334 self.xml_dict = dict(self.xml_dict, diamonds_build_url=self.diamonds_build_url) |
|
335 self.xml_dict = dict(self.xml_dict, testrun_name=self.testrun_name) |
|
336 self.xml_dict = dict(self.xml_dict, alias_name=self.alias_name) |
|
337 self.xml_dict = dict(self.xml_dict, device_type=self.device_type) |
|
338 |
|
339 def create_execution_block(self, config): |
|
340 """Parse flash images and creates execution block for matti""" |
|
341 execution_block_list = [] |
|
342 block_count = 0 |
|
343 for test_asset in config.test_assets: |
|
344 if os.path.exists(test_asset): |
|
345 test_plan = MattiTestPlan(config) |
|
346 block_count += 1 |
|
347 execution_block_list.append(test_plan.insert_execution_block(block_count, self.flash_images, self.matti_sis_files, test_asset, self.matti_parameters)) |
|
348 |
|
349 |
|
350 self.xml_dict = dict(self.xml_dict, execution_blocks=execution_block_list) |
|
351 |
|
352 def insert_post_data(self): |
|
353 """ |
|
354 Creates a dictionary for the data after |
|
355 the <execution> block ends. Or, Postaction data |
|
356 """ |
|
357 self.xml_dict = dict(self.xml_dict, report_email=self.report_email) |
|
358 self.xml_dict = dict(self.xml_dict, email_format=self.email_format) |
|
359 self.xml_dict = dict(self.xml_dict, email_subject=self.email_subject) |
|
360 |
|
361 return self.xml_dict |
|
362 |
|
363 def create_drop(config): |
|
364 """Create a test drop.""" |
|
365 xml_dict = {} |
|
366 |
|
367 _logger.debug("initialize Matti dictionary") |
|
368 drop_parser = MattiComponentParser(config) |
|
369 |
|
370 #Inserting data for test run and global through out the dictionary |
|
371 drop_parser.insert_pre_data() |
|
372 |
|
373 #for every asset path there should be a |
|
374 #separate execution block |
|
375 drop_parser.create_execution_block(config) |
|
376 |
|
377 #Inserting reporting and email data (post actions) |
|
378 xml_dict = drop_parser.insert_post_data() |
|
379 |
|
380 generator = MattiTestDropGenerator() |
|
381 |
|
382 _logger.info("generating drop file: %s" % config.drop_file) |
|
383 generator.generate(xml_dict, output_file=config.drop_file, template_loc=config.template_loc) |
|
384 |
|
385 def to_bool(param): |
|
386 """setting a true or false based on a param value""" |
|
387 param = str(param).lower() |
|
388 if "true" == param or "t" == param or "1" == param: |
|
389 return "True" |
|
390 else: |
|
391 return "False" |
|
392 |
|
393 def main(): |
|
394 """Main entry point.""" |
|
395 |
|
396 |
|
397 cli = OptionParser(usage="%prog [options] PATH1 [PATH2 [PATH3 ...]]") |
|
398 cli.add_option("--ats4-enabled", help="ATS4 enabled", default="True") |
|
399 cli.add_option("--build-drive", help="Build area root drive") |
|
400 cli.add_option("--drop-file", help="Name for the final drop zip file", default="MATTIDrop.zip") |
|
401 |
|
402 cli.add_option("--minimum-flash-images", help="Minimum amount of flash images", default=2) |
|
403 cli.add_option("--flash-images", help="Paths to the flash image files", default="") |
|
404 cli.add_option("--matti-sis-files", help="Sis files location", default="") |
|
405 |
|
406 cli.add_option("--testasset-location", help="MATTI test assets location", default="") |
|
407 cli.add_option("--template-loc", help="Custom template location", default="") |
|
408 cli.add_option("--sierra-enabled", help="Enabled or disabled Sierra", default=True) |
|
409 cli.add_option("--test-profiles", help="Test profiles e.g. bat, fute", default="") |
|
410 cli.add_option("--matti-parameters", help="Location of xml file contains additional parameters for Matti", default="") |
|
411 |
|
412 cli.add_option("--matti-timeout", help="Test execution timeout value (default: %default)", default="60") |
|
413 cli.add_option("--sierra-parameters", help="Additional sierra parameters for matti task", default="") |
|
414 cli.add_option("--file-store", help="Destination path for reports.", default="") |
|
415 cli.add_option("--report-email", help="Email notification receivers", default="") |
|
416 cli.add_option("--testrun-name", help="Name of the test run", default="run") |
|
417 cli.add_option("--alias-name", help="Name of the alias", default="sut_s60") |
|
418 cli.add_option("--device-type", help="Device type (e.g. 'PRODUCT')", default="unknown") |
|
419 cli.add_option("--diamonds-build-url", help="Diamonds build url") |
|
420 cli.add_option("--email-format", help="Format of an email", default="") |
|
421 cli.add_option("--email-subject", help="Subject of an email", default="Matti Testing") |
|
422 |
|
423 |
|
424 cli.add_option("--verbose", help="Increase output verbosity", action="store_true", default=False) |
|
425 |
|
426 opts, _ = cli.parse_args() |
|
427 |
|
428 ats4_enabled = to_bool(opts.ats4_enabled) |
|
429 |
|
430 if ats4_enabled == "False": |
|
431 cli.error("MATTI tests execute on ATS4. Set property 'ats4.enabled'") |
|
432 |
|
433 if not opts.flash_images: |
|
434 cli.error("no flash image files given") |
|
435 if len(opts.flash_images.split(",")) < int(opts.minimum_flash_images): |
|
436 cli.error("Not enough flash files: %i defined, %i needed" % (len(opts.flash_images.split(",")), int(opts.minimum_flash_images) )) |
|
437 |
|
438 if opts.verbose: |
|
439 _logger.setLevel(logging.DEBUG) |
|
440 logging.basicConfig(level=logging.DEBUG) |
|
441 _ = tempfile.mkdtemp() |
|
442 config = Configuration(opts) |
|
443 create_drop(config) |
|
444 |
|
445 if __name__ == "__main__": |
|
446 main() |