|
1 # -*- coding: latin-1 -*- |
|
2 |
|
3 #============================================================================ |
|
4 #Name : test_ats3_aste.py |
|
5 #Part of : Helium |
|
6 |
|
7 #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
8 #All rights reserved. |
|
9 #This component and the accompanying materials are made available |
|
10 #under the terms of the License "Eclipse Public License v1.0" |
|
11 #which accompanies this distribution, and is available |
|
12 #at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
13 # |
|
14 #Initial Contributors: |
|
15 #Nokia Corporation - initial contribution. |
|
16 # |
|
17 #Contributors: |
|
18 # |
|
19 #Description: |
|
20 #=============================================================================== |
|
21 |
|
22 """ Testing ATS3 ASTE framework. """ |
|
23 |
|
24 # pylint: disable-msg=E1101,W0603,W0142 |
|
25 |
|
26 from cStringIO import StringIO |
|
27 from pprint import pprint |
|
28 from xml.etree.ElementTree import fromstring, tostring |
|
29 from xml.etree import ElementTree as et |
|
30 import difflib |
|
31 import logging |
|
32 logging.getLogger().setLevel(logging.ERROR) |
|
33 import re |
|
34 import tempfile |
|
35 import zipfile |
|
36 import os |
|
37 |
|
38 from path import path |
|
39 import amara |
|
40 import mocker |
|
41 |
|
42 import ats3.aste |
|
43 |
|
44 TEST_PATH = None |
|
45 TEST_FILES = {} |
|
46 TEST_ASSET_FILES = {} |
|
47 TSRC = None |
|
48 OUTPUT = None |
|
49 TEST_ZIP_PATH = None |
|
50 |
|
51 # Shortcuts |
|
52 E = et.Element |
|
53 SE = et.SubElement |
|
54 |
|
55 _logger = logging.getLogger("test_ast3_aste") |
|
56 |
|
57 class Bunch(object): |
|
58 |
|
59 def __init__(self, **kwargs): |
|
60 self.__dict__.update(kwargs) |
|
61 |
|
62 |
|
63 def equal_xml(xml1, xml2): |
|
64 """Check the equality of the given XML snippets. |
|
65 |
|
66 Tag name equality: |
|
67 |
|
68 >>> equal_xml('<a/>', '<a/>') |
|
69 True |
|
70 >>> equal_xml('<a/>', '<b/>') |
|
71 False |
|
72 |
|
73 Attribute equality: |
|
74 |
|
75 >>> equal_xml('<a k="v"/>', '<a k="v"/>') |
|
76 True |
|
77 >>> equal_xml('<a k="v"/>', '<a k="w"/>') |
|
78 False |
|
79 |
|
80 Text content equality: |
|
81 |
|
82 >>> equal_xml('<a>v</a>', '<a>v</a>') |
|
83 True |
|
84 >>> equal_xml('<a>v</a>', '<a>w</a>') |
|
85 False |
|
86 >>> equal_xml('<a>v</a>', '<a></a>') |
|
87 False |
|
88 |
|
89 Text content equality when whitespace differs: |
|
90 >>> equal_xml('<a>v</a>', '<a>v </a>') |
|
91 True |
|
92 |
|
93 Equality of child elements: |
|
94 |
|
95 >>> equal_xml('<a><b><c k="v"/></b></a>', '<a><b><c k="v"/></b></a>') |
|
96 True |
|
97 >>> equal_xml('<a><b><c k="v"/></b></a>', '<a><b><c k="w"/></b></a>') |
|
98 False |
|
99 >>> equal_xml('<a><b><c k="v"/>v</b></a>', '<a><b><c k="v"/>w</b></a>') |
|
100 False |
|
101 >>> equal_xml('<a><b><c k="v"/>v</b></a>', '<a><b><c k="v"/>v </b></a>') |
|
102 True |
|
103 |
|
104 """ |
|
105 if isinstance(xml1, basestring): |
|
106 xml1 = fromstring(xml1) |
|
107 if isinstance(xml2, basestring): |
|
108 xml2 = fromstring(xml2) |
|
109 if xml1.tag != xml2.tag: |
|
110 return False |
|
111 if xml1.attrib != xml2.attrib: |
|
112 return False |
|
113 if xml1.text: |
|
114 if not xml2.text: |
|
115 return False |
|
116 if xml2.text: |
|
117 if not xml1.text: |
|
118 return False |
|
119 if xml1.text and xml2.text and xml1.text.strip() != xml2.text.strip(): |
|
120 return False |
|
121 if xml1.tail is not None and xml2.tail is not None: |
|
122 if xml1.tail.strip() != xml2.tail.strip(): |
|
123 return False |
|
124 elif xml1.tail != xml2.tail: |
|
125 return False |
|
126 children1 = list(xml1.getchildren()) |
|
127 children2 = list(xml2.getchildren()) |
|
128 if len(children1) != len(children2): |
|
129 return False |
|
130 for child1, child2 in zip(children1, children2): |
|
131 return equal_xml(child1, child2) |
|
132 return True |
|
133 |
|
134 |
|
135 def setup_module(): |
|
136 global TEST_PATH, OUTPUT, TEST_ZIP_PATH |
|
137 TEST_PATH = path(tempfile.mkdtemp()) |
|
138 OUTPUT = TEST_PATH.joinpath("TestAsset") |
|
139 TEST_ZIP_PATH = TEST_PATH.joinpath("test_zip") |
|
140 asset = TEST_PATH |
|
141 component = TEST_PATH |
|
142 component.joinpath("group").makedirs() |
|
143 for path_parts in (("output", "images", "file1.fpsx"), |
|
144 ("output", "images", "file2.fpsx")): |
|
145 filename = component.joinpath(*path_parts) |
|
146 if not filename.parent.exists(): |
|
147 filename.parent.makedirs() |
|
148 filename.touch() |
|
149 TEST_FILES.setdefault(path_parts[1], []).append(file) |
|
150 for path_parts in (("TestAsset", "Localisation", "S60", "localisation.txt"), |
|
151 ("TestAsset", "TestCases", "TC_100_Test0", "file1.sis"), |
|
152 ("TestAsset", "TestCases", "TC_100_Test0", "file2.tcf"), |
|
153 ("TestAsset", "Tools", "TestCaseCreator", "test_creator.ini"), |
|
154 ("TestAsset", "testdrop.xml"),): |
|
155 filename = asset.joinpath(*path_parts) |
|
156 if not filename.parent.exists(): |
|
157 filename.parent.makedirs() |
|
158 filename.touch() |
|
159 TEST_ASSET_FILES.setdefault(path_parts[1], []).append(file) |
|
160 try: |
|
161 zip_component = TEST_ZIP_PATH |
|
162 filename = zip_component.joinpath("TestAsset.zip") |
|
163 if not filename.parent.exists(): |
|
164 filename.parent.makedirs() |
|
165 filename.touch() |
|
166 zfile = zipfile.ZipFile(zip_component.joinpath("TestAsset.zip"), "w", zipfile.ZIP_DEFLATED) |
|
167 for p in TEST_ASSET_FILES: |
|
168 print p |
|
169 zfile.write(p) |
|
170 zfile.close() |
|
171 TEST_ASSET_FILES.setdefault("ZIP", []).append(file) |
|
172 except OSError: |
|
173 print "Got except OSError. Continuing...\n" |
|
174 |
|
175 |
|
176 def teardown_module(): |
|
177 path(TEST_PATH).rmtree() |
|
178 |
|
179 |
|
180 class TestTestPlan(mocker.MockerTestCase): |
|
181 |
|
182 def __init__(self, methodName="runTest"): |
|
183 mocker.MockerTestCase.__init__(self, methodName) |
|
184 |
|
185 def setUp(self): |
|
186 opts = Bunch(testrun_name="testrun", harness="ASTE", |
|
187 device_type="product", plan_name="ats3_test_plan", diamonds_build_url="", |
|
188 software_version="W810", software_release="SPP 51.32", device_language="English", |
|
189 testasset_location=TEST_PATH.joinpath("TestAsset"), testasset_caseids="100",repeat="1", report_email="", |
|
190 file_store=path(), test_timeout="60", device_hwid="5425", test_type="smoke") |
|
191 self.tp = ats3.aste.AsteTestPlan(opts) |
|
192 self.image_files = TEST_FILES["images"] |
|
193 self.test_timeout = self.tp["test_timeout"] |
|
194 self.device_hwid = self.tp["device_hwid"] |
|
195 self.test_harness = self.tp["harness"] |
|
196 self.device_language = self.tp["device_language"] |
|
197 self.software_release = self.tp["software_release"] |
|
198 self.software_version = self.tp["software_version"] |
|
199 self.testasset_caseids = self.tp["testasset_caseids"] |
|
200 self.testasset_location = self.tp["testasset_location"] |
|
201 self.test_type = self.tp["test_type"] |
|
202 |
|
203 if self.testasset_location != "": |
|
204 self.test_asset_testcases = [self.testasset_location.joinpath("TestCases", "TC_100_Test0", "file1.sis"), self.testasset_location.joinpath("TestCases", "TC_100_Test0", "file2.tcf")] |
|
205 self.test_asset_tools = [self.testasset_location.joinpath("Tools", "TestCaseCreator", "test_creator.ini")] |
|
206 self.test_asset_localisation = [self.testasset_location.joinpath("Localisation", "S60", "localisation.txt")] |
|
207 self.test_asset_testdrop = self.testasset_location.joinpath("testdrop.xml") |
|
208 else: |
|
209 self.test_asset_testcases = TEST_ASSET_FILES["TestCases"] |
|
210 self.test_asset_tools = TEST_ASSET_FILES["Tools"] |
|
211 self.test_asset_localisation = TEST_ASSET_FILES["Localisation"] |
|
212 self.test_asset_testdrop = TEST_ASSET_FILES["testdrop.xml"] |
|
213 |
|
214 |
|
215 def test_creation(self): |
|
216 assert self.tp["testrun_name"] == "testrun" |
|
217 assert self.tp["harness"] == "ASTE" |
|
218 assert self.tp["device_type"] == "product" |
|
219 |
|
220 def test_insert_set(self): |
|
221 |
|
222 self.tp.insert_set(image_files=self.image_files, |
|
223 test_timeout=self.test_timeout) |
|
224 |
|
225 assert self.tp.sets[0] == dict(name="set0", |
|
226 image_files=self.image_files, |
|
227 test_timeout=self.test_timeout, |
|
228 test_harness=self.test_harness) |
|
229 |
|
230 def test_post_actions_email(self): |
|
231 assert not self.tp.post_actions |
|
232 receiver = "joe.average@example.com" |
|
233 self.tp.report_email = receiver |
|
234 assert len(self.tp.post_actions) == 1 |
|
235 action, items = self.tp.post_actions[0] |
|
236 items = dict(items) |
|
237 assert action == "SendEmailAction" |
|
238 assert items["to"] == receiver |
|
239 |
|
240 def test_post_actions_ats3_report_only(self): |
|
241 file_store = path("path/to/files") |
|
242 self.tp.file_store = file_store |
|
243 self.tp.harness = "EUNIT" |
|
244 assert len(self.tp.post_actions) == 2 |
|
245 action, items = self.tp.post_actions[0] |
|
246 items = dict(items) |
|
247 assert action == "FileStoreAction" |
|
248 assert items["report-type"] == "ATS_REPORT" |
|
249 assert items["to-folder"].startswith(file_store) |
|
250 assert items["to-folder"].endswith("ATS3_REPORT") |
|
251 |
|
252 def test_post_actions_aste(self): |
|
253 file_store = path("path/to/files") |
|
254 self.tp.file_store = file_store |
|
255 assert len(self.tp.post_actions) == 2 |
|
256 action, items = self.tp.post_actions[1] |
|
257 items = dict(items) |
|
258 assert action == "FileStoreAction" |
|
259 assert items["report-type"] == "ASTE_REPORT" |
|
260 assert items["to-folder"].startswith(file_store) |
|
261 assert items["to-folder"].endswith("ASTE_REPORT") |
|
262 |
|
263 def test_post_actions_diamonds(self): |
|
264 self.tp.diamonds_build_url = "http://diamonds.nmp.company.com/diamonds/builds/1234" |
|
265 assert len(self.tp.post_actions) == 1 |
|
266 action, items = self.tp.post_actions[0] |
|
267 assert action == "DiamondsAction" |
|
268 assert not items |
|
269 |
|
270 |
|
271 |
|
272 class TestXMLGeneration(mocker.MockerTestCase): |
|
273 """ |
|
274 Unit tests for the test.xml generation. |
|
275 """ |
|
276 |
|
277 def __init__(self, methodName="runTest"): |
|
278 self.image_files = None |
|
279 self.report_email = None |
|
280 self.diamonds_build_url = None |
|
281 self.test_harness = None |
|
282 self.file_store = None |
|
283 self.testasset_location = None |
|
284 self.test_plan = None |
|
285 self.gen = None |
|
286 mocker.MockerTestCase.__init__(self, methodName) |
|
287 |
|
288 |
|
289 def generate_xml(self): |
|
290 def files(*paths): |
|
291 return [TEST_PATH.joinpath(p) for p in paths] |
|
292 self.image_files = files("output/images/file1.fpsx", "output/images/file2.fpsx") |
|
293 self.report_email = "test.receiver@company.com" |
|
294 self.diamonds_build_url = "http://diamonds.nmp.company.com/diamonds/builds/1234" |
|
295 self.test_harness = "ASTE" |
|
296 self.file_store = path(r"path/to/reports") |
|
297 self.testasset_location = OUTPUT |
|
298 |
|
299 self.mocker.restore() |
|
300 test_plan = self.mocker.mock(count=False) |
|
301 mocker.expect(test_plan["testrun_name"]).result("test") |
|
302 mocker.expect(test_plan["harness"]).result("ASTE") |
|
303 mocker.expect(test_plan["device_type"]).result("product") |
|
304 mocker.expect(test_plan["plan_name"]).result("test plan") |
|
305 mocker.expect(test_plan["diamonds_build_url"]).result(self.diamonds_build_url) |
|
306 mocker.expect(test_plan["test_timeout"]).result("60") |
|
307 mocker.expect(test_plan["device_hwid"]).result("5425") |
|
308 mocker.expect(test_plan["testasset_location"]).result(self.testasset_location) |
|
309 mocker.expect(test_plan["testasset_caseids"]).result("100") |
|
310 mocker.expect(test_plan["software_release"]).result("SPP 51.32") |
|
311 mocker.expect(test_plan["software_version"]).result("W810") |
|
312 mocker.expect(test_plan["device_language"]).result("English") |
|
313 mocker.expect(test_plan["test_type"]).result("smoke") |
|
314 mocker.expect(test_plan["temp_directory"]).result(TEST_PATH) |
|
315 mocker.expect(test_plan.sets).result([ |
|
316 dict(name="set0", image_files=self.image_files, test_harness="ASTE")]) |
|
317 mocker.expect(test_plan.post_actions).result([ |
|
318 ("SendEmailAction", (("subject", "email subject"), |
|
319 ("type", "ATS3_REPORT"), |
|
320 ("send-files", "true"), |
|
321 ("to", self.report_email))), |
|
322 ("FileStoreAction", (("to-folder", self.file_store), |
|
323 ("report-type", "ATS_REPORT"), |
|
324 ("date-format", "yyyyMMdd"), |
|
325 ("time-format", "HHmmss"))), |
|
326 ("FileStoreAction", (("to-folder", self.file_store), |
|
327 ("report-type", "ASTE_REPORT"), |
|
328 ("run-log", "true"), |
|
329 ("date-format", "yyyyMMdd"), |
|
330 ("time-format", "HHmmss"))), |
|
331 ("DiamondsAction", ()) |
|
332 ]) |
|
333 |
|
334 self.mocker.replay() |
|
335 self.test_plan = test_plan |
|
336 |
|
337 self.gen = ats3.aste.AsteTestDropGenerator() |
|
338 return self.gen.generate_xml(test_plan) |
|
339 |
|
340 def test_basic_structure(self): |
|
341 "Check that the overall test.xml structure is valid." |
|
342 xml = self.generate_xml() |
|
343 # Check basics. |
|
344 assert xml.find(".").tag == "test" |
|
345 assert xml.find("./name").text == "test" |
|
346 assert xml.find("./buildid").text == self.diamonds_build_url |
|
347 assert xml.find("./target").tag |
|
348 assert xml.find("./target/device").tag |
|
349 harness, hardware, device_hwid = xml.findall("./target/device/property") |
|
350 softwareVersion, softwareRelease, deviceLanguage = xml.findall("./target/device/setting") |
|
351 assert harness.get("value") == "ASTE" |
|
352 assert hardware.get("value") == "product" |
|
353 assert softwareVersion.get("value") == "W810" |
|
354 assert softwareRelease.get("value") == "SPP 51.32" |
|
355 assert deviceLanguage.get("value") == "English" |
|
356 assert device_hwid.get("value") == "5425" |
|
357 |
|
358 # Check generation of the test plan. |
|
359 assert xml.find("./plan").get("name") == "Plan smoke product" |
|
360 assert xml.find("./plan/session").tag |
|
361 sets = xml.findall("./plan/session/set") |
|
362 assert len(sets) == 1 |
|
363 assert sets[0].get("name") == "set0" |
|
364 assert sets[0].find("./target/device").tag |
|
365 |
|
366 def test_set_structure(self): |
|
367 "Check that a <set> element's structure is valid." |
|
368 xml = self.generate_xml() |
|
369 tstset = xml.find("./plan/session/set") |
|
370 assert tstset.tag |
|
371 case = tstset.find("./case") |
|
372 assert case.tag |
|
373 assert case.get("name") == "set0 case" |
|
374 |
|
375 def test_case_flash_elems(self): |
|
376 xml = self.generate_xml() |
|
377 case = xml.find("./plan/session/set/case") |
|
378 flashes = case.findall("./flash") |
|
379 assert len(flashes) == len(self.image_files) |
|
380 for i, flash_file in enumerate(self.image_files): |
|
381 assert flashes[i].get("target-alias") == "DEFAULT" |
|
382 assert flashes[i].get("images") == path(r"ATS3Drop" + os.sep + "images") / flash_file.name |
|
383 |
|
384 def test_steps(self): |
|
385 xml = self.generate_xml() |
|
386 steps = iter(xml.findall("./plan/session/set/case/step")) |
|
387 self.check_executeasset_step(steps) |
|
388 |
|
389 def check_executeasset_step(self, steps): |
|
390 step = steps.next() |
|
391 assert step.get("name") == "Execute asset zip step" |
|
392 assert step.findtext("./command") == "execute-asset" |
|
393 params = step.findall("./params/param") |
|
394 assert params[0].get("repeat") == "1" |
|
395 assert params[1].get("asset-source") == "ATS3Drop" + os.sep + "TestAssets" + os.sep + "TestAsset.zip" |
|
396 assert params[2].get("testcase-ids") == "100" |
|
397 |
|
398 def test_post_actions(self): |
|
399 "Post actions are inserted into XML." |
|
400 xml = self.generate_xml() |
|
401 post_actions = xml.findall("./postAction") |
|
402 self.check_send_email_action(post_actions[0]) |
|
403 self.check_ats_report_action(post_actions[1]) |
|
404 self.check_aste_report_action(post_actions[2]) |
|
405 self.check_diamonds_action(post_actions[3]) |
|
406 |
|
407 def check_send_email_action(self, action): |
|
408 assert action.findtext("./type") == "SendEmailAction" |
|
409 params = action.findall("./params/param") |
|
410 assert params[0].get("name") == "subject" |
|
411 assert params[0].get("value") == "email subject" |
|
412 assert params[1].get("name") == "type" |
|
413 assert params[1].get("value") == "ATS3_REPORT" |
|
414 assert params[2].get("name") == "send-files" |
|
415 assert params[2].get("value") == "true" |
|
416 assert params[3].get("name") == "to" |
|
417 assert params[3].get("value") == self.report_email |
|
418 |
|
419 def check_ats_report_action(self, action): |
|
420 assert action.findtext("./type") == "FileStoreAction" |
|
421 params = action.findall("./params/param") |
|
422 assert params[0].get("name") == "to-folder" |
|
423 assert params[0].get("value") == self.file_store |
|
424 assert params[1].get("name") == "report-type" |
|
425 assert params[1].get("value") == "ATS_REPORT" |
|
426 assert params[2].get("name") == "date-format" |
|
427 assert params[2].get("value") == "yyyyMMdd" |
|
428 assert params[3].get("name") == "time-format" |
|
429 assert params[3].get("value") == "HHmmss" |
|
430 |
|
431 def check_aste_report_action(self, action): |
|
432 assert action.findtext("./type") == "FileStoreAction" |
|
433 params = action.findall("./params/param") |
|
434 assert params[0].get("name") == "to-folder" |
|
435 assert params[0].get("value") == self.file_store |
|
436 assert params[1].get("name") == "report-type" |
|
437 assert params[1].get("value") == "ASTE_REPORT" |
|
438 assert params[2].get("name") == "run-log" |
|
439 assert params[2].get("value") == "true" |
|
440 assert params[3].get("name") == "date-format" |
|
441 assert params[3].get("value") == "yyyyMMdd" |
|
442 assert params[4].get("name") == "time-format" |
|
443 assert params[4].get("value") == "HHmmss" |
|
444 |
|
445 def check_diamonds_action(self, action): |
|
446 assert action.findtext("./type") == "DiamondsAction" |
|
447 assert not action.findall("./params/param") |
|
448 |
|
449 def test_files(self): |
|
450 xml = self.generate_xml() |
|
451 files = iter(xml.findall("./files/file")) |
|
452 assert files.next().text == r"ATS3Drop" + os.sep + "images" + os.sep + "file1.fpsx" |
|
453 assert files.next().text == r"ATS3Drop" + os.sep + "images" + os.sep + "file2.fpsx" |
|
454 assert files.next().text == r"ATS3Drop" + os.sep + "TestAssets" + os.sep + "TestAsset.zip" |
|
455 self.assertRaises(StopIteration, files.next) |
|
456 |
|
457 def test_generate_testasset_zip(self): |
|
458 self.generate_xml() |
|
459 if re.search(r"[.]zip", self.test_plan["testasset_location"]): |
|
460 pass |
|
461 else: |
|
462 strbuffer = StringIO() |
|
463 assert strbuffer == self.gen.generate_testasset_zip(self.test_plan, strbuffer) |
|
464 zfile = zipfile.ZipFile(strbuffer, "r") |
|
465 try: |
|
466 contents = sorted(path(p).normpath() for p in zfile.namelist()) |
|
467 expected = sorted(path(p).normpath() |
|
468 for p in [(r"Localisation" + os.sep + "S60" + os.sep + "localisation.txt"), |
|
469 (r"TestCases" + os.sep + "TC_100_Test0" + os.sep + "file1.sis"), |
|
470 (r"TestCases" + os.sep + "TC_100_Test0" + os.sep + "file2.tcf"), |
|
471 (r"Tools" + os.sep + "TestCaseCreator" + os.sep + "test_creator.ini"), |
|
472 (r"testdrop.xml")]) |
|
473 diff = difflib.context_diff(expected, contents) |
|
474 assert contents == expected, "\n".join(diff) |
|
475 finally: |
|
476 zfile.close() |
|
477 |
|
478 def test_generate_drop(self): |
|
479 "Manifest for ATS3Drop directory structure is generated." |
|
480 xml = self.generate_xml() |
|
481 strbuffer = StringIO() |
|
482 |
|
483 self.gen.generate_drop(self.test_plan, xml, strbuffer) |
|
484 zfile = zipfile.ZipFile(strbuffer, "r") |
|
485 try: |
|
486 contents = sorted(path(p).normpath() for p in zfile.namelist()) |
|
487 expected = sorted(path(p).normpath() |
|
488 for p in [r"ATS3Drop" + os.sep + "images" + os.sep + "file1.fpsx", |
|
489 r"ATS3Drop" + os.sep + "images" + os.sep + "file2.fpsx", |
|
490 r"ATS3Drop" + os.sep + "TestAssets" + os.sep + "TestAsset.zip", |
|
491 r"test.xml"]) |
|
492 diff = difflib.context_diff(expected, contents) |
|
493 assert contents == expected, "\n".join(diff) |
|
494 finally: |
|
495 zfile.close() |