1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """ The ATS related parsers """
23
24
25
26
27
28
29
30 import os
31 import re
32 import sys
33 import string
34 import logging
35 from path import path
36 from textwrap import dedent
37 import fnmatch
38 import subprocess
39 import codecs
40
41 _logger = logging.getLogger('ats3')
42
43 import configuration
60
61
63 """
64 Parser for CPP tool output. Returns cleaned output from the execution
65 of CPP with or without parent paths included in the output.
66 """
67
70
71 - def get_cpp_output(self, bld_path = None, output_parameter = "n", imacros = None):
72 """
73 To clean out conditionals from the compilation it is necessary to
74 use C preprocessing to clean out those.
75
76 If ('n' - normal) output is chosen, parser returns list of paths
77 If ('e' - extended) output is chosen parser returns list of (path, parent_path) tuples
78 If ('d' - dependency) output is chosen parser returns a dicitionary (can be a nested dictionary)
79 of paths dependency (-ies).
80
81 'imacros' can also be given as parameters for CPP options.
82
83 if bld file is not given, the function will try to find the file(s) on the given location with extension ".inf"
84 """
85
86 clean_path_list = []
87 path_list = []
88 temp_path = os.getcwd()
89 if "bld.inf" in str(bld_path).lower():
90 os.chdir(os.path.normpath(os.path.join(bld_path, os.pardir)))
91 else:
92 os.chdir(os.path.normpath(os.path.join(bld_path)))
93 if imacros is not None:
94 command = u"cpp -imacros %s bld.inf" % str(imacros)
95 else:
96 command = u"cpp bld.inf"
97 process = subprocess.Popen(command, shell = True, stdout = subprocess.PIPE)
98 pipe = process.stdout
99
100 if output_parameter == "d":
101 return self.create_dependency_dictionary(pipe, bld_path)
102
103 for line in pipe.readlines():
104 if re.search(r"\A#\s.*?", line.strip()) or re.search(r"\A#.*?[0-9]", line.strip()):
105 if line.strip() not in path_list:
106 path_list.append(line.strip())
107 process.wait()
108 if process.returncode == 1:
109 _logger.error('CPP failed: ' + command + ' in: ' + os.getcwd())
110 pipe.close()
111
112 os.chdir(temp_path)
113 if output_parameter is "n":
114 for _path in self.clean_cpp_output(bld_path, path_list):
115 clean_path_list.append(_path[0])
116 elif output_parameter is "e":
117 clean_path_list = self.clean_cpp_output(bld_path, path_list)
118
119 clean_path_list = list(set(clean_path_list))
120
121 bfp = BldFileParser()
122
123 for tsrc in clean_path_list:
124 mmp_path = bfp.get_test_mmp_files(tsrc[0])
125 if tsrc[0] == tsrc[1]:
126 if mmp_path == None or mmp_path == []:
127 clean_path_list.remove(tsrc)
128
129 return clean_path_list
130
132 """
133 The output from CPP is cleaned in a fashion that the output is
134 a dictionary (or nested dictionary) of paths and their dependencies.
135 """
136 bld_parser = BldFileParser()
137 pkg_parser = PkgFileParser()
138 mmp_parser = MmpFileParser()
139
140 temp_path = os.getcwd()
141 parent = os.getcwd()
142 self.path_to_bld = path_to_bld
143 test_case = {}
144 test_sets = {}
145 harness = ""
146 main_level = ""
147 test_case = []
148 test_cases = []
149 output_list = []
150
151 for line in _pipe_.readlines():
152 if re.match(r"#.*", line.lower()):
153 path_var = re.findall(r'(.?bld.inf.?)', line.lower())
154 output_list.append((line, os.path.dirname(os.path.normpath(os.path.join(self.path_to_bld, re.findall(r'"(.*bld.inf?)"', line.lower())[0])))))
155 _pipe_.close()
156 os.chdir(temp_path)
157
158
159 for case in output_list:
160 temp_test_case = test_case
161 if re.match(r".*[bld.inf][^0-9]\Z", string.strip(string.lower(case[0]))):
162 main_level = case[1]
163 parent = case[1]
164 os.chdir(case[1])
165 mmp_files = []
166 dlls = {}
167 test_cases.append((parent, case[1]))
168 elif re.match(r".*[1]\Z", string.strip(string.lower(case[0]))):
169 parent = os.getcwd()
170 os.chdir(case[1])
171 mmp_files = []
172 dlls = {}
173 test_files = False
174 test_cases.append((parent, case[1]))
175 elif re.match(r".*[2]\Z", string.strip(string.lower(case[0]))):
176 if test_cases:
177 for tcase in test_cases:
178 if parent in tcase[1]:
179 parent = tcase[0]
180 os.chdir(tcase[1])
181 break
182
183 for t_case in test_cases:
184 if t_case[0] == t_case[1]:
185 del t_case
186 elif t_case[0] in main_level:
187 test_sets[t_case[1]] = {}
188 test_sets[t_case[1]]['content'] = {}
189 test_sets[t_case[1]]['content'][t_case[1]] = {}
190 harness = mmp_parser.get_harness(t_case[1])
191
192 test_sets[t_case[1]]['content'][t_case[1]]['type'] = mmp_parser.get_dll_type(t_case[1])
193 test_sets[t_case[1]]['content'][t_case[1]]['harness'] = harness
194 test_sets[t_case[1]]['content'][t_case[1]]['pkg_files'] = pkg_parser.get_pkg_files(t_case[1], False)
195 test_sets[t_case[1]]['content'][t_case[1]]['mmp_files'] = bld_parser.get_test_mmp_files(t_case[1], False)
196 else:
197 for key, value in test_sets.items():
198 if t_case[0] in value['content'].keys():
199 harness = mmp_parser.get_harness(t_case[1])
200 if harness is "" or harness in test_sets[key]['content'][t_case[0]]['harness']:
201 test_sets[key]['content'][t_case[1]] = {}
202 test_sets[key]['content'][t_case[1]]['type'] = mmp_parser.get_dll_type(t_case[1])
203 test_sets[key]['content'][t_case[1]]['harness'] = harness
204 test_sets[key]['content'][t_case[1]]['pkg_files'] = pkg_parser.get_pkg_files(t_case[1], False)
205 test_sets[key]['content'][t_case[1]]['mmp_files'] = bld_parser.get_test_mmp_files(t_case[1], False)
206 else:
207 test_sets[t_case[1]] = {}
208 test_sets[t_case[1]]['content'] = {}
209 test_sets[t_case[1]]['content'][t_case[1]] = {}
210 test_sets[t_case[1]]['content'][t_case[1]]['type'] = mmp_parser.get_dll_type(t_case[1])
211 test_sets[t_case[1]]['content'][t_case[1]]['harness'] = harness
212 test_sets[t_case[1]]['content'][t_case[1]]['pkg_files'] = pkg_parser.get_pkg_files(t_case[1], False)
213 test_sets[t_case[1]]['content'][t_case[1]]['mmp_files'] = bld_parser.get_test_mmp_files(t_case[1], False)
214
215 return test_sets
216
217
219 """
220 The output from CPP needs to be "cleaned" so that extra chars needs
221 to be removed and also hierarchy which cpp is following is preserved
222 and returned as an output.
223 """
224
225 pat = ""
226 value = ""
227 cleaned_output = []
228 if "bld.inf" in bld_path:
229 path_to_parent = os.path.dirname(bld_path)
230 else:
231 path_to_parent = bld_path
232 pat = re.compile(r'\A#\s*?.*?[\"](.*?)[\"].*?')
233 for _path in path_list:
234 if re.match(r".*[bld.inf]\s*?[^0-9]\Z", string.strip(string.lower(_path))):
235 value = pat.match(_path.strip())
236 path_to_tc = os.path.dirname(os.path.normpath(os.path.join((bld_path), value.group(1))))
237 cleaned_output.append((path_to_tc, path_to_parent))
238 if re.match(r".*[1]\s*?\Z", string.strip(string.lower(_path))):
239 value = pat.match(_path.strip())
240 path_to_tc = os.path.dirname(os.path.normpath(os.path.join(bld_path, value.group(1))))
241 cleaned_output.append((path_to_tc, path_to_parent))
242 if re.match(r".*[2]\s*?\Z", string.strip(string.lower(_path))):
243 if cleaned_output:
244 for cout in cleaned_output:
245 if string.lower(path_to_parent) == string.lower(cout[0]):
246 path_to_tc = cout[1]
247 path_to_parent = path_to_tc
248 return cleaned_output
249
250
252 """
253 Parser for bld.inf files. Returns MACRO values.
254 Parsing Paths can be done using CPP parser
255 """
258
259
260
261
262
263
265 """
266 returns a list of test mmp files
267 Usage: if "x:\abc\bldfile", "PRJ_TESTMMPFILES".
268 1. get_test_mmp_files("x:\abc\bldfile") - with full paths e.g. ["x:\abc\abc.mmp"]
269 2. get_test_mmp_files("x:\abc\bldfile", False) - without full paths e.g. ["abc.mmp"]
270
271 if bld file is not given, the function will try to find the file(s) on the given location with extension ".inf"
272 """
273
274 if bld_file_path == None:
275 _logger.warning("Incorrect bld file")
276 return None
277 else:
278 self.bld_file_path = path(bld_file_path)
279 if not "bld.inf" in str(self.bld_file_path).lower():
280 self.bld_file_path = os.path.join(os.path.normpath(self.bld_file_path), "bld.inf")
281
282 if not os.path.exists(self.bld_file_path):
283 _logger.error(r"bld file path does not exist: '%s'" % self.bld_file_path)
284 return None
285
286 return self.get_files(path(self.bld_file_path), "PRJ_TESTMMPFILES", with_full_path)
287
288
289 - def get_files(self, bld_inf_path, bld_macro, with_full_path = True):
290 """
291 Component's MMP files, as stored in BLD.INF.
292 """
293
294 mmp_files = []
295 self.bld_inf_path = path(bld_inf_path)
296 bld_inf = self.bld_inf_path.text()
297 try:
298 bld_inf = re.compile(r"%s" % bld_macro).split(bld_inf)[1].strip()
299 bld_inf = re.compile(r"PRJ_+\S").split(bld_inf)[0].strip()
300
301 except IndexError:
302 try:
303 bld_inf = re.compile(r"%s" % bld_macro).split(bld_inf)[0].strip()
304 bld_inf = re.compile(r"PRJ_+\S").split(bld_inf)[0].strip()
305
306 except IndexError:
307 _logger.warning("Index Error while parsing bld.inf file")
308
309 comments_free_text = self.ignore_comments_from_input(bld_inf)
310
311 mmp_files = re.findall(r"(\S+?[.]mmp)", comments_free_text)
312
313
314
315 if with_full_path:
316 self.bld_dir = self.bld_inf_path.dirname()
317 return [path.joinpath(self.bld_dir, mmp).normpath()
318 for mmp in mmp_files]
319 else:
320 return mmp_files
321
350
351
352
353
354
355
356
358 """
359 Parser for .mmp files. Returns wanted information from the mmp-file
360 - file type (executable dll, plugin, exe, etc)
361 - test harness (STIF, EUNIT) if mmp is related to the test component
362 - file name
363 - libraries listed in the mmp
364 """
365
368
370 """
371 Filetype given using TARGETTYPE in .mmp file is returned.
372 If "c:\path\to\mmp" is a location where mmp file is stored
373 get_target_filetype("c:\path\to\mmp")
374
375 if mmp file is not given, the function will try to find the file(s) on the given location with extension ".mmp"
376 """
377 return self.read_information_from_mmp(path_to_mmp, 4)
378
380 """
381 Filename given using TARGET in .mmp file is returned
382 If "c:\path\to\mmp" is a location where mmp file is stored
383 get_target_filename("c:\path\to\mmp")
384
385 if mmp file is not given, the function will try to find the file(s) on the given location with extension ".mmp"
386 """
387 return self.read_information_from_mmp(path_to_mmp, 3)
388
390 """
391 Libraries listed in the MMP file are returned in a list
392 If "c:\path\to\mmp" is a location where mmp file is stored
393 get_libraries("c:\path\to\mmp")
394
395 if mmp file is not given, the function will try to find the file(s) on the given location with extension ".mmp"
396 """
397 return self.read_information_from_mmp(path_to_mmp, 5)
398
400 """
401 Returns harness of test component
402 If "c:\path\to\mmp" is a location where mmp file is stored
403 get_harness("c:\path\to\mmp")
404
405 if mmp file is not given, the function will try to find the file(s) on the given location with extension ".mmp"
406 """
407 return self.read_information_from_mmp(path_to_mmp, 6)
408
410 """
411 Returns type of test whether 'executable' or 'dependent' (dependent can be a stub or plugin)
412 If "c:\path\to\mmp" is a location where mmp file is stored
413 get_dll_type("c:\path\to\mmp")
414
415 if mmp file is not given, the function will try to find the file(s) on the given location with extension ".mmp"
416 """
417 return self.read_information_from_mmp(path_to_mmp, 7)
418
511
513 """
514 Parses .pkg files. Returns a list of:
515 a. src path of the file
516 b. dst path on the phone
517 c. type of the file
518 for every file in the pkg file
519 """
520
522 self.platform = platform
523
524 - def get_pkg_files(self, location = None, with_full_path = True):
525 """
526 Returns list of PKG files on the given location. If True, full path is returned
527 otherwise only filenames. Default is set to True
528
529 Assume at location "c:\abd\files", two pkg file '1.pkg' and '2.pkg', then the funtion
530 can be called as:
531 1. get_pkg_files("c:\abd\files") - will return a list of pkg files with full paths.
532 like ['c:\abd\files\1.pkg', 'c:\abd\files\2.pkg']
533 2. get_pkg_files("c:\abd\files", False) - will return a list of pkg files only.
534 like ['1.pkg', '2.pkg']
535 """
536 self.location = path(location)
537 self.pkg_files = []
538 if not self.location.exists():
539 return None
540
541 for p, subdirs, files in os.walk(self.location):
542 pfiles = [f for f in files if self.platform != None and f.endswith(self.platform)]
543 if self.platform != None and len(pfiles)>0:
544 if with_full_path:
545 self.pkg_files.append(os.path.join(p, pfiles[0]))
546 else:
547 self.pkg_files.append(str(pfiles[0]))
548 else:
549 for name in files:
550 if fnmatch.fnmatch(name, "*.pkg"):
551 if with_full_path:
552 self.pkg_files.append(os.path.join(p, name))
553 else:
554 self.pkg_files.append(str(name))
555
556 return self.pkg_files
557
559 """
560 Returns data files, source and destination of the files to be installed
561 on the phone
562 e.g. location = tsrc\testComponent\group
563
564 Function can be called in any of the following ways:
565 1. get_data_files("c:\abc\abc.pkg") - only data files' paths are returnd
566 as they are mention in the pkg file
567 2. get_data_files("c:\abc\abc.pkg", "x:") - Proper data files' paths are returnd
568 with drive letter included
569 3. get_data_files("c:\abc\abc.pkg", "x:", "\.dll") - Data files' paths are returnd with
570 drive letter included but the dll
571 files will be excluded if found in
572 the pkg file
573
574 if pkg file is not given, the function will try to find the file(s) on the given location with extension ".pkg"
575 """
576
577 self.drive = drive
578 self.exclude = exclude
579 self._files = []
580
581 if type(location) is not list:
582 locations = [location]
583 else:
584 locations = location
585
586 for _file_ in locations:
587
588
589 if ".pkg" in str(_file_).lower():
590 self._files = _file_
591 else:
592 self.location = path(_file_)
593
594 if not self.location.exists():
595 continue
596 for p_file in self.get_pkg_files(self.location, True):
597 self._files.append(p_file)
598
599 return self.__read_pkg_file(self._files)
600
602 """Parse package file to get the src and dst paths" for installing files"""
603 mmp_parser = MmpFileParser()
604 ext = ""
605 val1 = ""
606 val2 = ""
607 map_src = ""
608 map_dst = ""
609 self.pkg_file_path = pkg_file_path
610
611 if not self.exclude == "":
612 if re.search(r'%s' % self.exclude, pkg_line) is not None:
613 return None
614
615
616 result = re.search(r'^\s*"(.*?)".*?-.*?"(.*?)"', pkg_line)
617
618 if result is None:
619 return None
620 val1, val2 = result.groups()
621
622 if val1 != "":
623 if path.isabs(path(val1).normpath()):
624 map_src = str(path.joinpath(self.drive, val1).normpath())
625 elif re.search(r"\A\w", val1, 1):
626 map_src = str(path.joinpath(self.pkg_file_path + os.sep, os.path.normpath(val1)).normpath())
627 else:
628 map_src = str(path.joinpath(self.pkg_file_path, path(val1)).normpath())
629 map_dst = str(path(val2).normpath())
630 else:
631 map_src, map_dst = val1, val2
632 map_src = map_src.strip()
633
634
635 map_dst = map_dst.replace("!:", "c:")
636 map_dst = map_dst.replace("$:", "c:")
637 map_dst = re.sub(r'^(\w)', r'\1', map_dst).strip()
638 indx = map_dst.rsplit(".")
639 try:
640 ext = indx[1]
641 except IndexError:
642 _logger.warning("Index Error in map_pkg_path()")
643
644 _test_type_ = ""
645 _target_filename_ = ""
646
647 _target_filename_ = mmp_parser.get_target_filename(self.pkg_file_path)
648 _test_type_ = mmp_parser.get_dll_type(self.pkg_file_path)
649 _harness_ = mmp_parser.get_harness(self.pkg_file_path)
650
651 if ext == "ini":
652 file_type = "engine_ini"
653 elif ext == "cfg":
654 file_type = "conf"
655 elif ext == "dll":
656
657 if _test_type_ == "dependent":
658 file_type = "data" + ":%s" % _test_type_
659 else:
660 file_type = "testmodule"
661
662 elif ext == "exe":
663 if _test_type_ == "dependent":
664 file_type = "data" + ":%s" % _test_type_
665 else:
666 file_type = "testmodule"
667
668 elif ext == "sisx":
669 file_type = ""
670 elif ext == "xml":
671 file_type = "trace_init"
672 elif ext == "pmd":
673 file_type = "pmd"
674 else:
675 file_type = "data"
676
677 if not map_src or map_src == "." or not map_dst or map_dst == ".":
678 return None
679
680 return path(map_src).normpath(), path(map_dst).normpath(), file_type
681
683 """Reads contents of PKG file"""
684 pkg_paths = []
685 for pkg_file in pkg_files:
686 if not os.path.exists( pkg_file ):
687 _logger.error("No PKG -file in path specified")
688 continue
689 else:
690 file1 = codecs.open(pkg_file, 'r', 'utf16')
691 try:
692 lines = file1.readlines()
693 except UnicodeError:
694 file1 = open(pkg_file, 'r')
695 lines = file1.readlines()
696 pkg_file_path = path((pkg_file.rsplit(os.sep, 1))[0])
697 for line in lines:
698 pkg_path = self.__map_pkg_path(line, pkg_file_path)
699 if pkg_path is None:
700 continue
701 else:
702 pkg_paths.append(pkg_path)
703
704 return pkg_paths
705