1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import os
21 import re
22 import fileutils
23 import buildtools
24 import logging
25
26 _logger = logging.getLogger('delta_zip')
27 logging.basicConfig(level=logging.INFO)
28
30 """ MD5 CRC creation base class"""
31 - def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files):
32 """constructor"""
33 if not build_area_root.endswith(os.sep):
34 self.build_area_root = build_area_root + os.sep
35 self.nb_split = nb_split
36 self.temp_dir = temp_dir
37 self.exclude_dirs = exclude_dirs
38 self.list_of_files = list_of_files
39
41 """Create list of files (was list_files.pl)"""
42
43
44 if not os.path.exists(self.temp_dir):
45 os.mkdir(self.temp_dir)
46
47 fp_filelist = open(self.list_of_files, 'w')
48
49
50 scanner = fileutils.FileScanner(self.build_area_root)
51 scanner.add_include('**')
52
53 for _dir in self.exclude_dirs.split(','):
54 _dir = _dir.replace(self.build_area_root, "")
55 scanner.add_exclude(_dir)
56
57 for path in scanner.scan():
58 if (not os.path.isdir(path)) or (os.path.isdir(path) and (os.listdir(path) != [] and os.listdir(path) != ['.svn'])):
59 path = path.replace(self.build_area_root.lower(), "")
60 fp_filelist.write(path + "\n")
61
63 """Split the list of files for parallelalisation"""
64 md5_dir = os.path.join(self.temp_dir, "md5_temp")
65 self.dest_dir = os.path.join(md5_dir, str(self.nb_split))
66 if not os.path.exists(self.dest_dir):
67 os.makedirs(self.dest_dir)
68 fp_split = []
69
70
71 self.list_of_lists = self.dest_dir + "/list_of_lists.txt"
72 fp_list_of_lists = open(self.list_of_lists, 'w')
73 for i in range(self.nb_split):
74 filename = self.dest_dir + "/" + str(i) + ".txt"
75 _fp = open(filename, 'w')
76 fp_split.append(_fp)
77
78 fp_list_of_lists.write(filename + "\n")
79
80
81 fp_read = open(self.list_of_files, 'r')
82 line = fp_read.readline()
83 line_number = 0
84 while(line != ""):
85 fp_split[line_number % len(fp_split)].write(line)
86 line = fp_read.readline()
87 line_number += 1
88
89 fp_list_of_lists.close()
90 fp_read.close()
91 for _fp in fp_split:
92 _fp.close()
93
95 """ create the command to run evalid on each file in the list of files"""
96 liste = buildtools.CommandList()
97
98
99
100 for i in range(self.nb_split):
101
102 liste.addCommand(buildtools.Command("evalid", os.sep, ["", "-f", self.__get_partial_input_file_name(i) + " "+self.build_area_root, self.__get_partial_signature_file_name(i) ]))
103
104 return liste
105
109
111 """ get the signature file name string as has been created so far and add .md5 to it"""
112 return os.path.join(self.dest_dir, str(_nb) + ".md5")
113
115 """ concatenate all the files with the MD5 CRC in """
116
117 _fp = open(self.__get_partial_signature_file_name(0), 'r')
118 line = ""
119 header_temp = ""
120 header = ""
121 while (re.search(r'(\S+).*MD5=(\S+)', line) == None):
122 header_temp = header_temp + line
123 line = _fp.readline()
124
125 for line in header_temp.splitlines():
126 if re.match(r'Directory:.*', line):
127 line = "Directory:" + self.build_area_root
128 if re.match(r'FileList:.*', line):
129 line = "FileList:" + self.list_of_files
130 header = header + line + "\n"
131
132
133
134
135 header_size = len(header.splitlines())
136
137 fp_md5_signatures_file = open(signature_file, 'w')
138 fp_md5_signatures_file.write(header)
139 for i in range(self.nb_split):
140 _fp = open(self.__get_partial_signature_file_name(i), 'r')
141
142 for i in range(header_size):
143 _fp.readline()
144
145 fp_md5_signatures_file.write(_fp.read())
146 _fp.close()
147 fp_md5_signatures_file.close()
148
154
156 """ there should always be an overloaded version of this method in sub-classes"""
157 raise NotImplementedError()
158
159 - def build(self, signature_file):
164
166 """ there should always be an overlaoded version in the methos sub-class"""
167 raise NotImplementedError()
168
170 """ build the MD5 CRCs for all the files in the list of files"""
176
178 """Compute MD5 using the requested parallel build system"""
179 os.chdir(self.build_area_root)
180 os.system("perl -I%HELIUM_HOME%/tools/common/packages %HELIUM_HOME%/tools/compile/buildjob.pl -d " + self.makefile + " -l " + os.path.join(self.dest_dir, "md5.log") + " -n " + str(int(os.environ['NUMBER_OF_PROCESSORS'])*2))
181
182 """
183 Run the delta zipping over the EC build system
184 """
186 """ The MD5 CRC creation for delta zippinf for use on EC machines"""
187 - def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, ec_cluster_manager, ec_build_class, list_of_files):
188 MD5SignatureBuilder.__init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files)
189 self.ec_cluster_manager = ec_cluster_manager
190 self.ec_build_class = ec_build_class
191
197
199 """Compute MD5 using the requested parallel build system"""
200 root_path = os.environ['EMAKE_ROOT'] +";" + "c:\\apps;"
201 os.chdir(self.build_area_root)
202
203 print "emake --emake-cm=" + self.ec_cluster_manager + " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+ " --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile
204 os.system("emake --emake-cm=" + self.ec_cluster_manager + " --emake-annodetail=basic,history,file,waiting --emake-annofile="+self.temp_dir+"\\delta_zip_anno.xml"+ " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+" --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile)
205
207 """methods to create the delta zip after all the prep"""
208 - def __init__(self, build_area_root, temp_path, old_md5_signature, new_md5_signature):
209 self.build_area_root = os.path.join(build_area_root, os.sep)
210 self.temp_path = temp_path
211 self.old_md5_signature = old_md5_signature
212 self.new_md5_signature = new_md5_signature
213 self.sign_dic = SignaturesDict()
214
216 """ read each line of signature file search for .MD5"""
217 _fp = open(signature_file, 'r')
218 lines = _fp.read().splitlines()
219 _fp.close()
220 for line in lines:
221 info = re.search(r'([ \S]+) TYPE=.*MD5=(\S+)', line)
222 if info != None:
223 filename = info.group(1)
224 if not self.sign_dic.has_key(filename):
225 self.sign_dic[filename] = ["", ""]
226 self.sign_dic[filename][old_new] = info.group(2)
227
229 """Create Delta zip and list of file to delete."""
230
231 self.__fill_signature_dict(self.old_md5_signature, 0)
232 self.__fill_signature_dict(self.new_md5_signature, 1)
233
234
235
236
237
238 delete_list = []
239
240 if not os.path.exists(os.path.dirname(delete_list_file)):
241 os.mkdir(os.path.dirname(delete_list_file))
242 if not os.path.exists(self.temp_path):
243 os.mkdir(self.temp_path)
244
245 archive_txt = open(os.path.join(self.temp_path, 'create_zips.txt'), 'w')
246
247 for _file in self.sign_dic.keys():
248 filepath = os.path.join(self.build_area_root, _file)
249
250 signatures = self.sign_dic[_file]
251
252 ( _, rest) = os.path.splitdrive(filepath)
253 (frontpath, rest) = os.path.split(rest)
254
255 if (signatures[0] != signatures[1]):
256 if (signatures[0] != "") and (signatures[1] != ""):
257 if os.path.exists(filepath):
258 archive_txt.write(_file + "\n")
259 else:
260 if (signatures[1] != ""):
261 if os.path.exists(filepath):
262 archive_txt.write(_file + "\n")
263 else:
264 delete_list.append(filepath)
265
266 archive_txt.close()
267
268 splitter = MD5SignatureBuilder('', no_of_zips, self.temp_path, '', os.path.join(self.temp_path, 'create_zips.txt'))
269 splitter.split_file_list()
270
271 os.chdir(self.build_area_root)
272
273 (frontpath, rest) = os.path.split(zip_file)
274 stages = buildtools.CommandList()
275
276 for i in range(no_of_zips):
277 md5_dir = os.path.join(self.temp_path, "md5_temp")
278 path = os.path.join(md5_dir, os.path.join(str(no_of_zips), str(i) + '.txt'))
279 output = os.path.join(frontpath, rest.replace(".zip", "_part_%sof%s.zip" % (str(i+1), str(no_of_zips))))
280
281 cmd = buildtools.Command('7za.exe', self.build_area_root)
282 cmd.addArg('a')
283
284 cmd.addArg('-tzip')
285 cmd.addArg(output)
286 cmd.addArg('@' + path)
287
288 stages.addCommand(cmd)
289
290 writer = buildtools.AntWriter(ant_file)
291 writer.write(stages)
292
293 fp_delete = open(delete_list_file, 'w')
294 fp_delete.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
295 fp_delete.write("<updateinstructions>\n")
296 for i in delete_list:
297 fp_delete.write("<deletefileaction target=\"" + i[2:] + "\"/>\n")
298 fp_delete.write("</updateinstructions>\n")
299 fp_delete.close()
300
301
303 """ class to handle signature comparison"""
305 """ constructor"""
306 dict.__init__(self)
307
309 """ compare the tree structures"""
310 string = ""
311
312 both = False
313 only_old = False
314 only_new = False
315 for filename in self.keys():
316 signatures = self[filename]
317 if signatures[0] == signatures[1]:
318 both = True
319 elif (signatures[0] != "") and (signatures[1] != ""):
320 both = False
321 else:
322 if (signatures[1] != ""):
323 only_old = True
324 else:
325 only_new = True
326
327 string = string + filename + " " + str(both) + " " + " " + str(only_old) + " " + str(only_new) + " " + self[filename][0] + " " + self[filename][1] + "\n"
328
329 return string
330