Module delta_zip
[hide private]
[frames] | no frames]

Source Code for Module delta_zip

  1  #============================================================================  
  2  #Name        : delta_zip.py  
  3  #Part of     : Helium  
  4   
  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
  6  #All rights reserved. 
  7  #This component and the accompanying materials are made available 
  8  #under the terms of the License "Eclipse Public License v1.0" 
  9  #which accompanies this distribution, and is available 
 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
 11  # 
 12  #Initial Contributors: 
 13  #Nokia Corporation - initial contribution. 
 14  # 
 15  #Contributors: 
 16  # 
 17  #Description: 
 18  #=============================================================================== 
 19   
 20  import os 
 21  import re 
 22  import fileutils 
 23  import buildtools 
 24  import logging 
 25   
 26  _logger = logging.getLogger('delta_zip') 
 27  logging.basicConfig(level=logging.INFO) 
 28   
29 -class MD5SignatureBuilder(object):
30 """ MD5 CRC creation base class"""
31 - def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files):
32 """constructor""" 33 if not build_area_root.endswith(os.sep): 34 self.build_area_root = build_area_root + os.sep 35 self.nb_split = nb_split 36 self.temp_dir = temp_dir 37 self.exclude_dirs = exclude_dirs 38 self.list_of_files = list_of_files
39
40 - def create_file_list(self):
41 """Create list of files (was list_files.pl)""" 42 #list_of_files_symbol = os.path.join(self.temp_dir, "list_files_sym.txt") 43 44 if not os.path.exists(self.temp_dir): 45 os.mkdir(self.temp_dir) 46 47 fp_filelist = open(self.list_of_files, 'w') 48 #fp_filelist_sym = open(list_of_files_symbol, 'w') 49 50 scanner = fileutils.FileScanner(self.build_area_root) 51 scanner.add_include('**') 52 53 for _dir in self.exclude_dirs.split(','): 54 _dir = _dir.replace(self.build_area_root, "") 55 scanner.add_exclude(_dir) 56 57 for path in scanner.scan(): 58 if (not os.path.isdir(path)) or (os.path.isdir(path) and (os.listdir(path) != [] and os.listdir(path) != ['.svn'])): 59 path = path.replace(self.build_area_root.lower(), "") 60 fp_filelist.write(path + "\n")
61
62 - def split_file_list(self):
63 """Split the list of files for parallelalisation""" 64 md5_dir = os.path.join(self.temp_dir, "md5_temp") 65 self.dest_dir = os.path.join(md5_dir, str(self.nb_split)) 66 if not os.path.exists(self.dest_dir): 67 os.makedirs(self.dest_dir) 68 fp_split = [] 69 #Open files 70 #White list_of_lists.txt 71 self.list_of_lists = self.dest_dir + "/list_of_lists.txt" 72 fp_list_of_lists = open(self.list_of_lists, 'w') 73 for i in range(self.nb_split): 74 filename = self.dest_dir + "/" + str(i) + ".txt" 75 _fp = open(filename, 'w') 76 fp_split.append(_fp) 77 #Write in list_of_lists 78 fp_list_of_lists.write(filename + "\n") 79 80 #Write in files 81 fp_read = open(self.list_of_files, 'r') 82 line = fp_read.readline() 83 line_number = 0 84 while(line != ""): 85 fp_split[line_number % len(fp_split)].write(line) 86 line = fp_read.readline() 87 line_number += 1 88 89 fp_list_of_lists.close() 90 fp_read.close() 91 for _fp in fp_split: 92 _fp.close()
93
94 - def create_command_list(self):
95 """ create the command to run evalid on each file in the list of files""" 96 liste = buildtools.CommandList() 97 98 #tools_dir = os.path.join(self.build_area_root, "/epoc32/tools") 99 100 for i in range(self.nb_split): 101 #liste.addCommand(buildtools.Command("perl -I"+tools_dir, tools_dir, [os.path.join(tools_dir,"evalid_multiple.pl"), "-f", self.__get_partial_input_file_name(i), "> "+self.__get_partial_signature_file_name(i) ])) 102 liste.addCommand(buildtools.Command("evalid", os.sep, ["", "-f", self.__get_partial_input_file_name(i) + " "+self.build_area_root, self.__get_partial_signature_file_name(i) ])) 103 104 return liste
105
106 - def __get_partial_input_file_name(self, _nb):
107 """ get the input file name string as has been created so far and add .txt to it""" 108 return os.path.join(self.dest_dir, str(_nb) + ".txt")
109
110 - def __get_partial_signature_file_name(self, _nb):
111 """ get the signature file name string as has been created so far and add .md5 to it""" 112 return os.path.join(self.dest_dir, str(_nb) + ".md5")
113
114 - def concatenate_signature_files(self, signature_file):
115 """ concatenate all the files with the MD5 CRC in """ 116 # Get header 117 _fp = open(self.__get_partial_signature_file_name(0), 'r') 118 line = "" 119 header_temp = "" 120 header = "" 121 while (re.search(r'(\S+).*MD5=(\S+)', line) == None): 122 header_temp = header_temp + line 123 line = _fp.readline() 124 125 for line in header_temp.splitlines(): 126 if re.match(r'Directory:.*', line): 127 line = "Directory:" + self.build_area_root 128 if re.match(r'FileList:.*', line): 129 line = "FileList:" + self.list_of_files 130 header = header + line + "\n" 131 132 #re.sub(r'(Directory:).*\n', "\1"+self.build_area_root, header) 133 #re.sub(r'(FileList:).*\n', "\1"+self.list_of_files, header) 134 135 header_size = len(header.splitlines()) 136 137 fp_md5_signatures_file = open(signature_file, 'w') 138 fp_md5_signatures_file.write(header) 139 for i in range(self.nb_split): 140 _fp = open(self.__get_partial_signature_file_name(i), 'r') 141 142 for i in range(header_size): # Skip header 143 _fp.readline() 144 145 fp_md5_signatures_file.write(_fp.read()) 146 _fp.close() 147 fp_md5_signatures_file.close()
148
149 - def write_build_file(self):
150 """ create the file of the list of files to have a CRC created""" 151 self.create_file_list() 152 self.split_file_list() 153 self.create_build_file()
154
155 - def create_build_file(self):
156 """ there should always be an overloaded version of this method in sub-classes""" 157 raise NotImplementedError()
158
159 - def build(self, signature_file):
160 """create the list of files generate the MD5 CRC and create the final file with CRCs in""" 161 self.write_build_file() 162 self.compute_evalid_MD5() 163 self.concatenate_signature_files(signature_file)
164
165 - def compute_evalid_MD5(self):
166 """ there should always be an overlaoded version in the methos sub-class""" 167 raise NotImplementedError()
168
169 -class MD5SignatureBuilderEBS(MD5SignatureBuilder):
170 """ build the MD5 CRCs for all the files in the list of files"""
171 - def create_build_file(self):
172 """Create EBS XML""" 173 liste = self.create_command_list() 174 self.makefile = self.dest_dir + "/ebs.xml" 175 buildtools.convert(liste, self.makefile, "ebs")
176
177 - def compute_evalid_MD5(self):
178 """Compute MD5 using the requested parallel build system""" 179 os.chdir(self.build_area_root) 180 os.system("perl -I%HELIUM_HOME%/tools/common/packages %HELIUM_HOME%/tools/compile/buildjob.pl -d " + self.makefile + " -l " + os.path.join(self.dest_dir, "md5.log") + " -n " + str(int(os.environ['NUMBER_OF_PROCESSORS'])*2))
181 182 """ 183 Run the delta zipping over the EC build system 184 """
185 -class MD5SignatureBuilderEC(MD5SignatureBuilder):
186 """ The MD5 CRC creation for delta zippinf for use on EC machines"""
187 - def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, ec_cluster_manager, ec_build_class, list_of_files):
188 MD5SignatureBuilder.__init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files) 189 self.ec_cluster_manager = ec_cluster_manager 190 self.ec_build_class = ec_build_class
191
192 - def create_build_file(self):
193 """Create makefile""" 194 liste = self.create_command_list() 195 self.makefile = self.dest_dir + "/Makefile" 196 buildtools.convert(liste, self.makefile, "make")
197
198 - def compute_evalid_MD5(self):
199 """Compute MD5 using the requested parallel build system""" 200 root_path = os.environ['EMAKE_ROOT'] +";" + "c:\\apps;" 201 os.chdir(self.build_area_root) 202 203 print "emake --emake-cm=" + self.ec_cluster_manager + " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+ " --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile 204 os.system("emake --emake-cm=" + self.ec_cluster_manager + " --emake-annodetail=basic,history,file,waiting --emake-annofile="+self.temp_dir+"\\delta_zip_anno.xml"+ " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+" --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile)
205
206 -class DeltaZipBuilder(object):
207 """methods to create the delta zip after all the prep"""
208 - def __init__(self, build_area_root, temp_path, old_md5_signature, new_md5_signature):
209 self.build_area_root = os.path.join(build_area_root, os.sep) 210 self.temp_path = temp_path 211 self.old_md5_signature = old_md5_signature 212 self.new_md5_signature = new_md5_signature 213 self.sign_dic = SignaturesDict()
214
215 - def __fill_signature_dict(self, signature_file, old_new):
216 """ read each line of signature file search for .MD5""" 217 _fp = open(signature_file, 'r') 218 lines = _fp.read().splitlines() 219 _fp.close() 220 for line in lines: 221 info = re.search(r'([ \S]+) TYPE=.*MD5=(\S+)', line) 222 if info != None: 223 filename = info.group(1) 224 if not self.sign_dic.has_key(filename): 225 self.sign_dic[filename] = ["", ""] 226 self.sign_dic[filename][old_new] = info.group(2)
227
228 - def create_delta_zip(self, zip_file, delete_list_file, no_of_zips, ant_file):
229 """Create Delta zip and list of file to delete.""" 230 231 self.__fill_signature_dict(self.old_md5_signature, 0) 232 self.__fill_signature_dict(self.new_md5_signature, 1) 233 234 #fp_dic = open(zip_file + ".dic.txt", 'w') 235 #fp_dic.write(str(self.sign_dic)) 236 #fp_dic.close() 237 238 delete_list = [] 239 240 if not os.path.exists(os.path.dirname(delete_list_file)): 241 os.mkdir(os.path.dirname(delete_list_file)) 242 if not os.path.exists(self.temp_path): 243 os.mkdir(self.temp_path) 244 245 archive_txt = open(os.path.join(self.temp_path, 'create_zips.txt'), 'w') 246 247 for _file in self.sign_dic.keys(): 248 filepath = os.path.join(self.build_area_root, _file) 249 250 signatures = self.sign_dic[_file] 251 252 ( _, rest) = os.path.splitdrive(filepath) 253 (frontpath, rest) = os.path.split(rest) 254 255 if (signatures[0] != signatures[1]): #File changed between the 2 BAs 256 if (signatures[0] != "") and (signatures[1] != ""): # File is present in both BAs and has changed 257 if os.path.exists(filepath): # File could have been deleting after running 'build-md5': 258 archive_txt.write(_file + "\n") 259 else: 260 if (signatures[1] != ""): # New file 261 if os.path.exists(filepath): 262 archive_txt.write(_file + "\n") 263 else: # Deleted file 264 delete_list.append(filepath) 265 266 archive_txt.close() 267 268 splitter = MD5SignatureBuilder('', no_of_zips, self.temp_path, '', os.path.join(self.temp_path, 'create_zips.txt')) 269 splitter.split_file_list() 270 271 os.chdir(self.build_area_root) 272 273 (frontpath, rest) = os.path.split(zip_file) 274 stages = buildtools.CommandList() 275 276 for i in range(no_of_zips): 277 md5_dir = os.path.join(self.temp_path, "md5_temp") 278 path = os.path.join(md5_dir, os.path.join(str(no_of_zips), str(i) + '.txt')) 279 output = os.path.join(frontpath, rest.replace(".zip", "_part_%sof%s.zip" % (str(i+1), str(no_of_zips)))) 280 281 cmd = buildtools.Command('7za.exe', self.build_area_root) 282 cmd.addArg('a') 283 # Set the format to be zip-compatible 284 cmd.addArg('-tzip') 285 cmd.addArg(output) 286 cmd.addArg('@' + path) 287 288 stages.addCommand(cmd) 289 290 writer = buildtools.AntWriter(ant_file) 291 writer.write(stages) 292 293 fp_delete = open(delete_list_file, 'w') 294 fp_delete.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n") 295 fp_delete.write("<updateinstructions>\n") 296 for i in delete_list: 297 fp_delete.write("<deletefileaction target=\"" + i[2:] + "\"/>\n") 298 fp_delete.write("</updateinstructions>\n") 299 fp_delete.close()
300 301
302 -class SignaturesDict(dict):
303 """ class to handle signature comparison"""
304 - def __init__(self):
305 """ constructor""" 306 dict.__init__(self)
307
308 - def __str__(self):
309 """ compare the tree structures""" 310 string = "" 311 #o = OldNewBA() 312 both = False 313 only_old = False 314 only_new = False 315 for filename in self.keys(): 316 signatures = self[filename] 317 if signatures[0] == signatures[1]: #File did not change 318 both = True 319 elif (signatures[0] != "") and (signatures[1] != ""): # File is present in both BAs and has changed 320 both = False 321 else: 322 if (signatures[1] != ""): # New file 323 only_old = True 324 else: # Deleted file 325 only_new = True 326 327 string = string + filename + " " + str(both) + " " + " " + str(only_old) + " " + str(only_new) + " " + self[filename][0] + " " + self[filename][1] + "\n" 328 329 return string
330