diff -r 7685cec9fd3c -r f2ddfa555b0f doc/api/python/delta_zip-pysrc.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/api/python/delta_zip-pysrc.html Fri Sep 11 11:54:49 2009 +0100 @@ -0,0 +1,1224 @@ + + + + + delta_zip + + + + + + + + + + + + + + + + + + + + +
   Trees      Indices      Help   
+ + + + + +
+ + Module delta_zip + + + + + + +
[hide private]
[frames] | no frames]
+
+

Source Code for Module delta_zip

+
+  1  #============================================================================  
+  2  #Name        : delta_zip.py  
+  3  #Part of     : Helium  
+  4   
+  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
+  6  #All rights reserved. 
+  7  #This component and the accompanying materials are made available 
+  8  #under the terms of the License "Eclipse Public License v1.0" 
+  9  #which accompanies this distribution, and is available 
+ 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
+ 11  # 
+ 12  #Initial Contributors: 
+ 13  #Nokia Corporation - initial contribution. 
+ 14  # 
+ 15  #Contributors: 
+ 16  # 
+ 17  #Description: 
+ 18  #=============================================================================== 
+ 19   
+ 20  import os 
+ 21  import re 
+ 22  import fileutils 
+ 23  import buildtools 
+ 24  import logging 
+ 25   
+ 26  _logger = logging.getLogger('delta_zip') 
+ 27  logging.basicConfig(level=logging.INFO) 
+ 28   
+
 29 -class MD5SignatureBuilder(object): 
+
 30      """ MD5 CRC creation base class""" 
+
 31 -    def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files): 
+
 32          """constructor""" 
+ 33          if not build_area_root.endswith(os.sep): 
+ 34              self.build_area_root = build_area_root + os.sep 
+ 35          self.nb_split = nb_split 
+ 36          self.temp_dir = temp_dir 
+ 37          self.exclude_dirs = exclude_dirs 
+ 38          self.list_of_files = list_of_files 
+
 39           
+
 40 -    def create_file_list(self): 
+
 41          """Create list of files (was list_files.pl)""" 
+ 42          #list_of_files_symbol = os.path.join(self.temp_dir, "list_files_sym.txt") 
+ 43           
+ 44          if not os.path.exists(self.temp_dir): 
+ 45              os.mkdir(self.temp_dir) 
+ 46         
+ 47          fp_filelist = open(self.list_of_files, 'w') 
+ 48          #fp_filelist_sym = open(list_of_files_symbol, 'w') 
+ 49                           
+ 50          scanner = fileutils.FileScanner(self.build_area_root) 
+ 51          scanner.add_include('**') 
+ 52           
+ 53          for _dir in self.exclude_dirs.split(','): 
+ 54              _dir = _dir.replace(self.build_area_root, "") 
+ 55              scanner.add_exclude(_dir) 
+ 56   
+ 57          for path in scanner.scan(): 
+ 58              if (not os.path.isdir(path)) or (os.path.isdir(path) and (os.listdir(path) != []  and os.listdir(path) != ['.svn'])): 
+ 59                  path = path.replace(self.build_area_root.lower(), "") 
+ 60                  fp_filelist.write(path + "\n") 
+
 61               
+
 62 -    def split_file_list(self): 
+
 63          """Split the list of files for parallelalisation""" 
+ 64          md5_dir = os.path.join(self.temp_dir, "md5_temp") 
+ 65          self.dest_dir = os.path.join(md5_dir, str(self.nb_split)) 
+ 66          if not os.path.exists(self.dest_dir): 
+ 67              os.makedirs(self.dest_dir) 
+ 68          fp_split = [] 
+ 69          #Open files 
+ 70          #White list_of_lists.txt 
+ 71          self.list_of_lists = self.dest_dir + "/list_of_lists.txt" 
+ 72          fp_list_of_lists = open(self.list_of_lists, 'w') 
+ 73          for i in range(self.nb_split): 
+ 74              filename = self.dest_dir + "/" + str(i) + ".txt" 
+ 75              _fp = open(filename, 'w') 
+ 76              fp_split.append(_fp) 
+ 77              #Write in list_of_lists 
+ 78              fp_list_of_lists.write(filename + "\n") 
+ 79               
+ 80          #Write in files 
+ 81          fp_read = open(self.list_of_files, 'r')  
+ 82          line = fp_read.readline() 
+ 83          line_number = 0 
+ 84          while(line != ""): 
+ 85              fp_split[line_number % len(fp_split)].write(line) 
+ 86              line = fp_read.readline() 
+ 87              line_number += 1 
+ 88           
+ 89          fp_list_of_lists.close()     
+ 90          fp_read.close() 
+ 91          for _fp in fp_split: 
+ 92              _fp.close() 
+
 93               
+
 94 -    def create_command_list(self):         
+
 95          """ create the command to run evalid on each file in the list of files""" 
+ 96          liste = buildtools.CommandList() 
+ 97           
+ 98          #tools_dir = os.path.join(self.build_area_root, "/epoc32/tools") 
+ 99           
+100          for i in range(self.nb_split): 
+101              #liste.addCommand(buildtools.Command("perl -I"+tools_dir, tools_dir, [os.path.join(tools_dir,"evalid_multiple.pl"), "-f", self.__get_partial_input_file_name(i), "> "+self.__get_partial_signature_file_name(i) ])) 
+102              liste.addCommand(buildtools.Command("evalid", os.sep, ["", "-f", self.__get_partial_input_file_name(i) + " "+self.build_area_root, self.__get_partial_signature_file_name(i) ])) 
+103               
+104          return liste 
+
105       
+
106 -    def __get_partial_input_file_name(self, _nb): 
+
107          """ get the input file name string as has been created so far and add .txt to it""" 
+108          return os.path.join(self.dest_dir, str(_nb) + ".txt") 
+
109               
+
110 -    def __get_partial_signature_file_name(self, _nb): 
+
111          """ get the signature file name string as has been created so far and add .md5 to it""" 
+112          return os.path.join(self.dest_dir, str(_nb) + ".md5") 
+
113           
+
114 -    def concatenate_signature_files(self, signature_file): 
+
115          """ concatenate all the files with the MD5 CRC in """ 
+116          # Get header 
+117          _fp = open(self.__get_partial_signature_file_name(0), 'r') 
+118          line = "" 
+119          header_temp = "" 
+120          header = "" 
+121          while (re.search(r'(\S+).*MD5=(\S+)', line) == None): 
+122              header_temp = header_temp + line 
+123              line = _fp.readline() 
+124           
+125          for line in header_temp.splitlines(): 
+126              if re.match(r'Directory:.*', line): 
+127                  line =  "Directory:" + self.build_area_root 
+128              if re.match(r'FileList:.*', line): 
+129                  line = "FileList:" + self.list_of_files 
+130              header = header + line + "\n" 
+131           
+132          #re.sub(r'(Directory:).*\n', "\1"+self.build_area_root, header) 
+133          #re.sub(r'(FileList:).*\n', "\1"+self.list_of_files, header) 
+134               
+135          header_size = len(header.splitlines()) 
+136           
+137          fp_md5_signatures_file = open(signature_file, 'w') 
+138          fp_md5_signatures_file.write(header) 
+139          for i in range(self.nb_split): 
+140              _fp = open(self.__get_partial_signature_file_name(i), 'r') 
+141   
+142              for i in range(header_size): # Skip header 
+143                  _fp.readline() 
+144               
+145              fp_md5_signatures_file.write(_fp.read()) 
+146              _fp.close() 
+147          fp_md5_signatures_file.close() 
+
148   
+
149 -    def write_build_file(self): 
+
150          """ create the file of the list of files to have a CRC created""" 
+151          self.create_file_list() 
+152          self.split_file_list() 
+153          self.create_build_file() 
+
154       
+
155 -    def create_build_file(self): 
+
156          """ there should always be an overloaded version of this method in sub-classes""" 
+157          raise NotImplementedError() 
+
158       
+
159 -    def build(self, signature_file): 
+
160          """create the list of files generate the MD5 CRC and create the final file with CRCs in""" 
+161          self.write_build_file() 
+162          self.compute_evalid_MD5() 
+163          self.concatenate_signature_files(signature_file) 
+
164               
+
165 -    def compute_evalid_MD5(self): 
+
166          """ there should always be an overlaoded version in the methos sub-class""" 
+167          raise NotImplementedError() 
+
168       
+
169 -class MD5SignatureBuilderEBS(MD5SignatureBuilder): 
+
170      """ build the MD5 CRCs for all the files in the list of files""" 
+
171 -    def create_build_file(self): 
+
172          """Create EBS XML""" 
+173          liste = self.create_command_list() 
+174          self.makefile = self.dest_dir + "/ebs.xml" 
+175          buildtools.convert(liste, self.makefile, "ebs") 
+
176   
+
177 -    def compute_evalid_MD5(self): 
+
178          """Compute MD5 using the requested parallel build system""" 
+179          os.chdir(self.build_area_root) 
+180          os.system("perl -I%HELIUM_HOME%/tools/common/packages %HELIUM_HOME%/tools/compile/buildjob.pl -d " + self.makefile + " -l " + os.path.join(self.dest_dir, "md5.log") + " -n " + str(int(os.environ['NUMBER_OF_PROCESSORS'])*2)) 
+
181   
+182  """ 
+183  Run the delta zipping over the EC build system 
+184  """ 
+
185 -class MD5SignatureBuilderEC(MD5SignatureBuilder): 
+
186      """ The MD5 CRC creation for delta zippinf for use on EC machines""" 
+
187 -    def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, ec_cluster_manager, ec_build_class, list_of_files): 
+
188          MD5SignatureBuilder.__init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files) 
+189          self.ec_cluster_manager = ec_cluster_manager 
+190          self.ec_build_class = ec_build_class 
+
191       
+
192 -    def create_build_file(self): 
+
193          """Create makefile""" 
+194          liste = self.create_command_list() 
+195          self.makefile = self.dest_dir + "/Makefile" 
+196          buildtools.convert(liste, self.makefile, "make") 
+
197   
+
198 -    def compute_evalid_MD5(self): 
+
199          """Compute MD5 using the requested parallel build system""" 
+200          root_path = os.environ['EMAKE_ROOT'] +";" + "c:\\apps;" 
+201          os.chdir(self.build_area_root) 
+202           
+203          print "emake --emake-cm=" + self.ec_cluster_manager + " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+ " --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile 
+204          os.system("emake --emake-cm=" + self.ec_cluster_manager + " --emake-annodetail=basic,history,file,waiting --emake-annofile="+self.temp_dir+"\\delta_zip_anno.xml"+ " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+" --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile) 
+
205   
+
206 -class DeltaZipBuilder(object): 
+
207      """methods to create the delta zip after all the prep""" 
+
208 -    def __init__(self, build_area_root, temp_path, old_md5_signature, new_md5_signature): 
+
209          self.build_area_root = os.path.join(build_area_root, os.sep) 
+210          self.temp_path = temp_path 
+211          self.old_md5_signature = old_md5_signature 
+212          self.new_md5_signature = new_md5_signature 
+213          self.sign_dic = SignaturesDict() 
+
214           
+
215 -    def __fill_signature_dict(self, signature_file, old_new): 
+
216          """ read each line of signature file search for .MD5""" 
+217          _fp = open(signature_file, 'r') 
+218          lines = _fp.read().splitlines() 
+219          _fp.close() 
+220          for line in lines: 
+221              info = re.search(r'([ \S]+) TYPE=.*MD5=(\S+)', line) 
+222              if info != None: 
+223                  filename = info.group(1) 
+224                  if not self.sign_dic.has_key(filename): 
+225                      self.sign_dic[filename] = ["", ""] 
+226                  self.sign_dic[filename][old_new] = info.group(2)                 
+
227       
+
228 -    def create_delta_zip(self, zip_file, delete_list_file, no_of_zips, ant_file): 
+
229          """Create Delta zip and list of file to delete.""" 
+230           
+231          self.__fill_signature_dict(self.old_md5_signature, 0) 
+232          self.__fill_signature_dict(self.new_md5_signature, 1) 
+233           
+234          #fp_dic = open(zip_file + ".dic.txt", 'w') 
+235          #fp_dic.write(str(self.sign_dic)) 
+236          #fp_dic.close() 
+237           
+238          delete_list = [] 
+239           
+240          if not os.path.exists(os.path.dirname(delete_list_file)): 
+241              os.mkdir(os.path.dirname(delete_list_file)) 
+242          if not os.path.exists(self.temp_path): 
+243              os.mkdir(self.temp_path) 
+244           
+245          archive_txt = open(os.path.join(self.temp_path, 'create_zips.txt'), 'w') 
+246   
+247          for _file in self.sign_dic.keys(): 
+248              filepath = os.path.join(self.build_area_root, _file) 
+249               
+250              signatures = self.sign_dic[_file] 
+251               
+252              ( _, rest) = os.path.splitdrive(filepath) 
+253              (frontpath, rest) = os.path.split(rest) 
+254               
+255              if (signatures[0] != signatures[1]):  #File changed between the 2 BAs 
+256                  if (signatures[0] != "") and  (signatures[1] != ""): # File is present in both BAs and has changed 
+257                      if os.path.exists(filepath): # File could have been deleting after running 'build-md5': 
+258                          archive_txt.write(_file + "\n") 
+259                  else: 
+260                      if (signatures[1] != ""): # New file 
+261                          if os.path.exists(filepath): 
+262                              archive_txt.write(_file + "\n") 
+263                      else: # Deleted file 
+264                          delete_list.append(filepath) 
+265           
+266          archive_txt.close() 
+267           
+268          splitter = MD5SignatureBuilder('', no_of_zips, self.temp_path, '', os.path.join(self.temp_path, 'create_zips.txt')) 
+269          splitter.split_file_list() 
+270           
+271          os.chdir(self.build_area_root) 
+272           
+273          (frontpath, rest) = os.path.split(zip_file) 
+274          stages = buildtools.CommandList() 
+275           
+276          for i in range(no_of_zips): 
+277              md5_dir = os.path.join(self.temp_path, "md5_temp") 
+278              path = os.path.join(md5_dir, os.path.join(str(no_of_zips), str(i) + '.txt')) 
+279              output = os.path.join(frontpath, rest.replace(".zip", "_part_%sof%s.zip" % (str(i+1), str(no_of_zips)))) 
+280               
+281              cmd = buildtools.Command('7za.exe', self.build_area_root) 
+282              cmd.addArg('a') 
+283              # Set the format to be zip-compatible 
+284              cmd.addArg('-tzip') 
+285              cmd.addArg(output) 
+286              cmd.addArg('@' + path) 
+287               
+288              stages.addCommand(cmd) 
+289   
+290          writer = buildtools.AntWriter(ant_file) 
+291          writer.write(stages) 
+292   
+293          fp_delete = open(delete_list_file, 'w') 
+294          fp_delete.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n") 
+295          fp_delete.write("<updateinstructions>\n") 
+296          for i in delete_list: 
+297              fp_delete.write("<deletefileaction target=\"" + i[2:] + "\"/>\n") 
+298          fp_delete.write("</updateinstructions>\n") 
+299          fp_delete.close() 
+
300           
+301                   
+
302 -class SignaturesDict(dict): 
+
303      """ class to handle signature comparison""" 
+
304 -    def __init__(self): 
+
305          """ constructor""" 
+306          dict.__init__(self) 
+
307       
+
308 -    def __str__(self): 
+
309          """ compare the tree structures""" 
+310          string = "" 
+311          #o = OldNewBA() 
+312          both = False 
+313          only_old = False 
+314          only_new = False 
+315          for filename in self.keys(): 
+316              signatures = self[filename] 
+317              if signatures[0] == signatures[1]: #File did not change 
+318                  both = True 
+319              elif (signatures[0] != "") and  (signatures[1] != ""): # File is present in both BAs and has changed 
+320                  both = False 
+321              else: 
+322                  if (signatures[1] != ""): # New file 
+323                      only_old = True 
+324                  else: # Deleted file 
+325                      only_new = True 
+326               
+327              string = string + filename + " " + str(both) + " " + " " + str(only_old) + " " + str(only_new) + " " + self[filename][0] + " " + self[filename][1] + "\n" 
+328           
+329          return string 
+
330   
+
+
+ + + + + + + + + + + + + + + +
   Trees      Indices      Help   
+ + + + + +
+ Generated by Epydoc 3.0beta1 on Wed Sep 09 13:44:24 2009 + + http://epydoc.sourceforge.net +
+ + + + +