Module delta_zip |
[hide private]
[frames] | no frames]

# Source Code for Module delta_zip

      1  #============================================================================
      2  #Name        : delta_zip.py
      3  #Part of     : Helium
      4
      5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
      6  #All rights reserved.
      7  #This component and the accompanying materials are made available
      8  #under the terms of the License "Eclipse Public License v1.0"
      9  #which accompanies this distribution, and is available
     10  #at the URL "http://www.eclipse.org/legal/epl-v10.html".
     11  #
     12  #Initial Contributors:
     13  #Nokia Corporation - initial contribution.
     14  #
     15  #Contributors:
     16  #
     17  #Description:
     18  #===============================================================================
     19
     20  import os
     21  import re
     22  import fileutils
     23  import buildtools
     24  import logging
     25
     26  _logger = logging.getLogger('delta_zip')
     27  logging.basicConfig(level=logging.INFO)
     28
     29 -class MD5SignatureBuilder(object):
     30      """ MD5 CRC creation base class"""
     31 -    def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files):
     32          """constructor"""
     33          if not build_area_root.endswith(os.sep):
     34              self.build_area_root = build_area_root + os.sep
     35          self.nb_split = nb_split
     36          self.temp_dir = temp_dir
     37          self.exclude_dirs = exclude_dirs
     38          self.list_of_files = list_of_files
     39
     40 -    def create_file_list(self):
     41          """Create list of files (was list_files.pl)"""
     42          #list_of_files_symbol = os.path.join(self.temp_dir, "list_files_sym.txt")
     43
     44          if not os.path.exists(self.temp_dir):
     45              os.mkdir(self.temp_dir)
     46
     47          fp_filelist = open(self.list_of_files, 'w')
     48          #fp_filelist_sym = open(list_of_files_symbol, 'w')
     49
     50          scanner = fileutils.FileScanner(self.build_area_root)
     51          scanner.add_include('**')
     52
     53          for _dir in self.exclude_dirs.split(','):
     54              _dir = _dir.replace(self.build_area_root, "")
     55              scanner.add_exclude(_dir)
     56
     57          for path in scanner.scan():
     58              if (not os.path.isdir(path)) or (os.path.isdir(path) and (os.listdir(path) != []  and os.listdir(path) != ['.svn'])):
     59                  path = path.replace(self.build_area_root.lower(), "")
     60                  fp_filelist.write(path + "\n")
     61
     62 -    def split_file_list(self):
     63          """Split the list of files for parallelalisation"""
     64          md5_dir = os.path.join(self.temp_dir, "md5_temp")
     65          self.dest_dir = os.path.join(md5_dir, str(self.nb_split))
     66          if not os.path.exists(self.dest_dir):
     67              os.makedirs(self.dest_dir)
     68          fp_split = []
     69          #Open files
     70          #White list_of_lists.txt
     71          self.list_of_lists = self.dest_dir + "/list_of_lists.txt"
     72          fp_list_of_lists = open(self.list_of_lists, 'w')
     73          for i in range(self.nb_split):
     74              filename = self.dest_dir + "/" + str(i) + ".txt"
     75              _fp = open(filename, 'w')
     76              fp_split.append(_fp)
     77              #Write in list_of_lists
     78              fp_list_of_lists.write(filename + "\n")
     79
     80          #Write in files
     81          fp_read = open(self.list_of_files, 'r')
     82          line = fp_read.readline()
     83          line_number = 0
     84          while(line != ""):
     85              fp_split[line_number % len(fp_split)].write(line)
     86              line = fp_read.readline()
     87              line_number += 1
     88
     89          fp_list_of_lists.close()
     90          fp_read.close()
     91          for _fp in fp_split:
     92              _fp.close()
     93
     94 -    def create_command_list(self):
     95          """ create the command to run evalid on each file in the list of files"""
     96          liste = buildtools.CommandList()
     97
     98          #tools_dir = os.path.join(self.build_area_root, "/epoc32/tools")
     99
    100          for i in range(self.nb_split):
    101              #liste.addCommand(buildtools.Command("perl -I"+tools_dir, tools_dir, [os.path.join(tools_dir,"evalid_multiple.pl"), "-f", self.__get_partial_input_file_name(i), "> "+self.__get_partial_signature_file_name(i) ]))
    102              liste.addCommand(buildtools.Command("evalid", os.sep, ["", "-f", self.__get_partial_input_file_name(i) + " "+self.build_area_root, self.__get_partial_signature_file_name(i) ]))
    103
    104          return liste
    105
    106 -    def __get_partial_input_file_name(self, _nb):
    107          """ get the input file name string as has been created so far and add .txt to it"""
    108          return os.path.join(self.dest_dir, str(_nb) + ".txt")
    109
    110 -    def __get_partial_signature_file_name(self, _nb):
    111          """ get the signature file name string as has been created so far and add .md5 to it"""
    112          return os.path.join(self.dest_dir, str(_nb) + ".md5")
    113
    114 -    def concatenate_signature_files(self, signature_file):
    115          """ concatenate all the files with the MD5 CRC in """
    116          # Get header
    117          _fp = open(self.__get_partial_signature_file_name(0), 'r')
    118          line = ""
    119          header_temp = ""
    120          header = ""
    121          while (re.search(r'(\S+).*MD5=(\S+)', line) == None):
    122              header_temp = header_temp + line
    123              line = _fp.readline()
    124
    125          for line in header_temp.splitlines():
    126              if re.match(r'Directory:.*', line):
    127                  line =  "Directory:" + self.build_area_root
    128              if re.match(r'FileList:.*', line):
    129                  line = "FileList:" + self.list_of_files
    130              header = header + line + "\n"
    131
    132          #re.sub(r'(Directory:).*\n', "\1"+self.build_area_root, header)
    133          #re.sub(r'(FileList:).*\n', "\1"+self.list_of_files, header)
    134
    135          header_size = len(header.splitlines())
    136
    137          fp_md5_signatures_file = open(signature_file, 'w')
    138          fp_md5_signatures_file.write(header)
    139          for i in range(self.nb_split):
    140              _fp = open(self.__get_partial_signature_file_name(i), 'r')
    141
    142              for i in range(header_size): # Skip header
    143                  _fp.readline()
    144
    145              fp_md5_signatures_file.write(_fp.read())
    146              _fp.close()
    147          fp_md5_signatures_file.close()
    148
    149 -    def write_build_file(self):
    150          """ create the file of the list of files to have a CRC created"""
    151          self.create_file_list()
    152          self.split_file_list()
    153          self.create_build_file()
    154
    155 -    def create_build_file(self):
    156          """ there should always be an overloaded version of this method in sub-classes"""
    157          raise NotImplementedError()
    158
    159 -    def build(self, signature_file):
    160          """create the list of files generate the MD5 CRC and create the final file with CRCs in"""
    161          self.write_build_file()
    162          self.compute_evalid_MD5()
    163          self.concatenate_signature_files(signature_file)
    164
    165 -    def compute_evalid_MD5(self):
    166          """ there should always be an overlaoded version in the methos sub-class"""
    167          raise NotImplementedError()
    168
    169 -class MD5SignatureBuilderEBS(MD5SignatureBuilder):
    170      """ build the MD5 CRCs for all the files in the list of files"""
    171 -    def create_build_file(self):
    172          """Create EBS XML"""
    173          liste = self.create_command_list()
    174          self.makefile = self.dest_dir + "/ebs.xml"
    175          buildtools.convert(liste, self.makefile, "ebs")
    176
    177 -    def compute_evalid_MD5(self):
    178          """Compute MD5 using the requested parallel build system"""
    179          os.chdir(self.build_area_root)
    180          os.system("perl -I%HELIUM_HOME%/tools/common/packages %HELIUM_HOME%/tools/compile/buildjob.pl -d " + self.makefile + " -l " + os.path.join(self.dest_dir, "md5.log") + " -n " + str(int(os.environ['NUMBER_OF_PROCESSORS'])*2))
    181
    182  """
    183  Run the delta zipping over the EC build system
    184  """
    185 -class MD5SignatureBuilderEC(MD5SignatureBuilder):
    186      """ The MD5 CRC creation for delta zippinf for use on EC machines"""
    187 -    def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, ec_cluster_manager, ec_build_class, list_of_files):
    188          MD5SignatureBuilder.__init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files)
    189          self.ec_cluster_manager = ec_cluster_manager
    190          self.ec_build_class = ec_build_class
    191
    192 -    def create_build_file(self):
    193          """Create makefile"""
    194          liste = self.create_command_list()
    195          self.makefile = self.dest_dir + "/Makefile"
    196          buildtools.convert(liste, self.makefile, "make")
    197
    198 -    def compute_evalid_MD5(self):
    199          """Compute MD5 using the requested parallel build system"""
    200          root_path = os.environ['EMAKE_ROOT'] +";" + "c:\\apps;"
    201          os.chdir(self.build_area_root)
    202
    203          print "emake --emake-cm=" + self.ec_cluster_manager + " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+ " --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile
    204          os.system("emake --emake-cm=" + self.ec_cluster_manager + " --emake-annodetail=basic,history,file,waiting --emake-annofile="+self.temp_dir+"\\delta_zip_anno.xml"+ " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+" --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile)
    205
    206 -class DeltaZipBuilder(object):
    207      """methods to create the delta zip after all the prep"""
    208 -    def __init__(self, build_area_root, temp_path, old_md5_signature, new_md5_signature):
    209          self.build_area_root = os.path.join(build_area_root, os.sep)
    210          self.temp_path = temp_path
    211          self.old_md5_signature = old_md5_signature
    212          self.new_md5_signature = new_md5_signature
    213          self.sign_dic = SignaturesDict()
    214
    215 -    def __fill_signature_dict(self, signature_file, old_new):
    216          """ read each line of signature file search for .MD5"""
    217          _fp = open(signature_file, 'r')
    218          lines = _fp.read().splitlines()
    219          _fp.close()
    220          for line in lines:
    221              info = re.search(r'([ \S]+) TYPE=.*MD5=(\S+)', line)
    222              if info != None:
    223                  filename = info.group(1)
    224                  if not self.sign_dic.has_key(filename):
    225                      self.sign_dic[filename] = ["", ""]
    226                  self.sign_dic[filename][old_new] = info.group(2)
    227
    228 -    def create_delta_zip(self, zip_file, delete_list_file, no_of_zips, ant_file):
    229          """Create Delta zip and list of file to delete."""
    230
    231          self.__fill_signature_dict(self.old_md5_signature, 0)
    232          self.__fill_signature_dict(self.new_md5_signature, 1)
    233
    234          #fp_dic = open(zip_file + ".dic.txt", 'w')
    235          #fp_dic.write(str(self.sign_dic))
    236          #fp_dic.close()
    237
    238          delete_list = []
    239
    240          if not os.path.exists(os.path.dirname(delete_list_file)):
    241              os.mkdir(os.path.dirname(delete_list_file))
    242          if not os.path.exists(self.temp_path):
    243              os.mkdir(self.temp_path)
    244
    245          archive_txt = open(os.path.join(self.temp_path, 'create_zips.txt'), 'w')
    246
    247          for _file in self.sign_dic.keys():
    248              filepath = os.path.join(self.build_area_root, _file)
    249
    250              signatures = self.sign_dic[_file]
    251
    252              ( _, rest) = os.path.splitdrive(filepath)
    253              (frontpath, rest) = os.path.split(rest)
    254
    255              if (signatures[0] != signatures[1]):  #File changed between the 2 BAs
    256                  if (signatures[0] != "") and  (signatures[1] != ""): # File is present in both BAs and has changed
    257                      if os.path.exists(filepath): # File could have been deleting after running 'build-md5':
    258                          archive_txt.write(_file + "\n")
    259                  else:
    260                      if (signatures[1] != ""): # New file
    261                          if os.path.exists(filepath):
    262                              archive_txt.write(_file + "\n")
    263                      else: # Deleted file
    264                          delete_list.append(filepath)
    265
    266          archive_txt.close()
    267
    268          splitter = MD5SignatureBuilder('', no_of_zips, self.temp_path, '', os.path.join(self.temp_path, 'create_zips.txt'))
    269          splitter.split_file_list()
    270
    271          os.chdir(self.build_area_root)
    272
    273          (frontpath, rest) = os.path.split(zip_file)
    274          stages = buildtools.CommandList()
    275
    276          for i in range(no_of_zips):
    277              md5_dir = os.path.join(self.temp_path, "md5_temp")
    278              path = os.path.join(md5_dir, os.path.join(str(no_of_zips), str(i) + '.txt'))
    279              output = os.path.join(frontpath, rest.replace(".zip", "_part_%sof%s.zip" % (str(i+1), str(no_of_zips))))
    280
    281              cmd = buildtools.Command('7za.exe', self.build_area_root)
    282              cmd.addArg('a')
    283              # Set the format to be zip-compatible
    284              cmd.addArg('-tzip')
    285              cmd.addArg(output)
    286              cmd.addArg('@' + path)
    287
    288              stages.addCommand(cmd)
    289
    290          writer = buildtools.AntWriter(ant_file)
    291          writer.write(stages)
    292
    293          fp_delete = open(delete_list_file, 'w')
    294          fp_delete.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
    295          fp_delete.write("<updateinstructions>\n")
    296          for i in delete_list:
    297              fp_delete.write("<deletefileaction target=\"" + i[2:] + "\"/>\n")
    298          fp_delete.write("</updateinstructions>\n")
    299          fp_delete.close()
    300
    301
    302 -class SignaturesDict(dict):
    303      """ class to handle signature comparison"""
    304 -    def __init__(self):
    305          """ constructor"""
    306          dict.__init__(self)
    307
    308 -    def __str__(self):
    309          """ compare the tree structures"""
    310          string = ""
    311          #o = OldNewBA()
    312          both = False
    313          only_old = False
    314          only_new = False
    315          for filename in self.keys():
    316              signatures = self[filename]
    317              if signatures[0] == signatures[1]: #File did not change
    318                  both = True
    319              elif (signatures[0] != "") and  (signatures[1] != ""): # File is present in both BAs and has changed
    320                  both = False
    321              else:
    322                  if (signatures[1] != ""): # New file
    323                      only_old = True
    324                  else: # Deleted file
    325                      only_new = True
    326
    327              string = string + filename + " " + str(both) + " " + " " + str(only_old) + " " + str(only_new) + " " + self[filename][0] + " " + self[filename][1] + "\n"
    328
    329          return string
    330