1 #============================================================================ 

     2 #Name        : delta_zip.py 

     3 #Part of     : Helium 

     4 

     5 #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).

     6 #All rights reserved.

     7 #This component and the accompanying materials are made available

     8 #under the terms of the License "Eclipse Public License v1.0"

     9 #which accompanies this distribution, and is available

    10 #at the URL "http://www.eclipse.org/legal/epl-v10.html".

    11 #

    12 #Initial Contributors:

    13 #Nokia Corporation - initial contribution.

    14 #

    15 #Contributors:

    16 #

    17 #Description:

    18 #===============================================================================

    19 """delta zip create the MD5 signature"""

    20 

    21 import os

    22 import shutil

    23 import re

    24 import fileutils

    25 import buildtools

    26 import logging

    27 

    28 _logger = logging.getLogger('delta_zip')

    29 logging.basicConfig(level=logging.INFO)

    30 

    31 class MD5SignatureBuilder(object):

    32     """ MD5 CRC creation base class"""

    33     def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files):

    34         """constructor"""

    35         if not build_area_root.endswith(os.sep):

    36             self.build_area_root = build_area_root + os.sep

    37         self.nb_split = int(nb_split)

    38         self.temp_dir = temp_dir

    39         self.exclude_dirs = exclude_dirs

    40         self.list_of_files = list_of_files

    41         

    42     def create_file_list(self):

    43         """Create list of files (was list_files.pl)"""

    44         #list_of_files_symbol = os.path.join(self.temp_dir, "list_files_sym.txt")

    45         

    46         if not os.path.exists(self.temp_dir):

    47             os.mkdir(self.temp_dir)

    48       

    49         fp_filelist = open(self.list_of_files, 'w')

    50         #fp_filelist_sym = open(list_of_files_symbol, 'w')

    51                         

    52         scanner = fileutils.FileScanner(self.build_area_root)

    53         scanner.add_include('**')

    54         

    55         for _dir in self.exclude_dirs.split(','):

    56             _dir = _dir.replace(self.build_area_root, "")

    57             scanner.add_exclude(_dir)

    58 

    59         for path in scanner.scan():

    60             if (not os.path.isdir(path)) or (os.path.isdir(path) and (os.listdir(path) != []  and os.listdir(path) != ['.svn'])):

    61                 (drive, _) = os.path.splitdrive(path)

    62                 path = path.replace(drive + os.sep, "")

    63                 fp_filelist.write(path + "\n")

    64             

    65     def split_file_list(self):

    66         """Split the list of files for parallelalisation"""

    67         md5_dir = os.path.join(self.temp_dir, "md5_temp")

    68         self.dest_dir = os.path.join(md5_dir, str(self.nb_split))

    69         if not os.path.exists(self.dest_dir):

    70             os.makedirs(self.dest_dir)

    71         fp_split = []

    72         #Open files

    73         #White list_of_lists.txt

    74         self.list_of_lists = self.dest_dir + "/list_of_lists.txt"

    75         fp_list_of_lists = open(self.list_of_lists, 'w')

    76         for i in range(self.nb_split):

    77             filename = self.dest_dir + "/" + str(i) + ".txt"

    78             _fp = open(filename, 'w')

    79             fp_split.append(_fp)

    80             #Write in list_of_lists

    81             fp_list_of_lists.write(filename + "\n")

    82             

    83         #Write in files

    84         fp_read = open(self.list_of_files, 'r') 

    85         line = fp_read.readline()

    86         line_number = 0

    87         while(line != ""):

    88             fp_split[line_number % len(fp_split)].write(line)

    89             line = fp_read.readline()

    90             line_number += 1

    91         

    92         fp_list_of_lists.close()    

    93         fp_read.close()

    94         for _fp in fp_split:

    95             _fp.close()

    96             

    97     def create_command_list(self):        

    98         """ create the command to run evalid on each file in the list of files"""

    99         liste = buildtools.CommandList()

   100         

   101         #tools_dir = os.path.join(self.build_area_root, "/epoc32/tools")

   102         

   103         for i in range(self.nb_split):

   104             #liste.addCommand(buildtools.Command("perl -I"+tools_dir, tools_dir, [os.path.join(tools_dir,"evalid_multiple.pl"), "-f", self.__get_partial_input_file_name(i), "> "+self.__get_partial_signature_file_name(i) ]))

   105             liste.addCommand(buildtools.Command("evalid", os.sep, ["", "-f", self.__get_partial_input_file_name(i) + " "+self.build_area_root, self.__get_partial_signature_file_name(i) ]))

   106             

   107         return liste

   108     

   109     def __get_partial_input_file_name(self, _nb):

   110         """ get the input file name string as has been created so far and add .txt to it"""

   111         return os.path.join(self.dest_dir, str(_nb) + ".txt")

   112             

   113     def __get_partial_signature_file_name(self, _nb):

   114         """ get the signature file name string as has been created so far and add .md5 to it"""

   115         return os.path.join(self.dest_dir, str(_nb) + ".md5")

   116         

   117     def concatenate_signature_files(self, signature_file):

   118         """ concatenate all the files with the MD5 CRC in """

   119         # Get header

   120         _fp = open(self.__get_partial_signature_file_name(0), 'r')

   121         line = ""

   122         header_temp = ""

   123         header = ""

   124         while (re.search(r'(\S+).*MD5=(\S+)', line) == None):

   125             header_temp = header_temp + line

   126             line = _fp.readline()

   127         

   128         for line in header_temp.splitlines():

   129             if re.match(r'Directory:.*', line):

   130                 line =  "Directory:" + self.build_area_root

   131             if re.match(r'FileList:.*', line):

   132                 line = "FileList:" + self.list_of_files

   133             header = header + line + "\n"

   134         

   135         #re.sub(r'(Directory:).*\n', "\1"+self.build_area_root, header)

   136         #re.sub(r'(FileList:).*\n', "\1"+self.list_of_files, header)

   137             

   138         header_size = len(header.splitlines())

   139         

   140         fp_md5_signatures_file = open(signature_file, 'w')

   141         fp_md5_signatures_file.write(header)

   142         for i in range(self.nb_split):

   143             _fp = open(self.__get_partial_signature_file_name(i), 'r')

   144 

   145             for i in range(header_size): # Skip header

   146                 _fp.readline()

   147             

   148             fp_md5_signatures_file.write(_fp.read())

   149             _fp.close()

   150         fp_md5_signatures_file.close()

   151 

   152     def write_build_file(self):

   153         """ create the file of the list of files to have a CRC created"""

   154         self.create_file_list()

   155         self.split_file_list()

   156         self.create_build_file()

   157     

   158     def create_build_file(self):

   159         """ there should always be an overloaded version of this method in sub-classes"""

   160         raise NotImplementedError()

   161     

   162     def build(self, signature_file):

   163         """create the list of files generate the MD5 CRC and create the final file with CRCs in"""

   164         self.write_build_file()

   165         self.compute_evalid_MD5()

   166         self.concatenate_signature_files(signature_file)

   167             

   168     def compute_evalid_MD5(self):

   169         """ there should always be an overlaoded version in the methos sub-class"""

   170         raise NotImplementedError()

   171     

   172 class MD5SignatureBuilderEBS(MD5SignatureBuilder):

   173     """ build the MD5 CRCs for all the files in the list of files"""

   174     def create_build_file(self):

   175         """Create EBS XML"""

   176         liste = self.create_command_list()

   177         self.makefile = self.dest_dir + "/ebs.xml"

   178         buildtools.convert(liste, self.makefile, "ebs")

   179 

   180     def compute_evalid_MD5(self):

   181         """Compute MD5 using the requested parallel build system"""

   182         os.chdir(self.build_area_root)

   183         os.system("perl -I%HELIUM_HOME%/tools/common/packages %HELIUM_HOME%/tools/compile/buildjob.pl -d " + self.makefile + " -l " + os.path.join(self.dest_dir, "md5.log") + " -n " + str(int(os.environ['NUMBER_OF_PROCESSORS'])*2))

   184 

   185 

   186 # Run the delta zipping over the EC build system

   187 

   188 class MD5SignatureBuilderEC(MD5SignatureBuilder):

   189     """ The MD5 CRC creation for delta zippinf for use on EC machines"""

   190     def __init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, ec_cluster_manager, ec_build_class, list_of_files):

   191         MD5SignatureBuilder.__init__(self, build_area_root, nb_split, temp_dir, exclude_dirs, list_of_files)

   192         self.ec_cluster_manager = ec_cluster_manager

   193         self.ec_build_class = ec_build_class

   194     

   195     def create_build_file(self):

   196         """Create makefile"""

   197         liste = self.create_command_list()

   198         self.makefile = self.dest_dir + "/Makefile"

   199         buildtools.convert(liste, self.makefile, "make")

   200 

   201     def compute_evalid_MD5(self):

   202         """Compute MD5 using the requested parallel build system"""

   203         root_path = os.environ['EMAKE_ROOT'] +";" + "c:\\apps;"

   204         os.chdir(self.build_area_root)

   205         

   206         print "emake --emake-cm=" + self.ec_cluster_manager + " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+ " --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile

   207         os.system("emake --emake-cm=" + self.ec_cluster_manager + " --emake-annodetail=basic,history,file,waiting --emake-annofile="+self.temp_dir+"\\delta_zip_anno.xml"+ " --emake-class=" + self.ec_build_class + " --emake-root="+root_path+" --emake-emulation-table make=symbian,emake=symbian,nmake=nmake -f " + self.makefile)

   208 

   209 class DeltaZipBuilder(object):

   210     """methods to create the delta zip after all the prep"""

   211     def __init__(self, build_area_root, temp_path, old_md5_signature, new_md5_signature):

   212         self.build_area_root = os.path.join(build_area_root, os.sep)

   213         self.temp_path = temp_path

   214         self.old_md5_signature = old_md5_signature

   215         self.new_md5_signature = new_md5_signature

   216         self.sign_dic = SignaturesDict()

   217         

   218     def __fill_signature_dict(self, signature_file, old_new):

   219         """ read each line of signature file search for .MD5"""

   220         _fp = open(signature_file, 'r')

   221         lines = _fp.read().splitlines()

   222         _fp.close()

   223         for line in lines:

   224             info = re.search(r'([ \S]+) TYPE=.*MD5=(\S+)', line)

   225             if info != None:

   226                 filename = info.group(1)

   227                 if not self.sign_dic.has_key(filename):

   228                     self.sign_dic[filename] = ["", ""]

   229                 self.sign_dic[filename][old_new] = info.group(2)                

   230     

   231     def create_delta_zip(self, zip_file, delete_list_file, no_of_zips, ant_file):

   232         """Create Delta zip and list of file to delete."""

   233         

   234         no_of_zips = int(no_of_zips)

   235         self.__fill_signature_dict(self.old_md5_signature, 0)

   236         self.__fill_signature_dict(self.new_md5_signature, 1)

   237         

   238         #fp_dic = open(zip_file + ".dic.txt", 'w')

   239         #fp_dic.write(str(self.sign_dic))

   240         #fp_dic.close()

   241         

   242         delete_list = []

   243         

   244         if not os.path.exists(os.path.dirname(delete_list_file)):

   245             os.mkdir(os.path.dirname(delete_list_file))

   246         if not os.path.exists(self.temp_path):

   247             os.mkdir(self.temp_path)

   248         

   249         archive_txt = open(os.path.join(self.temp_path, 'create_zips.txt'), 'w')

   250 

   251         for _file in self.sign_dic.keys():

   252             filepath = os.path.join(self.build_area_root, _file)

   253             

   254             signatures = self.sign_dic[_file]

   255             

   256             ( _, rest) = os.path.splitdrive(filepath)

   257             (frontpath, rest) = os.path.split(rest)

   258             

   259             if (signatures[0] != signatures[1]):  #File changed between the 2 BAs

   260                 if (signatures[0] != "") and  (signatures[1] != ""): # File is present in both BAs and has changed

   261                     if os.path.exists(filepath): # File could have been deleting after running 'build-md5':

   262                         archive_txt.write(_file + "\n")

   263                 else:

   264                     if (signatures[1] != ""): # New file

   265                         if os.path.exists(filepath):

   266                             archive_txt.write(_file + "\n")

   267                     else: # Deleted file

   268                         delete_list.append(filepath)

   269         

   270         archive_txt.close()

   271         

   272         splitter = MD5SignatureBuilder('', no_of_zips, self.temp_path, '', os.path.join(self.temp_path, 'create_zips.txt'))

   273         splitter.split_file_list()

   274         

   275         os.chdir(self.build_area_root)

   276         

   277         (frontpath, rest) = os.path.split(zip_file)

   278         stages = buildtools.CommandList()

   279         

   280         for i in range(no_of_zips):

   281             md5_dir = os.path.join(self.temp_path, "md5_temp")

   282             path = os.path.join(md5_dir, os.path.join(str(no_of_zips), str(i) + '.txt'))

   283             output = os.path.join(frontpath, rest.replace(".zip", "_part_%sof%s.zip" % (str(i+1), str(no_of_zips))))

   284             

   285             cmd = buildtools.Command('7za.exe', self.build_area_root)

   286             cmd.addArg('a')

   287             # Set the format to be zip-compatible

   288             cmd.addArg('-tzip')

   289             cmd.addArg(output)

   290             cmd.addArg('@' + path)

   291             

   292             stages.addCommand(cmd)

   293 

   294         writer = buildtools.AntWriter(ant_file)

   295         writer.write(stages)

   296 

   297         fp_delete = open(delete_list_file, 'w')

   298         fp_delete.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")

   299         fp_delete.write("<updateinstructions>\n")

   300         for i in delete_list:

   301             fp_delete.write("<deletefileaction target=\"" + i[2:] + "\"/>\n")

   302         fp_delete.write("</updateinstructions>\n")

   303         fp_delete.close()

   304         

   305                 

   306 class SignaturesDict(dict):

   307     """ class to handle signature comparison"""

   308     def __init__(self):

   309         """ constructor"""

   310         dict.__init__(self)

   311     

   312     def __str__(self):

   313         """ compare the tree structures"""

   314         string = ""

   315         #o = OldNewBA()

   316         both = False

   317         only_old = False

   318         only_new = False

   319         for filename in self.keys():

   320             signatures = self[filename]

   321             if signatures[0] == signatures[1]: #File did not change

   322                 both = True

   323             elif (signatures[0] != "") and  (signatures[1] != ""): # File is present in both BAs and has changed

   324                 both = False

   325             else:

   326                 if (signatures[1] != ""): # New file

   327                     only_old = True

   328                 else: # Deleted file

   329                     only_new = True

   330             

   331             string = string + filename + " " + str(both) + " " + " " + str(only_old) + " " + str(only_new) + " " + self[filename][0] + " " + self[filename][1] + "\n"

   332         

   333         return string

   334 

   335 def readEvalid(dir):

   336     """read E valid"""

   337     filesdict = {}

   338     for root, _, files in os.walk(dir):

   339         for name in files:

   340             f_file = os.path.join(root, name)

   341             directory = None

   342             for md5line in open(f_file):

   343                 if md5line.startswith('Directory:'):

   344                     directory = md5line.replace('Directory:', '').replace('\n', '')

   345                 if 'MD5=' in md5line:

   346                     info = re.search(r'([ \S]+) TYPE=.*MD5=(\S+)', md5line)

   347                     if info != None:

   348                         assert directory

   349                         filesdict[os.path.join(directory, info.group(1))] = info.group(2)

   350     return filesdict

   351     

   352 def changedFiles(atsevalidpre, atsevalidpost):

   353     """changed Files"""

   354     filesbefore = readEvalid(atsevalidpre)

   355     filesafter = readEvalid(atsevalidpost)

   356     

   357     changedfiles = []

   358     

   359     for key in filesafter.keys():

   360         if key not in filesbefore:

   361             changedfiles.append(key)

   362         else:

   363             if filesafter[key] != filesbefore[key]:

   364                 changedfiles.append(key)

   365     

   366     return changedfiles

   367     

   368 def evalidAdomapping(builddrive, dest, adomappingfile):

   369     """evalid Ado mapping """

   370     os.chdir(builddrive)

   371     i = 0

   372     if os.path.exists(dest):

   373         shutil.rmtree(dest)

   374     os.mkdir(dest)

   375     for line in open(adomappingfile):

   376         dir = line.split('=')[0].replace(r'\:', ':')

   377         tmpfile = os.path.join(dest, str(i))

   378         os.system('evalid -g ' + dir + ' ' + tmpfile)

   379         i = i + 1