diff -r 7685cec9fd3c -r f2ddfa555b0f doc/api/python/archive.mappers-pysrc.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/api/python/archive.mappers-pysrc.html Fri Sep 11 11:54:49 2009 +0100 @@ -0,0 +1,2543 @@ + + + + + archive.mappers + + + + + + + + + + + + + + + + + + + + +
   Trees      Indices      Help   
+ + + + + +
+ + Package archive :: + Module mappers + + + + + + +
[hide private]
[frames] | no frames]
+
+

Source Code for Module archive.mappers

+
+  1  #============================================================================  
+  2  #Name        : mappers.py  
+  3  #Part of     : Helium  
+  4   
+  5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). 
+  6  #All rights reserved. 
+  7  #This component and the accompanying materials are made available 
+  8  #under the terms of the License "Eclipse Public License v1.0" 
+  9  #which accompanies this distribution, and is available 
+ 10  #at the URL "http://www.eclipse.org/legal/epl-v10.html". 
+ 11  # 
+ 12  #Initial Contributors: 
+ 13  #Nokia Corporation - initial contribution. 
+ 14  # 
+ 15  #Contributors: 
+ 16  # 
+ 17  #Description: 
+ 18  #=============================================================================== 
+ 19   
+ 20  """ Archive mappers that map how the input files are divided into archives. 
+ 21   
+ 22   
+ 23  """ 
+ 24   
+ 25  import buildtools 
+ 26  import os 
+ 27  import sys 
+ 28  import codecs 
+ 29  import fileutils 
+ 30  import logging 
+ 31  import symrec 
+ 32  import re 
+ 33  import csv 
+ 34  import shutil 
+ 35   
+ 36  _logger = logging.getLogger('logger.mappers') 
+ 37  _logger.setLevel(logging.INFO) 
+ 38   
+ 39  # Default value for missing/invalid policy files. 
+ 40  MISSING_POLICY = "9999" 
+ 41   
+ 42   
+
 43 -class Mapper(object): 
+
 44      """ Mapper Abstract class. Any custom implementation must derive it!. 
+ 45       
+ 46      It handles metadata creation. 
+ 47      """ 
+ 48       
+
 49 -    def __init__(self, config, tool): 
+
 50          self._tool = tool 
+ 51          self._config = config 
+ 52          self._metadata = None 
+ 53          if not os.path.exists(self._config['archives.dir']): 
+ 54              os.makedirs(self._config['archives.dir']) 
+ 55          if self._config.has_key("grace.metadata") and self._config.get_boolean("grace.metadata", False): 
+ 56              if self._config.has_key("grace.template") and os.path.exists(self._config["grace.template"]) and \ 
+ 57               not os.path.exists(os.path.join(self._config['archives.dir'], self._config['name'] + ".metadata.xml")): 
+ 58                  shutil.copy(config["grace.template"], os.path.join(self._config['archives.dir'], self._config['name'] + ".metadata.xml")) 
+ 59              self._metadata = symrec.ReleaseMetadata(os.path.join(self._config['archives.dir'], self._config['name']+ ".metadata.xml"), 
+ 60                                         service=self._config['grace.service'], 
+ 61                                         product=self._config['grace.product'], 
+ 62                                         release=self._config['grace.release']) 
+ 63              self._metadata.save()             
+
 64           
+
 65 -    def declare_package(self, filename, extract="single"): 
+
 66          """ Add a package to the metadata file. """ 
+ 67          if self._metadata is None: 
+ 68              return 
+ 69          self._metadata.add_package(os.path.basename(filename), extract=extract, filters=self._config.get_list('grace.filters', None), default=self._config.get_boolean('grace.default', True)) 
+ 70          self._metadata.save() 
+
 71       
+
 72 -    def create_commands(self, manifest): 
+
 73          """ Return a list of command list. """ 
+ 74          return [[self._tool.create_command(self._config['name'], manifests=[manifest])]] 
+
 75           
+ 76       
+
 77 -class DefaultMapper(Mapper): 
+
 78      """ The default mapper. It splits the content based on size characteristics. 
+ 79       
+ 80      'the max.files.per.archive' and 'max.uncompressed.size' properties define how the input files 
+ 81      are split between a number of part zips. 
+ 82      """ 
+
 83 -    def __init__(self, config, archiver): 
+
 84          """ Initialization. """ 
+ 85          Mapper.__init__(self, config, archiver) 
+
 86   
+
 87 -    def create_commands(self, manifest): 
+
 88          """ Return a list of command lists. """ 
+ 89          result = [] 
+ 90   
+ 91          _logger.info("  * Input manifest: " + manifest) 
+ 92          manifests = self._split_manifest_file(self._config['name'], manifest) 
+ 93          if not os.path.exists(self._config['archives.dir']): 
+ 94              _logger.info("  * Mkdir " + self._config['archives.dir']) 
+ 95              os.makedirs(self._config['archives.dir']) 
+ 96   
+ 97          for manifest in manifests: 
+ 98              _logger.info("  * Creating command for manifest: " + manifest) 
+ 99              filename = os.path.join(self._config['archives.dir'], os.path.splitext(os.path.basename(manifest))[0]) 
+100              if len(manifests) == 1: 
+101                  filename = os.path.join(self._config['archives.dir'], self._config['name']) 
+102              _logger.info("  * " + filename + self._tool.extension()) 
+103              self.declare_package(filename + self._tool.extension(), self._config.get('grace.extract', 'single')) 
+104              result.extend(self._tool.create_command(self._config.get('zip.root.dir', self._config['root.dir']), filename, manifests=[manifest])) 
+105           
+106          return [result] 
+
107   
+
108 -    def _split_manifest_file(self, name, manifest_file_path): 
+
109          """ This method return a list of files that contain the content of the zip parts to create. """ 
+110          filenames = [] 
+111           
+112          if (self._config.has_key('max.files.per.archive') or self._config.has_key('max.uncompressed.size')): 
+113              size = 0 
+114              files = 0 
+115              part = 0 
+116              filename = "" 
+117              output = None 
+118                           
+119              if os.path.exists(self._config['root.dir']) and os.path.isdir(self._config['root.dir']): 
+120                  curdir = os.path.abspath(os.curdir) 
+121                  os.chdir(self._config.get('zip.root.dir', self._config['root.dir']))             
+122                  maxfiles = self._config.get('max.files.per.archive', 100000000) 
+123                  _logger.info("Max number of files per archive: " + str(maxfiles)) 
+124                  max_uncompressed_size = self._config.get('max.uncompressed.size', 100000000) 
+125                  _logger.info("Max uncompressed size per archive: " + str(max_uncompressed_size)) 
+126                   
+127                  file_handle = codecs.open(manifest_file_path, "r", "utf-8" ) 
+128           
+129                  for line in file_handle.readlines(): 
+130                      line = line.rstrip() 
+131           
+132                      if(os.path.isfile(line)): 
+133                          if part == 0 or files == int(maxfiles) or size + os.path.getsize(line) >= int(max_uncompressed_size): 
+134                              if output != None: 
+135                                  output.close() 
+136           
+137                              size = 0 
+138                              files = 0 
+139                              part += 1 
+140           
+141                              filename = "%s_part%02d" % (name, part) 
+142                              filenames.append(os.path.join(self._config['temp.build.dir'], filename + ".txt")) 
+143           
+144                              output = codecs.open(os.path.join(self._config['temp.build.dir'], filename + ".txt"), 'w', "utf-8" ) 
+145           
+146                          files += 1 
+147                          size += os.path.getsize(line) 
+148           
+149                          output.write(u"".join([line, u'\n'])) 
+150                      elif(os.path.isdir(line)): 
+151                          if (len(os.listdir(line)) == 0): 
+152                              if part == 0 or files == int(maxfiles): 
+153                                  if output != None: 
+154                                      output.close() 
+155           
+156                                  size = 0 
+157                                  files = 0 
+158                                  part += 1 
+159           
+160                                  filename = "%s_part%02d" % (name, part) 
+161                                  filenames.append(os.path.join(self._config['temp.build.dir'], filename + ".txt")) 
+162                                   
+163                                  output = open(os.path.abspath(os.path.join(self._config['temp.build.dir'], filename + ".txt")), 'w') 
+164           
+165                              files += 1 
+166           
+167                              output.write(u"".join([line, u'\n'])) 
+168                      else: 
+169                          _logger.warning('Not recognized as file or directory: %s' % line) 
+170           
+171                  if output != None: 
+172                      output.close() 
+173           
+174                  file_handle.close() 
+175                  os.chdir(curdir) 
+176          else: 
+177              filenames.append(manifest_file_path) 
+178           
+179          return filenames 
+
180   
+181   
+
182 -class PolicyMapper(Mapper): 
+
183      """ Implements a policy content mapper. 
+184       
+185      It transforms a list of files into a list of commands with their inputs. 
+186      All files with policy 0 will be under the main archive. 
+187      All other files will get backed up by policy and then store into an second archive.  
+188      """ 
+189       
+
190 -    def __init__(self, config, archiver): 
+
191          """ Initialization. """ 
+192          Mapper.__init__(self, config, archiver) 
+193          self._policies = {} 
+194          self._policy_cache = {} 
+195          self._binary = {} 
+196          # Load csv 
+197          if self._config.has_key('policy.csv'): 
+198              if os.path.exists(self._config['policy.csv']): 
+199                  self.load_policy_binary(self._config['policy.csv']) 
+200              else: 
+201                  _logger.error("POLICY_ERROR: File not found '%s'." % self._config['policy.csv']) 
+
202   
+
203 -    def load_policy_binary(self, csvfile, column=1): 
+
204          """ Loads the binary IDs from the CSV file. """ 
+205          _logger.info("POLICY_INFO: Loading policy definition '%s'." % csvfile) 
+206          reader = csv.reader(open(csvfile, "rU")) 
+207          for row in reader: 
+208              if re.match(r"^((?:\d+)|(?:0842[0-9a-zA-Z]{3}))$", row[0].strip()):                 
+209                  _logger.info("POLICY_INFO: Adding policy: '%s' => '%s'" % (row[0].strip(), row[column].strip().lower())) 
+210                  self._binary[row[0].strip()] = row[column].strip().lower() 
+211              else: 
+212                  _logger.warning("POLICY_WARNING: Discarding policy: '%s'." % row[0].strip()) 
+
213   
+
214 -    def zip2zip(self): 
+
215          """ Should the non public zip be zipped up under a specific zip. """ 
+216          return self._config.get_boolean('policy.zip2zip', False) 
+
217       
+
218 -    def create_commands(self, manifest): 
+
219          """ Generates a list of build commands. """ 
+220          result = [] 
+221          stages = [] 
+222   
+223          # Create the archive output directory 
+224          if not os.path.exists(self._config['archives.dir']): 
+225              _logger.info("  * Mkdir " + self._config['archives.dir']) 
+226              os.makedirs(self._config['archives.dir']) 
+227           
+228          # Sort the manifest content, splitting it by policy 
+229          file_handle = codecs.open(manifest, "r", "utf-8") 
+230          for line in file_handle.readlines(): 
+231              line = line.rstrip() 
+232              self._sort_by_policy(line) 
+233          file_handle.close() 
+234           
+235          # Generating sublists. 
+236          for key in self._policies.keys(): 
+237              self._policies[key].close() 
+238              manifest = os.path.join(self._config['temp.build.dir'], self._config['name'] + "_%s" % key + ".txt") 
+239              filename = os.path.join(self._config['archives.dir'], self._config['name'] + "_%s" % key) 
+240              _logger.info("  * " + filename + self._tool.extension()) 
+241              result.extend(self._tool.create_command(self._config.get('zip.root.dir', self._config['root.dir']), filename, manifests=[manifest])) 
+242          stages.append(result) 
+243           
+244          # See if any internal archives need to be created 
+245          content = [] 
+246          for key in self._policies.keys(): 
+247              if not self.zip2zip(): 
+248                  self.declare_package(self._config['name'] + "_%s" % key + self._tool.extension()) 
+249              else: 
+250                  if key != "0": 
+251                      content.append(os.path.join(self._config['archives.dir'], self._config['name'] + "_%s" % key + self._tool.extension())) 
+252                  else: 
+253                      self.declare_package(self._config['name'] + "_%s" % key + self._tool.extension()) 
+254   
+255          # Creating zip that contains each policy zips. 
+256          if self.zip2zip() and len(content) > 0: 
+257              manifest = os.path.join(self._config['temp.build.dir'], self._config['name'] +  ".internal.txt") 
+258              file_handle = codecs.open( manifest, "w+", "utf-8" ) 
+259              file_handle.write(u"\n".join(content)) 
+260              file_handle.close() 
+261              internal = "internal" 
+262              if self._config.has_key('policy.internal.name'): 
+263                  internal = self._config['policy.internal.name'] 
+264              filename = os.path.join(self._config['archives.dir'], self._config['name'] +  "_" + internal) 
+265              _logger.info("  * " + filename + self._tool.extension()) 
+266              self.declare_package(filename + self._tool.extension(), "double") 
+267              stages.append(self._tool.create_command(self._config['archives.dir'], filename, manifests=[manifest])) 
+268   
+269              cmds = [] 
+270              for filename in content: 
+271                  cmds.append(buildtools.Delete(filename=filename)) 
+272              stages.append(cmds) 
+273          return stages 
+
274       
+
275 -    def get_dir_policy(self, dirname): 
+
276          """ Get policy value for a specific directory. """ 
+277          dirname = os.path.normpath(dirname) 
+278          if not self._policy_cache.has_key(dirname): 
+279              policyfile = None 
+280              for name in self.get_policy_filenames(): 
+281                  if sys.platform != 'win32': 
+282                      for filename in os.listdir(dirname): 
+283                          if filename.lower() == name.lower(): 
+284                              policyfile = os.path.join(dirname, filename) 
+285                              break 
+286                  elif os.path.exists(os.path.join(dirname, name)):  
+287                      policyfile = os.path.join(dirname, name) 
+288                      break 
+289               
+290              value = self._config.get('policy.default.value', MISSING_POLICY) 
+291              if policyfile != None: 
+292                  try: 
+293                      value = fileutils.read_policy_content(policyfile) 
+294                      if value not in self._binary.keys(): 
+295                          _logger.error("POLICY_ERROR: policy file found %s but policy %s value not exists in csv" % (policyfile, value)) 
+296                  except Exception, exc: 
+297                      _logger.error("POLICY_ERROR: %s" % exc)          
+298                      value = self._config.get('policy.default.value', MISSING_POLICY) 
+299              else: 
+300                  _logger.error("POLICY_ERROR: could not find a policy file under: '%s'" % dirname) 
+301              # saving the policy value for that directory. 
+302              self._policy_cache[dirname] = value 
+303          return self._policy_cache[dirname] 
+
304           
+
305 -    def get_policy_filenames(self): 
+
306          """ Returns the list of potential policy filenames. """ 
+307          return self._config.get_list('policy.filenames', ['Distribution.policy.s60']) 
+
308           
+
309 -    def _sort_by_policy(self, filename): 
+
310          """ Store the input file sorted by its policy number. """ 
+311          path = os.path.join(self._config['root.dir'], filename) 
+312          parentdir = os.path.dirname(path) 
+313          if os.path.isdir(path): 
+314              parentdir = path 
+315          value = self.get_dir_policy(parentdir) 
+316          if not value in self._policies: 
+317              self._policies[value] = codecs.open( os.path.join(self._config['temp.build.dir'], self._config['name'] + "_%s" % value + ".txt"), "w+", "utf-8" ) 
+318          self._policies[value].write(u"%s\n" % filename) 
+
319   
+320   
+
321 -class PolicyRemoverMapper(PolicyMapper): 
+
322      """ This class implements a variant of the policy mapper. 
+323           
+324      It removes the internal source. Only binary flagged content is kept. 
+325      """ 
+326       
+
327 -    def __init__(self, config, archiver): 
+
328          """ Initialization. """ 
+329          PolicyMapper.__init__(self, config, archiver) 
+330          self._rm_policy_cache = {} 
+
331   
+
332 -    def get_policy_root_dir(self): 
+
333          """ Return the policy.root.dir or root.dir if not set or not under root.dir.""" 
+334          if not self._config.has_key("policy.root.dir"): 
+335              return os.path.normpath(self._config['root.dir']) 
+336          else: 
+337              if fileutils.destinsrc(self._config['root.dir'], self._config['policy.root.dir']): 
+338                  return os.path.normpath(self._config['policy.root.dir']) 
+339              else: 
+340                  return os.path.normpath(self._config['root.dir']) 
+
341   
+
342 -    def get_rmdir_policy(self, dirname): 
+
343          """ check if the directory should be dropped or not""" 
+344          dirname = os.path.normpath(dirname) 
+345          # check if parent is banned... 
+346          prootdir = os.path.normpath(self.get_policy_root_dir()) 
+347          rootdir = os.path.normpath(self._config['root.dir']) 
+348          if sys.platform == 'win32': 
+349              dirname = dirname.lower() 
+350              prootdir = prootdir.lower() 
+351              rootdir = rootdir.lower()         
+352           
+353          # else get real value... 
+354          if not self._rm_policy_cache.has_key(dirname): 
+355              self._rm_policy_cache[dirname] = self.get_dir_policy(dirname) 
+356           
+357          return self._rm_policy_cache[dirname] 
+
358           
+
359 -    def create_commands(self, manifest): 
+
360          """ Generates a list of build commands. """ 
+361          stages = PolicyMapper.create_commands(self, manifest) 
+362           
+363          if not self._config.has_key('policy.csv'): 
+364              _logger.error("POLICY_ERROR: Property 'policy.csv' not defined everything will get removed.") 
+365          cmds = [] 
+366          file_handle = codecs.open( manifest, "r", "utf-8" ) 
+367          for line in file_handle.readlines(): 
+368              line = line.rstrip() 
+369              filepath = os.path.normpath(os.path.join(self._config.get('zip.root.dir', self._config['root.dir']), line)) 
+370              value = self.get_rmdir_policy(os.path.dirname(filepath))             
+371              delete = True 
+372              if value in self._binary.keys(): 
+373                  if self._binary[value] == "yes": 
+374                      _logger.info("POLICY_INFO: Keeping %s (%s=>yes)!" % (filepath, value)) 
+375                      delete = False 
+376                  elif self._binary[value] == "bin": 
+377                      _logger.info("POLICY_INFO: Keeping %s (%s=>bin)!" % (filepath, value)) 
+378                      delete = False 
+379              else: 
+380                  _logger.error("POLICY_ERROR: %s value for %s not in csv file. Will be removed!!" % (value, filepath)) 
+381                  
+382              if delete: 
+383                  _logger.info("POLICY_INFO: File %s will be removed!" % filepath) 
+384                  cmds.append(buildtools.Delete(filename=filepath)) 
+385          file_handle.close() 
+386          if len(cmds) > 0: 
+387              stages.append(cmds) 
+388          return stages 
+
389   
+390   
+
391 -class SFPolicyRemoverMapper(PolicyRemoverMapper): 
+
392      """ Implement an SFL column based policy remover. """ 
+393   
+
394 -    def __init__(self, config, archiver): 
+
395          """ Initialization. """ 
+396          PolicyRemoverMapper.__init__(self, config, archiver) 
+
397   
+
398 -    def load_policy_binary(self, csvfile): 
+
399          """ Loading the policy using the 3rd column. """ 
+400          _logger.info("POLICY_INFO: Loading actions from the 3rd column") 
+401          PolicyRemoverMapper.load_policy_binary(self, csvfile, column=3) 
+
402   
+
403 -class EPLPolicyRemoverMapper(PolicyRemoverMapper): 
+
404      """ Implement an EPL column based policy remover. """ 
+
405 -    def __init__(self, config, archiver): 
+
406          """ Initialization. """ 
+407          PolicyRemoverMapper.__init__(self, config, archiver) 
+
408   
+
409 -    def load_policy_binary(self, csvfile): 
+
410          """ Loading the policy using the 4th column. """ 
+411          _logger.info("POLICY_INFO: Loading actions from the 4th column") 
+412          PolicyRemoverMapper.load_policy_binary(self, csvfile, column=4) 
+
413           
+414           
+415  MAPPERS = {'default': DefaultMapper, 
+416               'policy': PolicyMapper, 
+417               'policy.remover': PolicyRemoverMapper, 
+418               'sfl.policy.remover': SFPolicyRemoverMapper, 
+419               'epl.policy.remover': EPLPolicyRemoverMapper,} 
+420   
+
421 -def get_mapper(name, config, archiver): 
+
422      """ Get mapper instance from its string id. """ 
+423      if name in MAPPERS: 
+424          return MAPPERS[name](config, archiver) 
+425      raise Exception("ERROR: Could not find mapper '%s'." % name) 
+
426   
+
+
+ + + + + + + + + + + + + + + +
   Trees      Indices      Help   
+ + + + + +
+ Generated by Epydoc 3.0beta1 on Wed Sep 09 13:44:25 2009 + + http://epydoc.sourceforge.net +
+ + + + +