Module buildtools |
[hide private]
[frames] | no frames]

# Source Code for Module buildtools

      1  #============================================================================
      2  #Name        : buildtools.py
      3  #Part of     : Helium
      4
      5  #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
      6  #All rights reserved.
      7  #This component and the accompanying materials are made available
      8  #under the terms of the License "Eclipse Public License v1.0"
      9  #which accompanies this distribution, and is available
     10  #at the URL "http://www.eclipse.org/legal/epl-v10.html".
     11  #
     12  #Initial Contributors:
     13  #Nokia Corporation - initial contribution.
     14  #
     15  #Contributors:
     16  #
     17  #Description:
     18  #===============================================================================
     19
     20  """Enables creation of build command list in several formats.
     21
     22  This module implements class that represent shell commands.
     23  It supports build stage and command parallelization (depends of the output format).
     24  CommandList can be generated in different format: ant, make, ebs, batch.
     25
     26  Example:
     27  from mc.buildtools import CommandList, Convert
     28  list = CommandList()
     29  list.addCommand("\\epoc32\\rombuild", "make_fpsx.bat..yy...", "build_xx_rom")
     30  list.addCommand("\\epoc32\\rombuild", "make_fpsx.bat..xx...", "build_yy_rom")
     31  list.addCommand("\\epoc32\\rombuild", "copy \\foo \\bar", "simple copy", False)
     32
     33  convert(list, "outputfile.mk", "make")
     34  convert(list, "outputfile.ant.xml", "ant")
     35  convert(list, "outputfile.ebs.xml", "ebs")
     36  convert(list, "outputfile.bat", "bat")
     37
     38  """
     39  import os
     40  import types
     41  import xml.dom.minidom
     42  import sys
     43
     44 -class PreBuilder(object):
     45      """ This class implements an abstract prebuilder.
     46          A prebuilder takes a configurationset as input and generates a build file.
     47      """
     48 -    def __init__(self, configSet):
     49          self.configSet = configSet
     50          # Select the first configuration as a default, for referencing common properties
     51          self.config = configSet.getConfigurations()[0]
     52
     53 -    def writeBuildFile(self, taskList, buildFilePath, output='ant'):
     54          """ Converting a task list into output format and writing it into buildFilePath file. """
     55          writer = None
     56          #if 'build.tool' in self.config:
     57              # Choose appropriate build tool
     58          #    print 'choosing build tool!'
     59          #    pass
     60          #else:
     61          # Choose Ant by default for now
     62          print self.config.keys()
     63          buildFileDir = os.path.dirname(buildFilePath)
     64          if not os.path.exists(buildFileDir):
     65              os.makedirs(buildFileDir)
     66          writer = get_writer(output, open(buildFilePath, 'w'))
     67          writer.write(taskList)
     68
     69 -class Task(object):
     70      """ Abstract Task object. """
     71      pass
     72
     73 -class Command(Task):
     74      """
     75          This class implements a command definition.
     76          It handles command id and stage.
     77          All command from one stage should be finished before starting the next stage.
     78      """
     79 -    def __init__(self, executable, path, args=None, name=''):
     80          Task.__init__(self)
     81          if args == None:
     82              args = []
     83          self._id  = 1
     84          self._stage = 1
     85          self._name = name
     86          self._executable = executable
     87          self._path = path
     88          self._args = args
     89
     90 -    def setJobId(self, idn):
     91          """ Set the command id. """
     92          self._id = idn
     93
     94 -    def setStage(self, stage):
     95          """ Set the command stage. """
     96          self._stage = stage
     97
     98 -    def jobId(self):
     99          """ Get the command id. """
    100          return self._id
    101
    102 -    def stage(self):
    103          """ Get the command stage. """
    104          return self._stage
    105
    106 -    def name(self):
    107          """ Get the command name. """
    108          return self._name
    109
    110 -    def executable(self):
    111          """ Get the command executable. """
    112          return self._executable
    113
    114 -    def path(self):
    115          """ Get the command path. """
    116          return self._path
    117
    118 -    def cmd(self):
    119          """ Get the command line. """
    120          return ' '.join(self._args)
    121
    122 -    def addArg(self, arg):
    123          """ Add a command line argument. """
    124          self._args.append(arg)
    125
    126 -    def __repr__(self):
    127          argsString = ' '.join(self._args)
    128          return "%s: %s: %s" % (self.name(), self.path(), argsString)
    129
    130 -class AntTask(Task):
    131      """ Interface that defines supports for an Ant task rendering. """
    132
    133 -    def toAntTask(self, doc):
    134          """ Override this method to convert a specific command into Ant command.
    135              e.g: Delete Class will use delete task from Ant, else convert into perl ... remove filename.__getCommandByStage
    136          """
    137          pass
    138
    139 -class Delete(AntTask, Command):
    140      """ Implements file/directory deleletion mechanism. """
    141
    142 -    def __init__(self, filename=None, dirname=None):
    143          Command.__init__(self, "perl", "")
    144          AntTask.__init__(self)
    145          self._filename = filename
    146          self._dir = dirname
    147          self._args.append("-MExtUtils::Command")
    148          self._args.append("-e")
    149          if self._filename != None:
    150              self._args.append("rm_f")
    151              self._args.append('"' + self._filename + '"')
    152          elif self._dir != None:
    153              self._args.append("rm_rf")
    154              self._args.append('"' + self._dir + '"')
    155
    156 -    def toAntTask(self, doc):
    157          """ Render the delete as an Ant task. """
    158          node = doc.createElementNS("", "delete")
    159          node.setAttributeNS("", "verbose", "true")
    160          node.setAttributeNS("", "failonerror", "false")
    161          if self._filename != None:
    162              node.setAttributeNS("", "file", self._filename)
    163          elif self._dir != None:
    164              node.setAttributeNS("", "dir", self._dir)
    165          return node
    166
    167
    168 -class Copy(AntTask, Command):
    169      """ Implement copy command. """
    170 -    def __init__(self, srcFile, todir):
    171          Command.__init__(self, "perl", os.path.dirname(srcFile))
    172          AntTask.__init__(self)
    173          self.srcFile = srcFile
    174          self.todir = todir
    175          self._args.append("-MExtUtils::Command")
    176          self._args.append("-e")
    177          self._args.append("cp")
    178          self._args.append('"' + self.srcFile + '"')
    179          self._args.append('"' + os.path.join(self.todir, os.path.basename(self.srcFile)) + '"')
    180
    181 -    def toAntTask(self, doc):
    182          """ Render the copy as an Ant task. """
    183          node = doc.createElementNS("", "copy")
    184          node.setAttributeNS("", "verbose", "true")
    185          node.setAttributeNS("", "failonerror", "false")
    186          node.setAttributeNS("", "file", self.srcFile)
    187          node.setAttributeNS("", "todir", self.todir)
    188          return node
    189
    190
    191 -class CommandList(object):
    192      """
    193          This class allows to safely handle Command object into lists
    194      """
    195 -    def __init__(self):
    196          self.__cmds = []
    197
    198 -    def allCommands(self):
    199          """ Returns all command list. """
    200          return self.__cmds
    201
    202 -    def addCommand(self, cmd, newstage=False):
    203          """ Add a Command to the list. """
    204          stage = 1
    205          idn = 1
    206          if len(self.__cmds) > 0:
    207              lastcmd = self.__cmds[-1]
    208              idn = lastcmd.jobId() + 1
    209              stage = lastcmd.stage()
    210              if newstage:
    211                  stage = stage + 1
    212          cmd.setStage(stage)
    213          cmd.setJobId(idn)
    214          self.__cmds.append(cmd)
    215
    216
    217 -class AbstractOutputWriter:
    218      """Base class which contains define an AbstractOutputWriter.
    219
    220      The subclass must implement a convert method which compute a command list into
    221      some output file.
    222      """
    223 -    def __init__(self, fileOut):
    224          if isinstance(fileOut, types.StringType):
    225              self._fileOut = open(fileOut, 'w')
    226          else:
    227              self._fileOut = fileOut
    228
    229 -    def write(self, cmdList):
    230          """ Method to override to implement format specific output. """
    231 -    def writeTopLevel(self, config_list, spec_name, output_path, xml_file):
    232          """ Method to override to implement top level commands. """
    233
    234 -    def __call__(self, cmdList):
    235          self.write(cmdList)
    236
    237 -    def close(self):
    238          """ Close the output stream. """
    239          self._fileOut.close()
    240
    241
    242 -class StringWriter(AbstractOutputWriter):
    243      """ Implements a Writer which is able to directly write to the output stream. """
    244
    245 -    def __init__(self, fileOut):
    246          AbstractOutputWriter.__init__(self, fileOut)
    247
    248 -    def write(self, content):
    249          """ Write content to the output. """
    250          self._fileOut.write(content)
    251
    252
    253 -class EBSWriter(AbstractOutputWriter):
    254      """ Implements EBS XML output format. """
    255
    256 -    def __init__(self, fileOut):
    257          AbstractOutputWriter.__init__(self, fileOut)
    258
    259 -    def write(self, cmdList):
    260          """ Write the command list to EBS format. """
    261          doc = xml.dom.minidom.Document()
    262          productnode = doc.createElementNS("", "Product")
    263          cmdsnode = doc.createElementNS("", "Commands")
    264          productnode.appendChild(cmdsnode)
    265          doc.appendChild(productnode)
    266
    267          for cmd in cmdList.allCommands():
    268              cmdsnode.appendChild(self.__commandToXml(doc, cmd))
    269
    270          self._fileOut.write(doc.toprettyxml())
    271
    272      @staticmethod
    273 -    def __commandToXml(doc, cmd):
    274          """ Convert a Command into an EBS command. """
    275          # <Execute ID="1" Stage="1" Component="MAS" Cwd="%EPOCROOT%" CommandLine="getrel MAS 92_013_Symbian_OS"/>
    276          cmdsnode = doc.createElementNS("", "Execute")
    277          cmdsnode.setAttributeNS("", "ID", "%d" % cmd.jobId())
    278          cmdsnode.setAttributeNS("", "Stage", "%d" % cmd.stage())
    279          cmdsnode.setAttributeNS("", "Component", cmd.name())
    280          cmdsnode.setAttributeNS("", "Cwd", cmd.path())
    281          cmdsnode.setAttributeNS("", "CommandLine", cmd.executable()+" "+cmd.cmd())
    282          return cmdsnode
    283
    284
    285 -class AntWriter(AbstractOutputWriter):
    286      """ Implements Ant XML output format. """
    287
    288 -    def __init__(self, fileOut):
    289          AbstractOutputWriter.__init__(self, fileOut)
    290
    291 -    def writeTopLevel(self, config_list, spec_name, output_path, xml_file):
    292          doc = xml.dom.minidom.Document()
    293          projectnode = doc.createElementNS("", "project")
    294          projectnode.setAttributeNS("", "name", '')
    295          projectnode.setAttributeNS("", "default", "all")
    296          doc.appendChild(projectnode)
    297          target = doc.createElementNS("", "target")
    298          target.setAttributeNS("", "name", "all")
    299          projectnode.appendChild(target)
    300
    301          parallel = doc.createElementNS("", "parallel")
    302          parallel.setAttributeNS("", "threadCount", "${number.of.threads}")
    303          target.appendChild(parallel)
    304          index = 0
    305          script_loc = os.path.normpath(os.path.join(os.environ['HELIUM_HOME'], 'tools/common/python/lib/CreateZipInput.py'))
    306          for config in config_list:
    307              sequential = doc.createElementNS("", "sequential")
    308              outputfile = os.path.normpath(os.path.join(output_path, config + ".xml"))
    309              exec_element = doc.createElementNS("", "exec")
    310              exec_element.setAttributeNS("", "executable", "python")
    311
    312              args = doc.createElementNS("", "arg")
    313              args.setAttributeNS("", "value", "%s" % script_loc)
    314              exec_element.appendChild(args)
    315
    316              args = doc.createElementNS("", "arg")
    317              args.setAttributeNS("", "line", "--output=%s" % outputfile)
    318              exec_element.appendChild(args)
    319              args = doc.createElementNS("", "arg")
    320              args.setAttributeNS("", "line", "--config=%s" % spec_name)
    321              exec_element.appendChild(args)
    322              args = doc.createElementNS("", "arg")
    323              args.setAttributeNS("", "line", "--filename=%s" % xml_file)
    324              exec_element.appendChild(args)
    325              args = doc.createElementNS("", "arg")
    326              args.setAttributeNS("", "line", "--id=%d" % index)
    327              exec_element.appendChild(args)
    328              args = doc.createElementNS("", "arg")
    329              args.setAttributeNS("", "line", "--writertype=ant")
    330              exec_element.appendChild(args)
    331              sequential.appendChild(exec_element)
    332              index += 1
    333              ant_exec = doc.createElementNS("", "ant")
    334              ant_exec.setAttributeNS("", "antfile", outputfile)
    335              sequential.appendChild(ant_exec)
    336              parallel.appendChild(sequential)
    337
    338          self._fileOut.write(doc.toprettyxml())
    339 -    def write(self, cmdList):
    340          """ Writes the command list to Ant format. """
    341          doc = xml.dom.minidom.Document()
    342          projectnode = doc.createElementNS("", "project")
    343          projectnode.setAttributeNS("", "name", '')
    344          projectnode.setAttributeNS("", "default", "all")
    345          doc.appendChild(projectnode)
    346
    347          stages = self.__getCommandByStage(cmdList)
    348
    349          for stage in stages.keys():
    350              projectnode.appendChild(self.__stageToTarget(doc, stage, stages[stage]))
    351
    352          target = doc.createElementNS("", "target")
    353          target.setAttributeNS("", "name", "all")
    354          def __toStage(stage):
    355              """ Convert the stage id into and Ant target name. """
    356              return "stage%s" % stage
    357          target.setAttributeNS("", "depends", ','.join([__toStage(stage) for stage in stages.keys()]))
    358          projectnode.appendChild(target)
    359
    360          self._fileOut.write(doc.toprettyxml())
    361
    362 -    def __stageToTarget(self, doc, stage, cmds):
    363          """ Convert a stage into an Ant target. """
    364          target = doc.createElementNS("", "target")
    365          target.setAttributeNS("", "name", "stage%s" % stage)
    366          parallel = doc.createElementNS("", "parallel")
    367          parallel.setAttributeNS("", "threadCount", "${number.of.threads}")
    368          target.appendChild(parallel)
    369
    370          for cmd in cmds:
    371              parallel.appendChild(self.__commandToAnt(doc, cmd))
    372          return target
    373
    374      @staticmethod
    375 -    def __commandToAnt(doc, cmd):
    376          """ Convert a command into an Ant task. """
    377          # does the API support Ant task conversion.
    378          # else treat it as a cmd
    379          if issubclass(type(cmd), AntTask):
    380              return cmd.toAntTask(doc)
    381          else:
    382              execnode = doc.createElementNS("", "exec")
    383              execnode.setAttributeNS("", "executable", cmd.executable())
    384              execnode.setAttributeNS("", "dir", cmd.path())
    385              arg = doc.createElementNS("", "arg")
    386              arg.setAttributeNS("", "line", cmd.cmd())
    387              execnode.appendChild(arg)
    388              return execnode
    389
    390      @staticmethod
    391 -    def __getCommandByStage(cmdList):
    392          """ Reorder a CommandList into a list of stages. """
    393          stages = {}
    394          for cmd in cmdList.allCommands():
    395              if not stages.has_key(cmd.stage()):
    396                  stages[cmd.stage()]=[]
    397              stages[cmd.stage()].append(cmd)
    398
    399          return stages
    400
    401
    402 -class MakeWriter(AbstractOutputWriter):
    403      """ Implements Makefile writer. """
    404
    405 -    def __init__(self, fileOut):
    406          AbstractOutputWriter.__init__(self, fileOut)
    407
    408 -    def writeTopLevel(self, config_list, spec_name, output_path, xml_file):
    409          content = "\n\nall: zip_inputs zip_files\n\n"
    410          index = 0
    411          input_list = "zip_inputs: "
    412          zip_list = "\n\nzip_files: "
    413          full_content = ""
    414          script_path =  os.path.normpath(os.path.join(os.environ['HELIUM_HOME'], 'tools/compile/ec'))
    415          for config in config_list:
    416              outputfile = os.path.normpath(os.path.join(output_path, config + ".mk"))
    417              input_list += " \\\n\t zip_input%d" % index
    418              zip_list += " \\\n\t zip_files%d" % index
    419              content += "\n\nzip_input%d :\n" % index
    420              content += "\t@echo === identifying files for %s\n" % config
    421
    422              content += "\tpython $(HELIUM_HOME)\\tools\\common\\python\\lib\\CreateZipInput.py --config=%s --filename=%s --id=%d --output=%s --writertype=%s\n\n" % (spec_name, xml_file, index, outputfile,'make')
    423              content += "\n\nzip_files%d :zip_input%d\n" % (index, index)
    424              content += "\t@echo === identifying files for %s\n" % config
    425              content += "\t$(MAKE) -f %s" % (outputfile)
    426              index += 1
    427
    428          full_content += input_list
    429          full_content += zip_list
    430          full_content += content
    431          self._fileOut.write(full_content)
    432 -    def write(self, cmdList):
    433          """ Converts the list of command into Makefile. """
    434          stages = {}
    435          for cmd in cmdList.allCommands():
    436              if not stages.has_key(cmd.stage()):
    437                  stages[cmd.stage()] = []
    438              stages[cmd.stage()].append(cmd)
    439
    440          # Write the all rule
    441          def __toStage(stage):
    442              """ Convert stage Id into a target name. """
    443              return "stage%s" % stage
    444
    445          #self._fileOut.write("all : %s\n" % ' '.join(map(__toStage, max(stages.keys())))
    446          if len(stages.keys()) > 0:
    447              self._fileOut.write("all : stage%s ;\n" % max(stages.keys()))
    448          else:
    449              self._fileOut.write("all: ;\n")
    450
    451          for stage in stages.keys():
    452              # Write each stage rule
    453              def __toId(cmd):
    454                  """ Convert command Id into a target name. """
    455                  self.__commandToTarget(cmd)
    456                  return "id%s" % cmd.jobId()
    457              self._fileOut.write("stage%s : %s\n" % (stage, ' '.join([__toId(task) for task in stages[stage]])))
    458
    459
    460 -    def __commandToTarget(self, cmd):
    461          """ Converting a Command into a Makefile target. """
    462          deps = ""
    463          if cmd.stage() > 1:
    464              deps = " stage%s" % (cmd.stage() - 1)
    465          self._fileOut.write("id%s:%s\n" % (cmd.jobId(), deps))
    466          self._fileOut.write("\t@echo Target %s\n" % cmd.name())
    467          winargs = ""
    468          if sys.platform == "win32":
    469              winargs = "/d"
    470          self._fileOut.write("\tcd %s %s && %s " % (winargs, cmd.path(), cmd.executable()))
    471          self._fileOut.write("%s\n" % cmd.cmd())
    472          self._fileOut.write("\n")
    473
    474
    475  __writerConstructors = { 'ant': AntWriter,
    476                           'make': MakeWriter,
    477                           'ebs': EBSWriter }
    478
    479 -def convert(cmdList, filename, outputtype="ant"):
    480      """ Helper to directly convert a command list into a specific runnable command format.
    481          e.g:
    482          cmdList = CommandList()
    483          cmdList.addCommand(...)
    484          convert(cmdList, "echo Hello world", "ant")
    485      """
    486      writer = __writerConstructors[outputtype](filename)
    487      writer(cmdList)
    488
    489 -def get_writer(buildTool, fileOut):
    490      """ Get a Writer for a specific format. """
    491      return __writerConstructors[buildTool](fileOut)
    492
    493
    494 -def supported_writers():
    495      """ Return the list of supported Writer. """
    496      return __writerConstructors.keys()
    497