1 #============================================================================ 

     2 #Name        : dependancygraph.py

     3 #Part of     : Helium 

     4 

     5 #Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).

     6 #All rights reserved.

     7 #This component and the accompanying materials are made available

     8 #under the terms of the License "Eclipse Public License v1.0"

     9 #which accompanies this distribution, and is available

    10 #at the URL "http://www.eclipse.org/legal/epl-v10.html".

    11 #

    12 #Initial Contributors:

    13 #Nokia Corporation - initial contribution.

    14 #

    15 #Contributors:

    16 #

    17 #Description:

    18 #===============================================================================

    19 

    20 import os

    21 import amara

    22 import codecs

    23 import zipfile

    24 from Ft.Lib import Uri

    25 

    26 class Library:

    27     def __init__(self, name, license, version=''):

    28         self.name = name

    29         self.license = license

    30         self.version = version

    31         self.requires = []

    32 

    33 class ModuleGroup:

    34     def __init__(self):

    35         self.libraries = {}

    36     def addConf(self, name, des, color):

    37         self.libraries[name] = (des, [], color)

    38     def addLibrary(self, conf, library):

    39         for lib in self.getLibraries(conf):

    40             if lib.name.lower() == library.name.lower():

    41                 lib.license = library.license

    42                 return

    43         self.getLibraries(conf).append(library)

    44     def getLibraries(self, conf):

    45         (_, libs, _) = self.libraries[conf]

    46         return libs

    47     def getDescription(self, conf):

    48         (des, _, _) = self.libraries[conf]

    49         return des

    50     def getColor(self, conf):

    51         (_, _, color) = self.libraries[conf]

    52         return color

    53 

    54 COLORS = ['pink', 'red', 'lightblue', 'orange', 'green', 'yellow', 'turquoise', 'limegreen']

    55 

    56 class ReadIvyConfig:

    57     def __init__(self, ivyfilename):

    58         self.ivyfilename = ivyfilename

    59         self.ivyxml = amara.parse(Uri.OsPathToUri(ivyfilename))

    60         self.group = ModuleGroup()

    61 

    62     def readConfigurations(self):

    63         for conf in self.ivyxml['ivy-module'].configurations.conf:

    64             color = COLORS.pop()

    65             self.group.addConf(conf.name, conf.description, color)

    66 

    67     def readModules(self):

    68         license = ''

    69         for module in self.ivyxml['ivy-module'].dependencies.xml_children:

    70             if hasattr(module, 'data'):

    71                 if 'License:' in module.data:

    72                     license = module.data.strip()

    73             elif hasattr(module, 'name'):

    74                 modulename = module.name.replace('-', '_')

    75             

    76                 if module.org != 'SWEPT':

    77                     self.group.addLibrary(module.conf, Library(modulename, license))

    78                     license = ''

    79 

    80     def readSubModules(self):

    81         for module in self.ivyxml['ivy-module'].dependencies.xml_children:

    82             if hasattr(module, 'name'):

    83                 if 'jars' in module.name:

    84                     ivydir = os.path.dirname(self.ivyfilename)

    85                     ivydir = os.path.join(ivydir, 'modules')

    86                     ivyjarfile = os.path.join(ivydir, module.name + '-1.0.ivy.xml')

    87                     ivymodulexml = amara.parse(Uri.OsPathToUri(ivyjarfile))

    88                     license = ''

    89                     for artifact in ivymodulexml['ivy-module'].publications.xml_children:

    90                         if hasattr(artifact, 'data'):

    91                             if 'License:' in artifact.data:

    92                                 license = artifact.data.strip()

    93                         elif hasattr(artifact, 'name'):

    94                             bits = artifact.name.split('-')

    95                             name = bits[0]

    96                             version = ''

    97                             if len(bits) > 1:

    98                                 version = bits[1]

    99                             self.group.addLibrary(module.conf, Library(name, license, version))

   100                             license = ''

   101 

   102 PYTHON_GROUP = True

   103 SUBCON_PYTHON_GROUP = False

   104 

   105 def readEggs(libraries, dirtosearch, internaldir):

   106     libraries.addConf(PYTHON_GROUP, 'Python libs', libraries.getColor('core_install'))

   107     libraries.addConf(SUBCON_PYTHON_GROUP, 'Python subcon libs', libraries.getColor('subcon'))

   108     

   109     for x in [os.walk(dirtosearch, topdown=False), os.walk(internaldir, topdown=False)]:

   110         for root, _, files in x:

   111             notinsubcon = os.path.normpath(internaldir) in os.path.normpath(root)

   112             

   113             for fname in files:

   114                 filename = os.path.join(root, fname)

   115                 if fname == 'PKG-INFO':

   116                     pkgmetafile = open(filename)

   117                     library = readPkgInfo(pkgmetafile)

   118                     pkgmetafile.close()

   119                     

   120                     requirefilename = os.path.join(filename, '..', 'requires.txt')

   121                     if os.path.exists(requirefilename):

   122                         requiresfile = open(requirefilename)

   123                         readRequiresFile(requiresfile, library)

   124                         requiresfile.close()

   125                         

   126                     libraries.addLibrary(notinsubcon, library)

   127                     

   128                 if os.path.isfile(filename) and fname.endswith('.egg'):

   129                     eggfile = zipfile.ZipFile(filename, 'r', zipfile.ZIP_DEFLATED)

   130                     

   131                     data = eggfile.read('EGG-INFO/PKG-INFO')

   132                     

   133                     library = readPkgInfo(data.split('\n'))

   134                     

   135                     if 'EGG-INFO/requires.txt' in eggfile.namelist():

   136                         requiresdata = eggfile.read('EGG-INFO/requires.txt')

   137                         readRequiresFile(requiresdata.split('\n'), library)

   138                         

   139                     libraries.addLibrary(notinsubcon, library)

   140                     

   141                     eggfile.close()

   142 

   143 def readRequiresFile(data, library):

   144     for line in data:

   145         line = line.strip()

   146         if line != '' and not (line.startswith('[') and line.endswith(']')):

   147             library.requires.append(line.split('>=')[0].strip())

   148 

   149 def readPkgInfo(data):

   150     name = ''

   151     version = ''

   152     license = ''

   153     license2 = ''

   154   

   155     for line in data:

   156         if 'Name:' in line:

   157             name = line.strip().replace('Name: ', '')

   158         if 'Version:' in line:

   159             version = line.strip().replace('Version: ', '')

   160         if 'License:' in line:

   161             license = line.strip().replace('License: ', '')                    

   162         if 'Classifier: License :: ' in line:

   163             license2 = license2 + ' ' + line.strip().replace('Classifier: License :: ', '').replace('OSI Approved :: ', '')

   164     

   165     if license.lower() == 'unknown' or license == '' or license2 != '':

   166         license = license2

   167     

   168     return Library(name, license, version)

   169 

   170 def addLicensesColors(graphdata, group):

   171     newgraphdata = []

   172     for line in graphdata:

   173         newline = line

   174         for conf in group.libraries:

   175             for module in group.getLibraries(conf):

   176                 if module.name.lower() in line.lower() and 'label=' in line:

   177                     newline = line.replace('label=', 'color=%s,label=' % group.getColor(conf))

   178                     

   179                     if module.license != '':

   180                         newline = newline.replace("\"];", "|%s\"];" % module.license)

   181                     

   182                     break

   183         newgraphdata.append(newline)

   184     return newgraphdata

   185     

   186 def createKey(group):

   187     key = """subgraph cluster1 {

   188     label = "Key";

   189     style=filled;

   190     color=lightgrey;

   191     """

   192     

   193     for conf in group.libraries:

   194         if conf != PYTHON_GROUP and conf != SUBCON_PYTHON_GROUP:

   195             key = key + "\"%s: %s\" [style=filled,color=%s];" % (conf, group.getDescription(conf), group.getColor(conf))

   196     

   197     key = key + "}"

   198     return key

   199 

   200 def createGraph(ivyxmlfilename, graphfilename, dirtosearch, internaldir, subcon):

   201     readivy = ReadIvyConfig(ivyxmlfilename)

   202     readivy.readConfigurations()

   203     readivy.readModules()

   204     readivy.readSubModules()

   205     

   206     group = readivy.group

   207     

   208     readEggs(group, dirtosearch, internaldir)

   209     

   210     key = createKey(group)

   211     

   212     graphdata = loadGraphFile(graphfilename)

   213     

   214     newgraphdata = addLicensesColors(graphdata, group)

   215     

   216     #add key to graph

   217     newgraphdata[-1] = newgraphdata[-1].replace('}', key + '\n}')

   218     

   219     graphwritefile = codecs.open(graphfilename, 'w', 'utf8')

   220     graphwritefile.writelines(newgraphdata)

   221     graphwritefile.close()

   222     

   223     linkPythonLibs(group, graphfilename, subcon)

   224 

   225 def loadGraphFile(graphfilename):

   226     destgraphfile = codecs.open(graphfilename, 'r', 'utf8')

   227     graphdata = []

   228     for line in destgraphfile:

   229         graphdata.append(line)

   230     destgraphfile.close()

   231     return graphdata

   232 

   233 def addToGraph(graphfilenametoadd, destgraphfilename):

   234     graphdata = loadGraphFile(destgraphfilename)

   235     

   236     graphfile = codecs.open(graphfilenametoadd, 'r', 'utf8')

   237     graphdatatoadd = ''

   238     for line in graphfile:

   239         line = line.replace('digraph {', '')

   240         graphdatatoadd = graphdatatoadd + line

   241     graphfile.close()

   242     

   243     graphdata[-1] = graphdata[-1].replace('}', graphdatatoadd)

   244     

   245     graphwritefile = codecs.open(destgraphfilename, 'w', 'utf8')

   246     graphwritefile.writelines(graphdata)

   247     graphwritefile.close()

   248 

   249 def linkPythonLibs(libraries, destgraphfilename, subcon):    

   250     graphdata = loadGraphFile(destgraphfilename)

   251   

   252     output = "helium_ant -> helium_python;\n"

   253     

   254     if subcon:

   255         list = [SUBCON_PYTHON_GROUP]

   256     else:

   257         list = [SUBCON_PYTHON_GROUP, PYTHON_GROUP]

   258     

   259     for group in list:

   260         for lib in libraries.getLibraries(group):

   261             output = output + ("helium_python -> \"%s\";\n" % lib.name)

   262             output = output + ("\"%s\" [style=filled,shape=record,color=%s,label=\"%s %s|%s\"];\n" % (lib.name, libraries.getColor(group), lib.name, lib.version, lib.license))

   263             

   264             for require in lib.requires:

   265                 output = output + ("\"%s\" -> \"%s\";\n" % (lib.name, require))

   266     

   267     graphdata.reverse()

   268     for line in graphdata:

   269         if line.strip() == '':

   270             graphdata.pop(0)

   271         else:

   272             break

   273     graphdata.reverse()

   274     

   275     graphdata[-1] = graphdata[-1].replace('}', output + '}')

   276     

   277     graphwritefile = codecs.open(destgraphfilename, 'w', 'utf8')

   278     graphwritefile.writelines(graphdata)

   279     graphwritefile.close()

   280 

   281 def externalDependancies(database, output):

   282     out = open(output, 'w')

   283     db = amara.parse(Uri.OsPathToUri(database))

   284     out.write('digraph G {\n')

   285     for p in db.antDatabase.project:

   286         items = []

   287         if hasattr(p, 'property'):

   288             for prop in p.property:

   289                 if 'external' + os.sep in os.path.abspath(str(prop.defaultValue)):

   290                     items.append(str(prop.defaultValue))

   291         if hasattr(p, 'fileDependency'):

   292             for dep in p.fileDependency:

   293                 dep = str(dep).split(' ')[0]

   294                 if 'external' + os.sep in os.path.abspath(str(dep)):

   295                     items.append(str(dep))

   296                     

   297         items = set(items)

   298         for i in items:

   299             out.write('\"%s\" -> \"%s\"\n' % (str(p.name), i.replace(os.environ['HELIUM_HOME'], 'helium').replace(os.sep, '/')))

   300     out.write('}')                

   301     out.close()

   302 

   303 def appendLogs(t, p, output, macro=False):

   304     if hasattr(t, 'signal'):

   305         for signal in t.signal:

   306             if macro:

   307                 output.append("\"%s\" [fontname=\"Times-Italic\"];" % str(t.name))

   308             output.append('subgraph \"cluster%s\" {label = \"%s\"; \"%s\"}\n' % (str(p.name), str(p.name), str(t.name)))

   309             s = str(signal).split(',')

   310             if len(s) > 1:

   311                 if s[1] == 'now':

   312                     color = 'red'

   313                 elif s[1] == 'defer':

   314                     color = 'yellow'

   315                 else:

   316                     color = 'green'

   317                 output.append('subgraph \"cluster%s\" {color=%s;style=filled;label = \"Failbuild: %s\"; \"%s\"}\n' % (str(s[1]), color, str(s[1]), str(s[0])))

   318             output.append('\"%s\" -> \"%s\" [style=dotted]\n' % (str(t.name), str(s[0])))

   319     if hasattr(t, 'log'):

   320         for log in t.log:

   321             logdir = '/output/logs/'

   322             logname = os.path.basename(str(log))

   323             if not ('**' in logname):

   324                 logname = logname.replace('*', '${sysdef.configuration}').replace('--logfile=', '')

   325                 if not logdir in logname:

   326                     logname = logdir + logname

   327                 logname = logname.replace(os.sep, '/')

   328                 

   329                 if macro:

   330                     output.append("\"%s\" [fontname=\"Times-Italic\"];" % str(t.name))

   331                 output.append('subgraph \"cluster%s\" {label = \"%s\"; \"%s\"}\n' % (str(p.name), str(p.name), str(t.name)))

   332                 output.append('\"%s\" -> \"%s\"\n' % (str(t.name), logname))

   333 

   334 def findLogFiles(database, output):

   335     out = open(output, 'w')

   336     db = amara.parse(Uri.OsPathToUri(database))

   337     out.write('digraph G {\n')

   338     output = []

   339     

   340     for p in db.antDatabase.project:

   341         if hasattr(p, 'macro'):

   342             for t in p.macro:

   343                 appendLogs(t, p, output, True)

   344         if hasattr(p, 'target'):

   345             for t in p.target:

   346                 appendLogs(t, p, output)

   347     for l in set(output):

   348         out.write(l)

   349     out.write('}')                

   350     out.close()