Initial version of code cleanup. Now have classes for the environments. checkenv and diffenv updated to use it (although there are minor changes to the paths).
authorMattD <mattd@symbian.org>
Wed, 17 Mar 2010 12:27:45 +0000
changeset 203 e274d29c8bc9
parent 202 f6ae410bd493
child 204 d19c45914cb2
child 219 fa4867282f20
Initial version of code cleanup. Now have classes for the environments. checkenv and diffenv updated to use it (although there are minor changes to the paths).
dbrtools/dbr/checkenv.py
dbrtools/dbr/cleanenv.py
dbrtools/dbr/dbrbaseline.py
dbrtools/dbr/dbrenv.py
dbrtools/dbr/dbrpatch.py
dbrtools/dbr/dbrutils.py
dbrtools/dbr/diffenv.py
--- a/dbrtools/dbr/checkenv.py	Tue Mar 16 12:28:04 2010 +0000
+++ b/dbrtools/dbr/checkenv.py	Wed Mar 17 12:27:45 2010 +0000
@@ -1,4 +1,4 @@
-# Copyright (c) 2009 Symbian Foundation Ltd
+# Copyright (c) 2010 Symbian Foundation Ltd
 # This component and the accompanying materials are made available
 # under the terms of the License "Eclipse Public License v1.0"
 # which accompanies this distribution, and is available
@@ -11,35 +11,29 @@
 # mattd <mattd@symbian.org>
 #
 # Description:
-# DBR checkenv - Checks your environment against what was installed
+# new checkenv - uses OO interface.
 
-import dbrbaseline
-import dbrpatch
-import dbrutils
-
-import os.path
-
-def main():
-    dbfilename = dbrutils.defaultdb()
+import dbrenv
 
-    baseline = dbrbaseline.readdb(dbfilename)
-    if(len(baseline ) > 0):
-        patches = dbrpatch.loadpatches(dbrpatch.dbrutils.patchpath())
-        db = dbrpatch.createpatchedbaseline(baseline,patches)
-        env = dbrutils.scanenv()
-        dbrpatch.newupdatedb(db,env)
-        baseline = dbrpatch.updatebaseline(baseline, db)
-        patches = dbrpatch.updatepatches(patches, db)
+def run(args):
+  location = '/'
+#needs a fix to scanenv for this to work...  
+#  if(len(args)):
+#    location = args[0]
+  db = dbrenv.CreateDB(location)
+  local = dbrenv.DBRLocalEnv(location)
+  results = db.compare(local)
+  local.verify(results.unknown)
+  results2 = db.compare(local)hg diff -U 
+  results2.printdetail()
+  results2.printsummary()
+  db.update(local, results2.touched)
+  db.save()
+    
+def help():
+  print "Checks the status of the current environment"
+  print "Usage:"
+  print "\tdbr checkenv"
+    
+  
 
-        dbrpatch.savepatches(patches)        
-    else:
-        baseline = dbrbaseline.createdb()
-    dbrbaseline.writedb(baseline,dbfilename)
-
-
-def run(args):  
-  main()
-
-def help():
-  print "Shows the current state of the environment"
-  print "Usage\n\tdbr checkenv"
\ No newline at end of file
--- a/dbrtools/dbr/cleanenv.py	Tue Mar 16 12:28:04 2010 +0000
+++ b/dbrtools/dbr/cleanenv.py	Wed Mar 17 12:27:45 2010 +0000
@@ -46,8 +46,10 @@
         baseline = dbrpatch.updatebaseline(baseline, db)
         patches = dbrpatch.updatepatches(patches, db)
 
-        dbrpatch.savepatches(patches)        
-
+        dbrpatch.savepatches(patches)
+        dbrbaseline.writedb(baseline,dbfilename)        
+        
+    
 
 def run(args):  
   main(args)
--- a/dbrtools/dbr/dbrbaseline.py	Tue Mar 16 12:28:04 2010 +0000
+++ b/dbrtools/dbr/dbrbaseline.py	Wed Mar 17 12:27:45 2010 +0000
@@ -135,22 +135,5 @@
 
 
 def readzippeddb(drive):
-  env = dict()
-  #Note that this is really crude. I'm seeing if it'll work before cleaning things up...
-  #see if we have a build_md5.zip file
-  md5zip = os.path.join(drive,'build_md5.zip')
-  temp_dir = tempfile.mkdtemp()
-  print temp_dir 
-  if(os.path.exists(md5zip)):
-    files = set();
-    files.add('*')
-    dbrutils.extractfromzip(files,md5zip,temp_dir)
-    globsearch = os.path.join(temp_dir, os.path.join(dbrutils.patch_path_internal(),'*.md5'))
-    print globsearch 
-    hashes = glob.glob(globsearch)
-    for file in hashes:
-        print 'Reading: %s\n' % file
-        dbrutils.gethashes(env, file, True)
-  shutil.rmtree(temp_dir)
-  return env
+  return dbrutils.getzippedDB(drive)
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/dbrtools/dbr/dbrenv.py	Wed Mar 17 12:27:45 2010 +0000
@@ -0,0 +1,233 @@
+# Copyright (c) 2010 Symbian Foundation Ltd
+# This component and the accompanying materials are made available
+# under the terms of the License "Eclipse Public License v1.0"
+# which accompanies this distribution, and is available
+# at the URL "http://www.eclipse.org/legal/epl-v10.html".
+#
+# Initial Contributors:
+# Symbian Foundation Ltd - initial contribution.
+#
+# Contributors:
+# mattd <mattd@symbian.org>
+#
+# Description:
+# DBREnv - OO rewrite of the Environments  
+
+#I'm using the existing stuff as helpers until things get relocated...
+import os.path
+import glob
+
+import dbrutils
+import dbrbaseline
+import dbrpatch
+
+
+def CreateDB(location): #virtual constructor
+  print location
+#  print dbrutils.patch_path_internal()
+  if(os.path.exists(os.path.join(location,dbrutils.defaultdb()))):
+#    print 'loading baseline environment'
+#    return DBRBaselineEnv(location)
+    print 'loading patched baseline environment'
+    return DBRPatchedBaselineEnv(location)
+  if(os.path.exists(os.path.join(location,'build_md5.zip'))):
+    print 'loading zipped environment'
+    return DBRZippedEnv(location)
+  if(os.path.exists(os.path.join(location,dbrutils.patch_path_internal()))): #should do something more fun with creating a basleine if we have MD5s
+    print 'loading new env...warning: this is only here for compatibility'
+    return DBRNewLocalEnv(location)
+  if(os.path.exists(os.path.join(location,'epoc32'))): 
+    print 'loading localenv'
+    return DBRLocalEnv(location)
+
+  return DBREnv(location)
+
+
+
+class DBREnv:
+  db = dict()
+  location = ''
+  name = ''
+  def __init__(self, location):
+    self.location = location
+
+  def compare(self, other):
+    db1files = set(self.db.keys())
+    db2files = set(other.db.keys())
+
+    removed = db1files - db2files
+    added = db2files - db1files
+    common = db1files & db2files
+
+    touched = set()
+    for file in common:
+      if(int(self.db[file]['time']) != int(other.db[file]['time'])):
+        touched.add(file)
+
+    sizechanged = set()
+    for file in common:
+      if(int(self.db[file]['size']) != int(other.db[file]['size'])):
+        sizechanged.add(file)
+#can be funny with some zip files...suggest we don't use sizechanged...        
+#    changed = sizechanged 
+    changed = set()    
+    touched = touched - changed
+    unknown = set()
+    for file in touched:
+      if((self.db[file]['md5'] == "xxx") or (other.db[file]['md5'] == "xxx")):
+        unknown.add(file)
+#        if((self.db[file]['md5'] == "xxx")):
+#          print 'unknown left: %s' % file
+#        else:
+#          print 'unknown right: %s' % file
+      else:
+        if(self.db[file]['md5'] != other.db[file]['md5']):
+#          print '%s %s %s' % (file, self.db[file]['md5'], other.db[file]['md5'] )
+          changed.add(file)
+    touched = touched - unknown     
+    touched = touched - changed     
+          
+    results = DBRCompResults(added, removed, touched, changed, unknown)   
+    return results
+    
+  def verify(self, files):
+    print 'this is a pure virtual...'
+  def save(self):
+    print 'this is a pure virtual...'
+
+  def remove(self, files):
+    for file in files:
+      if(file in self.db):
+        del self.db[file]
+      else:
+        print 'warning: del: %s isnt defined' % file  
+
+  def add(self, other, files):
+    for file in files:
+      if(file in self.db):
+        print 'warning: add: %s already defined' % file
+      else:    
+        if(other.db[file]['md5'] == 'xxx'): #don't update a null md5
+          print 'warning: MD5: %s isnt defined' % file  
+        else:
+          self.db[file] = other.db[file]
+              
+  def update(self, other, files):
+    for file in files:
+      if(other.db[file]['md5'] != 'xxx'): #don't update a null md5 
+        self.db[file]['md5'] = other.db[file]['md5']                           
+      else:
+        print 'warning: MD5: %s isnt defined' % file  
+
+      self.db[file]['time'] = other.db[file]['time']              
+      self.db[file]['size'] = other.db[file]['size']
+
+
+#Database plus local filesystem access
+class DBRLocalEnv (DBREnv):
+  def __init__(self, location):
+    DBREnv.__init__(self, location)
+    #load up local files...        
+    self.db = dbrutils.scanenv()
+
+  def verify(self, files):
+    #should assert that the files are in the local DB.
+    localfiles = set(self.db.keys())
+    if(localfiles.issuperset(files)):
+      md5s = dbrutils.generateMD5s(files)
+      for file in files:
+        self.db[file]['md5'] = md5s[file]['md5']
+
+class DBRNewLocalEnv (DBRLocalEnv):
+  def __init__(self, location):
+    DBRLocalEnv.__init__(self, location)
+    #load up local files...            
+    hashes = glob.glob(os.path.join(dbrutils.patchpath(),'*.md5'))
+    for file in hashes:
+      print 'Reading: %s\n' % file
+      dbrutils.gethashes(self.db, file, False)
+
+  def save(self):
+    filename = os.path.join(self.location,dbrutils.defaultdb())
+    print 'Saving %s' % filename 
+    dbrbaseline.writedb(self.db,filename)
+
+        
+
+    
+#zipped files, contains MD5s.   
+class DBRZippedEnv (DBREnv):
+  def __init__(self, location):
+    DBREnv.__init__(self, location)
+    #load up zip MD5 and stuff
+    self.db = dbrutils.getzippedDB(self.location)        
+      
+#Database, but no filesystem access
+class DBRBaselineEnv (DBREnv):
+  def __init__(self, location):
+    DBREnv.__init__(self, location)
+    #load up database...        
+    filename = os.path.join(self.location,dbrutils.defaultdb())
+    print 'Loading %s' % filename 
+    self.db = dbrbaseline.readdb(filename)
+
+  def save(self):
+    filename = os.path.join(self.location,dbrutils.defaultdb())
+    print 'Saving %s' % filename 
+    dbrbaseline.writedb(self.db,filename)
+
+class DBRPatchedBaselineEnv (DBRBaselineEnv):
+  patches = []
+  baseline = []
+  def __init__(self, location):
+    DBRBaselineEnv.__init__(self, location)
+    #load up patches...        
+    if(len(self.db) > 0):
+      self.baseline = self.db      
+      self.patches = dbrpatch.loadpatches(os.path.join(self.location,dbrutils.patchpath()))
+      self.db = dbrpatch.createpatchedbaseline(self.baseline,self.patches)
+
+  def save(self):
+      self.baseline = dbrpatch.updatebaseline(self.baseline, self.db)
+      self.patches = dbrpatch.updatepatches(self.patches, self.db)
+      dbrpatch.savepatches(self.patches)
+      self.db = self.baseline
+      DBRBaselineEnv.save(self)
+      
+          
+class CBREnv (DBREnv): # placeholder for handling CBR components...
+  def __init__(self, location):
+    DBREnv.__init__(self, location)
+
+
+#comparison results...
+class DBRCompResults:
+  added = set()
+  removed = set()
+  touched = set()
+  changed = set()
+  unknown = set()
+  def __init__(self, added, removed, touched, changed, unknown):
+    #Should probably assert that these are disjoint.
+    self.added = added
+    self.removed = removed
+    self.touched = touched
+    self.changed = changed
+    self.unknown = unknown
+     
+  def printdetail(self):
+    for file in sorted(self.added):
+      print 'added:', file
+    for file in sorted(self.removed):
+      print 'removed:', file
+    for file in sorted(self.changed):
+      print 'changed:', file
+    for file in sorted(self.unknown):
+      print 'unknown:', file
+    
+  def printsummary(self):
+    if(len(self.added | self.removed | self.changed | self.unknown)):
+      print 'status: dirty'
+    else:
+      print 'status: clean' 
+                     
\ No newline at end of file
--- a/dbrtools/dbr/dbrpatch.py	Tue Mar 16 12:28:04 2010 +0000
+++ b/dbrtools/dbr/dbrpatch.py	Wed Mar 17 12:27:45 2010 +0000
@@ -30,12 +30,13 @@
 
     touched = set()
     for file in common:
-        if(db1[file]['time'] != db2[file]['time']):
+        if(int(db1[file]['time']) != int(db2[file]['time'])):
+            print 'touched %s %s - %s' % (db1[file]['time'], db2[file]['time'],file)  
             touched.add(file)
 
     sizechanged = set()
     for file in common:
-        if(db1[file]['size'] != db2[file]['size']):
+        if(int(db1[file]['size']) != int(db2[file]['size'])):
             sizechanged.add(file)
 
     changed = set()
@@ -56,22 +57,11 @@
         if(db1[file]['md5'] != db2[file]['md5']):                    
           changed.add(file)
     touched = touched - changed
-
-    untestable1 = set()
-    untestable2 = set()
-    for file in common:
-        if(db1[file]['md5'] == "xxx"):
-          untestable1.add(file)  
-        if(db2[file]['md5'] == 'xxx'):
-          untestable2.add(file)
-          
-    untestable = untestable1 & untestable2         
-    changed = changed - untestable
+    
 
     #remove the ones we know are changed
     touched = touched - changed
-    touched = touched - untestable
- 
+    
     results = dict()
     results['added'] = dict()
     results['removed'] = dict()
@@ -84,11 +74,11 @@
     for file in removed:
       results['removed'][file] = 0
     for file in touched:
-      results['touched'][file] = db2[file]  
+      results['touched'][file] = db2[file]
     for file in changed:
       results['changed'][file] = db2[file]  
-    for file in untestable:
-      results['untestable'][file] = 0  
+#    for file in untestable:
+#      results['untestable'][file] = 0  
     return results
 
 def printresults(results):
@@ -96,12 +86,12 @@
       print 'added:', file
     for file in sorted (results['removed']):
       print 'removed:', file              
-    for file in sorted (results['touched']):   
-      print 'touched:', file              
+#    for file in sorted (results['touched']):   
+#      print 'touched:', file              
     for file in sorted (results['changed']):
       print 'changed:', file          
-    for file in sorted (results['untestable']):
-      print 'untestable:', file          
+#    for file in sorted (results['untestable']):
+#      print 'untestable:', file          
     if(len(results['added']) + len(results['removed']) + len(results['changed']) + len(results['untestable']) == 0):
       print '\nStatus:\tclean'
     else:
--- a/dbrtools/dbr/dbrutils.py	Tue Mar 16 12:28:04 2010 +0000
+++ b/dbrtools/dbr/dbrutils.py	Wed Mar 17 12:27:45 2010 +0000
@@ -17,15 +17,17 @@
 import os
 import sys
 import string
+import shutil
+import time
 from os.path import join, isfile, stat
 from stat import *
 
 import glob # temporary (I hope) used for grabbing stuf from zip files...
-
+import tempfile
 
 
 def defaultdb():
-  return os.path.join(patchpath(),'baseline.db')
+  return os.path.join(patch_path_internal(),'baseline.db')
 
 def patchpath():
   return os.path.join(epocroot(),'%s/' % patch_path_internal())
@@ -93,7 +95,61 @@
     for file in files:
       print 'deleting %s' %file
       os.unlink(file)
-          
+            
+
+def getzippedDB(location):
+    db = dict()
+    #This is really crude, but will do for now
+#    temp = tempfile.NamedTemporaryFile()
+#    print temp.name
+    #       Date      Time    Attr         Size   Compressed  Name
+    #------------------- ----- ------------ ------------  ------------------------
+    #2010-03-10 01:02:30 D....            0            0  epoc32
+    #2010-03-10 01:02:30 D....            0            0  epoc32\relinfo
+    #2010-03-10 00:49:12 .....      2327835       575578  epoc32\relinfo\epoc32_data.md5
+    reattribs = re.compile('(\d+-\d+-\d+\s\d+:\d+:\d+)\s+\..+\s+(\d+)\s+\d*\s*(\S.+)')
+    fixpath = re.compile('\\\\')
+
+    tmpfilename = os.tmpnam( )
+    print tmpfilename    
+
+#    exestr = '7z l -i!epoc32 -x!epoc32/relinfo %s/*.zip >%s' % (path,temp.name)
+    exestr = '7z l -i!epoc32 -x!epoc32/relinfo %s/*.zip >%s' % (location,tmpfilename)
+
+    exeresult = os.system(exestr) 
+    if(exeresult):
+      sys.exit('Fatal error executing: %s\nReported error: %s' % (exestr,os.strerror(exeresult)))
+    else:
+      temp = open(tmpfilename,'r')           
+      for line in temp:
+        res = reattribs.match(line)
+        if(res):
+            entry = dict()
+            entry['time'] = int(time.mktime(time.strptime(res.group(1), '%Y-%m-%d %H:%M:%S')))
+            entry['size'] = res.group(2)
+            entry['md5'] = 'xxx'
+            filename = string.lower(fixpath.sub('/',res.group(3))) 
+            db[filename] = entry
+      temp.close()
+      os.unlink(tmpfilename)      
+      #now fill with the MD5s...
+      md5zip = os.path.join(location,'build_md5.zip')
+      print md5zip 
+      temp_dir = tempfile.mkdtemp()
+      print temp_dir 
+      if(os.path.exists(md5zip)):
+        files = set();
+        files.add('*')
+        extractfromzip(files,md5zip,temp_dir)
+        globsearch = os.path.join(temp_dir, os.path.join(patch_path_internal(),'*.md5'))
+        print globsearch 
+        hashes = glob.glob(globsearch)
+        for file in hashes:
+            print 'Reading: %s\n' % file
+            gethashes(db, file, False)
+      shutil.rmtree(temp_dir)            
+    return db
+
 
 def generateMD5s(testset):
     db = dict()
--- a/dbrtools/dbr/diffenv.py	Tue Mar 16 12:28:04 2010 +0000
+++ b/dbrtools/dbr/diffenv.py	Wed Mar 17 12:27:45 2010 +0000
@@ -1,4 +1,4 @@
-# Copyright (c) 2009 Symbian Foundation Ltd
+# Copyright (c) 2010 Symbian Foundation Ltd
 # This component and the accompanying materials are made available
 # under the terms of the License "Eclipse Public License v1.0"
 # which accompanies this distribution, and is available
@@ -11,23 +11,30 @@
 # mattd <mattd@symbian.org>
 #
 # Description:
-# DBR diffenv - compares two environments
+# new diffenv - uses OO interface and can have 
 
-import sys
-import dbrpatch
+import dbrenv
 
 def run(args):
-    if(len(args) == 2):
-      first = args[0]
-      second = args[1]      
-      dbrpatch.newcomparepatcheddbs(first, second)
+    if(len(args)):
+      if(len(args) == 1):
+        first = '/'
+        second = args[0]
+      else:
+        first = args[0]
+        second = args[1]    
+      db1=dbrenv.CreateDB(first)
+      db2=dbrenv.CreateDB(second)
+      results = db1.compare(db2)
+      results.printdetail()
+      results.printsummary()
     else:
       help()
       
 def help():
   print "Compares two environments"
   print "Usage:"
-  print "\tdbr diffenv <drive1> <drive2>"
+  print "\tdbr diffenv <drive1> (<drive2>)"
     
+  
 
-