1 #!/usr/bin/env python

     2 #

     3 # SQUEEZE

     4 # $Id$

     5 #

     6 # squeeze a python program

     7 #

     8 # installation:

     9 # - use this script as is, or squeeze it using the following command:

    10 #

    11 # python squeezeTool.py -1su -o squeeze -b squeezeTool squeezeTool.py

    12 #

    13 # notes:

    14 # - this is pretty messy.  make sure to test everything carefully

    15 #   if you change anything

    16 #

    17 # - the name "squeeze" is taken from an ABC800 utility which did

    18 #   about the same thing with Basic II bytecodes.

    19 #

    20 # history:

    21 # 1.0 1997-04-22 fl   Created

    22 # 1.1 1997-05-25 fl   Added base64 embedding option (-1)

    23 #     1997-05-25 fl   Check for broken package file

    24 # 1.2 1997-05-26 fl   Support uncompressed packages (-u)

    25 # 1.3 1997-05-27 fl   Check byte code magic, eliminated StringIO, etc.

    26 # 1.4 1997-06-04 fl   Removed last bits of white space, removed try/except

    27 # 1.5 1997-06-17 fl   Added squeeze archive capabilities (-x)

    28 # 1.6 1998-05-04 fl   Minor fixes in preparation for public source release

    29 #

    30 # reviews:

    31 #       "Fredrik Lundh is a friggin genius"

    32 #       -- Aaron Watters, author of 'Internet Programming with Python'

    33 #

    34 #       "I agree ... this is a friggin Good Thing"

    35 #       -- Paul Everitt, Digital Creations

    36 #

    37 # Copyright (c) 1997 by Fredrik Lundh.

    38 # Copyright (c) 1997-1998 by Secret Labs AB

    39 #

    40 # info@pythonware.com

    41 # http://www.pythonware.com

    42 #

    43 # --------------------------------------------------------------------

    44 # Permission to use, copy, modify, and distribute this software and

    45 # its associated documentation for any purpose and without fee is

    46 # hereby granted.  This software is provided as is.

    47 # --------------------------------------------------------------------

    48 

    49 VERSION = "1.6/1998-05-04"

    50 MAGIC   = "[SQUEEZE]"

    51 

    52 import base64, imp, marshal, os, string, sys, md5

    53 

    54 # --------------------------------------------------------------------

    55 # usage

    56 

    57 def usage():

    58     print

    59     print "SQUEEZE", VERSION, "(c) 1997-1998 by Secret Labs AB"

    60     print """\

    61 Convert a Python application to a compressed module package.

    62 

    63 Usage: squeeze [-1ux] -o app [-b start] modules... [-d files...]

    64 

    65 This utility creates a compressed package file named "app.pyz", which

    66 contains the given module files.  It also creates a bootstrap script

    67 named "app.py", which loads the package and imports the given "start"

    68 module to get things going.  Example:

    69 

    70         squeeze -o app -b appMain app*.py

    71 

    72 The -1 option tells squeeze to put the package file inside the boot-

    73 strap script using base64 encoding.  The result is a single text file

    74 containing the full application.

    75 

    76 The -u option disables compression.  Otherwise, the package will be

    77 compressed using zlib, and the user needs zlib to run the resulting

    78 application.

    79 

    80 The -d option can be used to put additional files in the package file.

    81 You can access these files via "__main__.open(filename)" (returns a

    82 StringIO file object).

    83 

    84 The -x option can be used with -d to create a self-extracting archive,

    85 instead of a package.  When the resulting script is executed, the

    86 data files are extracted.  Omit the -b option in this case.

    87 """

    88     sys.exit(1)

    89 

    90 

    91 # --------------------------------------------------------------------

    92 # squeezer -- collect squeezed modules

    93 

    94 class Squeezer:

    95 

    96     def __init__(self):

    97 

    98         self.rawbytes = self.bytes = 0

    99         self.modules = {}

   100 

   101     def addmodule(self, file):

   102 

   103         if file[-1] == "c":

   104             file = file[:-1]

   105 

   106         m = os.path.splitext(os.path.split(file)[1])[0]

   107 

   108         # read sourcefile

   109         f = open(file)

   110         codestring = f.read()

   111         f.close()

   112 

   113         # dump to file

   114         self.modules[m] = compile(codestring, file, "exec")

   115 

   116     def adddata(self, file):

   117 

   118         self.modules["+"+file] = open(file, "rb").read()

   119 

   120     def getarchive(self):

   121 

   122         # marshal our module dictionary

   123         data = marshal.dumps(self.modules)

   124         self.rawbytes = len(data)

   125 

   126         # return (compressed) dictionary

   127         if zlib:

   128             data = zlib.compress(data, 9)

   129         self.bytes = len(data)

   130 

   131         return data

   132 

   133     def getstatus(self):

   134         return self.bytes, self.rawbytes

   135 

   136 

   137 # --------------------------------------------------------------------

   138 # loader (used in bootstrap code)

   139 

   140 loader = """

   141 import ihooks

   142 

   143 PYZ_MODULE = 64

   144 

   145 class Loader(ihooks.ModuleLoader):

   146 

   147     def __init__(self, modules):

   148         self.__modules = modules

   149         return ihooks.ModuleLoader.__init__(self)

   150 

   151     def find_module(self, name, path = None):

   152         try:

   153             self.__modules[name]

   154             return None, None, (None, None, PYZ_MODULE)

   155         except KeyError:

   156             return ihooks.ModuleLoader.find_module(self, name, path)

   157 

   158     def load_module(self, name, stuff):

   159         file, filename, (suff, mode, type) = stuff

   160         if type != PYZ_MODULE:

   161             return ihooks.ModuleLoader.load_module(self, name, stuff)

   162         #print "PYZ:", "import", name

   163         code = self.__modules[name]

   164         del self.__modules[name] # no need to keep this one around

   165         m = self.hooks.add_module(name)

   166         m.__file__ = filename

   167         exec code in m.__dict__

   168         return m

   169 

   170 def boot(name, fp, size, offset = 0):

   171 

   172     global data

   173 

   174     try:

   175         import %(modules)s

   176     except ImportError:

   177         #print "PYZ:", "failed to load marshal and zlib libraries"

   178         return # cannot boot from PYZ file

   179     #print "PYZ:", "boot from", name+".PYZ"

   180 

   181     # load archive and install import hook

   182     if offset:

   183         data = fp[offset:]

   184     else:

   185         data = fp.read(size)

   186         fp.close()

   187 

   188     if len(data) != size:

   189         raise IOError, "package is truncated"

   190 

   191     data = marshal.loads(%(data)s)

   192 

   193     ihooks.install(ihooks.ModuleImporter(Loader(data)))

   194 """

   195 

   196 loaderopen = """

   197 def open(name):

   198     import StringIO

   199     try:

   200         return StringIO.StringIO(data["+"+name])

   201     except KeyError:

   202         raise IOError, (0, "no such file")

   203 """

   204 

   205 loaderexplode = """

   206 

   207 def explode():

   208     for k, v in data.items():

   209         if k[0] == "+":

   210             try:

   211                 open(k[1:], "wb").write(v)

   212                 print k[1:], "extracted ok"

   213             except IOError, v:

   214                 print k[1:], "failed:", "IOError", v

   215 

   216 """

   217 

   218 def getloader(data, zlib, package):

   219 

   220     s = loader

   221 

   222     if data:

   223         if explode:

   224             s = s + loaderexplode

   225         else:

   226             s = s + loaderopen

   227 

   228     if zlib:

   229         dict = {

   230             "modules": "marshal, zlib",

   231             "data":    "zlib.decompress(data)",

   232             }

   233     else:

   234         dict = {

   235             "modules": "marshal",

   236             "data":    "data",

   237             }

   238 

   239     s = s % dict

   240 

   241     return marshal.dumps(compile(s, "<package>", "exec"))

   242 

   243 

   244 # --------------------------------------------------------------------

   245 # Main

   246 # --------------------------------------------------------------------

   247 

   248 #

   249 # parse options

   250 

   251 import getopt, glob, sys

   252 

   253 try:

   254     opt, arg = getopt.getopt(sys.argv[1:], "1b:o:suzxd")

   255 except: usage()

   256 

   257 app = ""

   258 start = ""

   259 embed = 0

   260 zlib = 1

   261 explode = 0

   262 

   263 data = None

   264 

   265 for i, v in opt:

   266     if i == "-o":

   267         app = v

   268     elif i == "-b":

   269         start = "import " + v

   270     elif i == "-d":

   271         data = 0

   272     elif i == "-1":

   273         embed = 1

   274     elif i == "-z":

   275         zlib = 1

   276     elif i == "-u":

   277         zlib = 0

   278     elif i == "-x":

   279         explode = 1

   280         start = "explode()"

   281 

   282 print app, start

   283 

   284 if not app or not start:

   285     usage()

   286 

   287 bootstrap = app + ".py"

   288 archive   = app + ".pyz"

   289 

   290 archiveid = app

   291 if explode:

   292     archiveid = "this is a self-extracting archive. run the script to unpack"

   293 elif embed:

   294     archiveid = "this is an embedded package"

   295 elif zlib:

   296     archiveid = "this is a bootstrap script for a compressed package"

   297 else:

   298     archiveid = "this is a bootstrap script for an uncompressed package"

   299 

   300 #

   301 # import compression library (as necessary)

   302 

   303 if zlib:

   304     try:

   305         import zlib

   306     except ImportError:

   307         print "You must have the zlib module to generate compressed archives."

   308         print "Squeeze will create an uncompressed archive."

   309         zlib = None

   310 

   311 #

   312 # avoid overwriting files not generated by squeeze

   313 

   314 try:

   315     fp = open(bootstrap)

   316     s = fp.readline()

   317     s = fp.readline()

   318     string.index(s, MAGIC)

   319 except IOError:

   320     pass

   321 except ValueError:

   322     print bootstrap, "was not created by squeeze.  You have to manually"

   323     print "remove the file to proceed."

   324     sys.exit(1)

   325 

   326 #

   327 # collect modules

   328 

   329 sq = Squeezer()

   330 for patt in arg:

   331     if patt == "-d":

   332         data = 0

   333     else:

   334         for file in glob.glob(patt):

   335             if file != bootstrap:

   336                 if data is not None:

   337                     print file, "(data)"

   338                     sq.adddata(file)

   339                     data = data + 1

   340                 else:

   341                     print file

   342                     sq.addmodule(file)

   343 

   344 package = sq.getarchive()

   345 size = len(package)

   346 

   347 #

   348 # get loader

   349 

   350 loader = getloader(data, zlib, package)

   351 

   352 if zlib:

   353     zbegin, zend = "zlib.decompress(", ")"

   354     zimport = 'try:import zlib\n'\

   355         'except:raise RuntimeError,"requires zlib"\n'

   356     loader = zlib.compress(loader, 9)

   357 else:

   358     zbegin = zend = zimport = ""

   359 

   360 loaderlen = len(loader)

   361 

   362 magic = repr(imp.get_magic())

   363 version = string.split(sys.version)[0]

   364 

   365 magictest = 'import imp\n'\

   366     's="requires python %s or bytecode compatible"\n'\

   367     'if imp.get_magic()!=%s:raise RuntimeError,s' % (version, magic)

   368 

   369 #

   370 # generate script and package files

   371 

   372 if embed:

   373 

   374     # embedded archive

   375     data = base64.encodestring(loader + package)

   376 

   377     fp = open(bootstrap, "w")

   378     fp.write('''\

   379 #!/usr/bin/env python

   380 #%(MAGIC)s %(archiveid)s

   381 %(magictest)s

   382 %(zimport)simport base64,marshal

   383 s=base64.decodestring("""

   384 %(data)s""")

   385 exec marshal.loads(%(zbegin)ss[:%(loaderlen)d]%(zend)s)

   386 boot("%(app)s",s,%(size)d,%(loaderlen)d)

   387 %(start)s

   388 ''' % locals())

   389     bytes = fp.tell()

   390 

   391 else:

   392 

   393     # separate archive file

   394 

   395     fp = open(archive, "wb")

   396 

   397     fp.write(loader)

   398     fp.write(package)

   399 

   400     bytes = fp.tell()

   401 

   402     #

   403     # create bootstrap code

   404 

   405     fp = open(bootstrap, "w")

   406     fp.write("""\

   407 #!/usr/bin/env python

   408 #%(MAGIC)s %(archiveid)s

   409 %(magictest)s

   410 %(zimport)simport marshal,sys,os

   411 for p in filter(os.path.exists,map(lambda p:os.path.join(p,"%(archive)s"),sys.path)):

   412  f=open(p,"rb")

   413  exec marshal.loads(%(zbegin)sf.read(%(loaderlen)d)%(zend)s)

   414  boot("%(app)s",f,%(size)d)

   415  break

   416 %(start)s # failed to load package

   417 """ % locals())

   418     bytes = bytes + fp.tell()

   419 

   420 #

   421 # show statistics

   422 

   423 dummy, rawbytes = sq.getstatus()

   424 

   425 print "squeezed", rawbytes, "to", bytes, "bytes",

   426 print "(%d%%)" % (bytes * 100 / rawbytes)