1
|
1 |
# Copyright (C) 2005, Giovanni Bajo
|
|
2 |
# Based on previous work under copyright (c) 2002 McMillan Enterprises, Inc.
|
|
3 |
#
|
|
4 |
# This program is free software; you can redistribute it and/or
|
|
5 |
# modify it under the terms of the GNU General Public License
|
|
6 |
# as published by the Free Software Foundation; either version 2
|
|
7 |
# of the License, or (at your option) any later version.
|
|
8 |
#
|
|
9 |
# In addition to the permissions in the GNU General Public License, the
|
|
10 |
# authors give you unlimited permission to link or embed the compiled
|
|
11 |
# version of this file into combinations with other programs, and to
|
|
12 |
# distribute those combinations without any restriction coming from the
|
|
13 |
# use of this file. (The General Public License restrictions do apply in
|
|
14 |
# other respects; for example, they cover modification of the file, and
|
|
15 |
# distribution when not linked into a combine executable.)
|
|
16 |
#
|
|
17 |
# This program is distributed in the hope that it will be useful,
|
|
18 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
19 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
20 |
# GNU General Public License for more details.
|
|
21 |
#
|
|
22 |
# You should have received a copy of the GNU General Public License
|
|
23 |
# along with this program; if not, write to the Free Software
|
|
24 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
|
25 |
|
|
26 |
# subclasses may not need marshal or struct, but since they're
|
|
27 |
# builtin, importing is safe.
|
|
28 |
#
|
|
29 |
# While an Archive is really an abstraction for any "filesystem
|
|
30 |
# within a file", it is tuned for use with imputil.FuncImporter.
|
|
31 |
# This assumes it contains python code objects, indexed by the
|
|
32 |
# the internal name (ie, no '.py').
|
|
33 |
# See carchive.py for a more general archive (contains anything)
|
|
34 |
# that can be understood by a C program.
|
|
35 |
|
|
36 |
_verbose = 0
|
|
37 |
_listdir = None
|
|
38 |
_environ = None
|
|
39 |
|
|
40 |
# **NOTE** This module is used during bootstrap. Import *ONLY* builtin modules.
|
|
41 |
import marshal
|
|
42 |
import struct
|
|
43 |
import imp
|
|
44 |
import sys
|
|
45 |
|
|
46 |
_c_suffixes = filter(lambda x: x[2] == imp.C_EXTENSION, imp.get_suffixes())
|
|
47 |
|
|
48 |
for nm in ('nt', 'posix', 'dos', 'os2', 'mac'):
|
|
49 |
if nm in sys.builtin_module_names:
|
|
50 |
mod = __import__(nm)
|
|
51 |
_listdir = mod.listdir
|
|
52 |
_environ = mod.environ
|
|
53 |
break
|
|
54 |
|
|
55 |
if hasattr(sys, 'version_info'):
|
|
56 |
versuffix = '%d%d'%(sys.version_info[0],sys.version_info[1])
|
|
57 |
else:
|
|
58 |
vers = sys.version
|
|
59 |
dot1 = dot2 = 0
|
|
60 |
for i in range(len(vers)):
|
|
61 |
if vers[i] == '.':
|
|
62 |
if dot1:
|
|
63 |
dot2 = i
|
|
64 |
break
|
|
65 |
else:
|
|
66 |
dot1 = i
|
|
67 |
else:
|
|
68 |
dot2 = len(vers)
|
|
69 |
versuffix = '%s%s' % (vers[:dot1], vers[dot1+1:dot2])
|
|
70 |
|
|
71 |
if "-vi" in sys.argv[1:]:
|
|
72 |
_verbose = 1
|
|
73 |
|
|
74 |
class Archive:
|
|
75 |
""" A base class for a repository of python code objects.
|
|
76 |
The extract method is used by imputil.ArchiveImporter
|
|
77 |
to get code objects by name (fully qualified name), so
|
|
78 |
an enduser "import a.b" would become
|
|
79 |
extract('a.__init__')
|
|
80 |
extract('a.b')
|
|
81 |
"""
|
|
82 |
MAGIC = 'PYL\0'
|
|
83 |
HDRLEN = 12 # default is MAGIC followed by python's magic, int pos of toc
|
|
84 |
TOCPOS = 8
|
|
85 |
TRLLEN = 0 # default - no trailer
|
|
86 |
TOCTMPLT = {} #
|
|
87 |
os = None
|
|
88 |
_bincache = None
|
|
89 |
def __init__(self, path=None, start=0):
|
|
90 |
"Initialize an Archive. If path is omitted, it will be an empty Archive."
|
|
91 |
self.toc = None
|
|
92 |
self.path = path
|
|
93 |
self.start = start
|
|
94 |
import imp
|
|
95 |
self.pymagic = imp.get_magic()
|
|
96 |
if path is not None:
|
|
97 |
self.lib = open(self.path, 'rb')
|
|
98 |
self.checkmagic()
|
|
99 |
self.loadtoc()
|
|
100 |
|
|
101 |
####### Sub-methods of __init__ - override as needed #############
|
|
102 |
def checkmagic(self):
|
|
103 |
""" Overridable.
|
|
104 |
Check to see if the file object self.lib actually has a file
|
|
105 |
we understand.
|
|
106 |
"""
|
|
107 |
self.lib.seek(self.start) #default - magic is at start of file
|
|
108 |
if self.lib.read(len(self.MAGIC)) != self.MAGIC:
|
|
109 |
raise RuntimeError, "%s is not a valid %s archive file" \
|
|
110 |
% (self.path, self.__class__.__name__)
|
|
111 |
if self.lib.read(len(self.pymagic)) != self.pymagic:
|
|
112 |
raise RuntimeError, "%s has version mismatch to dll" % (self.path)
|
|
113 |
self.lib.read(4)
|
|
114 |
|
|
115 |
def loadtoc(self):
|
|
116 |
""" Overridable.
|
|
117 |
Default: After magic comes an int (4 byte native) giving the
|
|
118 |
position of the TOC within self.lib.
|
|
119 |
Default: The TOC is a marshal-able string.
|
|
120 |
"""
|
|
121 |
self.lib.seek(self.start + self.TOCPOS)
|
|
122 |
(offset,) = struct.unpack('=i', self.lib.read(4))
|
|
123 |
self.lib.seek(self.start + offset)
|
|
124 |
self.toc = marshal.load(self.lib)
|
|
125 |
|
|
126 |
######## This is what is called by FuncImporter #######
|
|
127 |
## Since an Archive is flat, we ignore parent and modname.
|
|
128 |
#XXX obsolete - imputil only code
|
|
129 |
## def get_code(self, parent, modname, fqname):
|
|
130 |
#### if _verbose:
|
|
131 |
#### print "I: get_code(%s, %s, %s, %s)" % (self, parent, modname, fqname)
|
|
132 |
## iname = fqname
|
|
133 |
## if parent:
|
|
134 |
## iname = '%s.%s' % (parent.__dict__.get('__iname__', parent.__name__), modname)
|
|
135 |
#### if _verbose:
|
|
136 |
#### print "I: get_code: iname is %s" % iname
|
|
137 |
## rslt = self.extract(iname) # None if not found, (ispkg, code) otherwise
|
|
138 |
#### if _verbose:
|
|
139 |
#### print 'I: get_code: rslt', rslt
|
|
140 |
## if rslt is None:
|
|
141 |
#### if _verbose:
|
|
142 |
#### print 'I: get_code: importer', getattr(parent, "__importer__", None),'self',self
|
|
143 |
## # check the cache if there is no parent or self is the parents importer
|
|
144 |
## if parent is None or getattr(parent, "__importer__", None) is self:
|
|
145 |
#### if _verbose:
|
|
146 |
#### print 'I: get_code: cached 1',iname
|
|
147 |
## file, desc = Archive._bincache.get(iname, (None, None))
|
|
148 |
#### if _verbose:
|
|
149 |
#### print 'I: get_code: file',file,'desc',desc
|
|
150 |
## if file:
|
|
151 |
## try:
|
|
152 |
## fp = open(file, desc[1])
|
|
153 |
## except IOError:
|
|
154 |
## pass
|
|
155 |
## else:
|
|
156 |
## module = imp.load_module(fqname, fp, file, desc)
|
|
157 |
## if _verbose:
|
|
158 |
## print "I: import %s found %s" % (fqname, file)
|
|
159 |
## return 0, module, {'__file__':file}
|
|
160 |
## if _verbose:
|
|
161 |
## print "I: import %s failed" % fqname
|
|
162 |
##
|
|
163 |
## return None
|
|
164 |
##
|
|
165 |
## ispkg, code = rslt
|
|
166 |
## values = {'__file__' : code.co_filename, '__iname__' : iname}
|
|
167 |
## if ispkg:
|
|
168 |
## values['__path__'] = [fqname]
|
|
169 |
## if _verbose:
|
|
170 |
## print "I: import %s found %s" % (fqname, iname)
|
|
171 |
## return ispkg, code, values
|
|
172 |
|
|
173 |
####### Core method - Override as needed #########
|
|
174 |
def extract(self, name):
|
|
175 |
""" Get the object corresponding to name, or None.
|
|
176 |
For use with imputil ArchiveImporter, object is a python code object.
|
|
177 |
'name' is the name as specified in an 'import name'.
|
|
178 |
'import a.b' will become:
|
|
179 |
extract('a') (return None because 'a' is not a code object)
|
|
180 |
extract('a.__init__') (return a code object)
|
|
181 |
extract('a.b') (return a code object)
|
|
182 |
Default implementation:
|
|
183 |
self.toc is a dict
|
|
184 |
self.toc[name] is pos
|
|
185 |
self.lib has the code object marshal-ed at pos
|
|
186 |
"""
|
|
187 |
ispkg, pos = self.toc.get(name, (0,None))
|
|
188 |
if pos is None:
|
|
189 |
return None
|
|
190 |
self.lib.seek(self.start + pos)
|
|
191 |
return ispkg, marshal.load(self.lib)
|
|
192 |
|
|
193 |
########################################################################
|
|
194 |
# Informational methods
|
|
195 |
|
|
196 |
def contents(self):
|
|
197 |
"""Return a list of the contents
|
|
198 |
Default implementation assumes self.toc is a dict like object.
|
|
199 |
Not required by ArchiveImporter.
|
|
200 |
"""
|
|
201 |
return self.toc.keys()
|
|
202 |
|
|
203 |
########################################################################
|
|
204 |
# Building
|
|
205 |
|
|
206 |
####### Top level method - shouldn't need overriding #######
|
|
207 |
def build(self, path, lTOC):
|
|
208 |
"""Create an archive file of name 'path'.
|
|
209 |
lTOC is a 'logical TOC' - a list of (name, path, ...)
|
|
210 |
where name is the internal name, eg 'a'
|
|
211 |
and path is a file to get the object from, eg './a.pyc'.
|
|
212 |
"""
|
|
213 |
self.path = path
|
|
214 |
self.lib = open(path, 'wb')
|
|
215 |
#reserve space for the header
|
|
216 |
if self.HDRLEN:
|
|
217 |
self.lib.write('\0'*self.HDRLEN)
|
|
218 |
|
|
219 |
#create an empty toc
|
|
220 |
|
|
221 |
if type(self.TOCTMPLT) == type({}):
|
|
222 |
self.toc = {}
|
|
223 |
else: # assume callable
|
|
224 |
self.toc = self.TOCTMPLT()
|
|
225 |
|
|
226 |
for tocentry in lTOC:
|
|
227 |
self.add(tocentry) # the guts of the archive
|
|
228 |
|
|
229 |
tocpos = self.lib.tell()
|
|
230 |
self.save_toc(tocpos)
|
|
231 |
if self.TRLLEN:
|
|
232 |
self.save_trailer(tocpos)
|
|
233 |
if self.HDRLEN:
|
|
234 |
self.update_headers(tocpos)
|
|
235 |
self.lib.close()
|
|
236 |
|
|
237 |
|
|
238 |
####### manages keeping the internal TOC and the guts in sync #######
|
|
239 |
def add(self, entry):
|
|
240 |
"""Override this to influence the mechanics of the Archive.
|
|
241 |
Assumes entry is a seq beginning with (nm, pth, ...) where
|
|
242 |
nm is the key by which we'll be asked for the object.
|
|
243 |
pth is the name of where we find the object. Overrides of
|
|
244 |
get_obj_from can make use of further elements in entry.
|
|
245 |
"""
|
|
246 |
if self.os is None:
|
|
247 |
import os
|
|
248 |
self.os = os
|
|
249 |
nm = entry[0]
|
|
250 |
pth = entry[1]
|
|
251 |
pynm, ext = self.os.path.splitext(self.os.path.basename(pth))
|
|
252 |
ispkg = pynm == '__init__'
|
|
253 |
assert ext in ('.pyc', '.pyo')
|
|
254 |
self.toc[nm] = (ispkg, self.lib.tell())
|
|
255 |
f = open(entry[1], 'rb')
|
|
256 |
f.seek(8) #skip magic and timestamp
|
|
257 |
self.lib.write(f.read())
|
|
258 |
|
|
259 |
def save_toc(self, tocpos):
|
|
260 |
"""Default - toc is a dict
|
|
261 |
Gets marshaled to self.lib
|
|
262 |
"""
|
|
263 |
marshal.dump(self.toc, self.lib)
|
|
264 |
|
|
265 |
def save_trailer(self, tocpos):
|
|
266 |
"""Default - not used"""
|
|
267 |
pass
|
|
268 |
|
|
269 |
def update_headers(self, tocpos):
|
|
270 |
"""Default - MAGIC + Python's magic + tocpos"""
|
|
271 |
self.lib.seek(self.start)
|
|
272 |
self.lib.write(self.MAGIC)
|
|
273 |
self.lib.write(self.pymagic)
|
|
274 |
self.lib.write(struct.pack('=i', tocpos))
|
|
275 |
|
|
276 |
class DummyZlib:
|
|
277 |
def decompress(self, data):
|
|
278 |
return data
|
|
279 |
def compress(self, data, lvl):
|
|
280 |
return data
|
|
281 |
|
|
282 |
import iu
|
|
283 |
##############################################################
|
|
284 |
#
|
|
285 |
# ZlibArchive - an archive with compressed entries
|
|
286 |
#
|
|
287 |
class ZlibArchive(Archive):
|
|
288 |
MAGIC = 'PYZ\0'
|
|
289 |
TOCPOS = 8
|
|
290 |
HDRLEN = 16
|
|
291 |
TRLLEN = 0
|
|
292 |
TOCTMPLT = {}
|
|
293 |
LEVEL = 9
|
|
294 |
|
|
295 |
def __init__(self, path=None, offset=None, level=9):
|
|
296 |
if path is None:
|
|
297 |
offset = 0
|
|
298 |
elif offset is None:
|
|
299 |
for i in range(len(path)-1, -1, -1):
|
|
300 |
if path[i] == '?':
|
|
301 |
offset = int(path[i+1:])
|
|
302 |
path = path[:i]
|
|
303 |
break
|
|
304 |
else:
|
|
305 |
offset = 0
|
|
306 |
self.LEVEL = level
|
|
307 |
Archive.__init__(self, path, offset)
|
|
308 |
# dynamic import so not imported if not needed
|
|
309 |
global zlib
|
|
310 |
if self.LEVEL:
|
|
311 |
try:
|
|
312 |
import zlib
|
|
313 |
except ImportError:
|
|
314 |
zlib = DummyZlib()
|
|
315 |
else:
|
|
316 |
zlib = DummyZlib()
|
|
317 |
|
|
318 |
|
|
319 |
def extract(self, name):
|
|
320 |
(ispkg, pos, lngth) = self.toc.get(name, (0, None, 0))
|
|
321 |
if pos is None:
|
|
322 |
return None
|
|
323 |
self.lib.seek(self.start + pos)
|
|
324 |
try:
|
|
325 |
co = marshal.loads(zlib.decompress(self.lib.read(lngth)))
|
|
326 |
except EOFError:
|
|
327 |
raise ImportError, "PYZ entry '%s' failed to unmarshal" % name
|
|
328 |
return ispkg, co
|
|
329 |
|
|
330 |
def add(self, entry):
|
|
331 |
if self.os is None:
|
|
332 |
import os
|
|
333 |
self.os = os
|
|
334 |
nm = entry[0]
|
|
335 |
pth = entry[1]
|
|
336 |
base, ext = self.os.path.splitext(self.os.path.basename(pth))
|
|
337 |
ispkg = base == '__init__'
|
|
338 |
try:
|
|
339 |
txt = open(pth[:-1], 'r').read()+'\n'
|
|
340 |
except (IOError, OSError):
|
|
341 |
try:
|
|
342 |
f = open(pth, 'rb')
|
|
343 |
f.seek(8) #skip magic and timestamp
|
|
344 |
bytecode = f.read()
|
|
345 |
marshal.loads(bytecode).co_filename # to make sure it's valid
|
|
346 |
obj = zlib.compress(bytecode, self.LEVEL)
|
|
347 |
except (IOError, ValueError, EOFError, AttributeError):
|
|
348 |
raise ValueError("bad bytecode in %s and no source" % pth)
|
|
349 |
else:
|
|
350 |
txt = iu._string_replace(txt, '\r\n', '\n')
|
|
351 |
try:
|
|
352 |
co = compile(txt, "%s/%s" % (self.path, nm), 'exec')
|
|
353 |
except SyntaxError, e:
|
|
354 |
print "Syntax error in", pth[:-1]
|
|
355 |
print e.args
|
|
356 |
raise
|
|
357 |
obj = zlib.compress(marshal.dumps(co), self.LEVEL)
|
|
358 |
self.toc[nm] = (ispkg, self.lib.tell(), len(obj))
|
|
359 |
self.lib.write(obj)
|
|
360 |
def update_headers(self, tocpos):
|
|
361 |
"""add level"""
|
|
362 |
Archive.update_headers(self, tocpos)
|
|
363 |
self.lib.write(struct.pack('!i', self.LEVEL))
|
|
364 |
def checkmagic(self):
|
|
365 |
Archive.checkmagic(self)
|
|
366 |
self.LEVEL = struct.unpack('!i', self.lib.read(4))[0]
|
|
367 |
|
|
368 |
class PYZOwner(iu.Owner):
|
|
369 |
def __init__(self, path):
|
|
370 |
self.pyz = ZlibArchive(path)
|
|
371 |
iu.Owner.__init__(self, path)
|
|
372 |
def getmod(self, nm, newmod=imp.new_module):
|
|
373 |
rslt = self.pyz.extract(nm)
|
|
374 |
if rslt is None:
|
|
375 |
return None
|
|
376 |
ispkg, co = rslt
|
|
377 |
mod = newmod(nm)
|
|
378 |
try:
|
|
379 |
mod.__file__ = co.co_filename
|
|
380 |
except AttributeError:
|
|
381 |
raise ImportError, "PYZ entry '%s' (%s) is not a valid code object" % (nm, repr(co))
|
|
382 |
if ispkg:
|
|
383 |
if _environ.has_key('_MEIPASS2'):
|
|
384 |
localpath = _environ['_MEIPASS2'][:-1]
|
|
385 |
else:
|
|
386 |
localpath = iu._os_path_dirname(self.path)
|
|
387 |
mod.__path__ = [self.path, localpath, iu._os_path_dirname(mod.__file__)]
|
|
388 |
#print "PYZOwner setting %s's __path__: %s" % (nm, mod.__path__)
|
|
389 |
importer = iu.PathImportDirector(mod.__path__,
|
|
390 |
{self.path:PkgInPYZImporter(nm, self),
|
|
391 |
localpath:ExtInPkgImporter(localpath, nm)},
|
|
392 |
[iu.DirOwner])
|
|
393 |
mod.__importsub__ = importer.getmod
|
|
394 |
mod.__co__ = co
|
|
395 |
return mod
|
|
396 |
|
|
397 |
class PkgInPYZImporter:
|
|
398 |
def __init__(self, name, owner):
|
|
399 |
self.name = name
|
|
400 |
self.owner = owner
|
|
401 |
def getmod(self, nm):
|
|
402 |
#print "PkgInPYZImporter.getmod %s -> %s" % (nm, self.name+'.'+nm)
|
|
403 |
return self.owner.getmod(self.name+'.'+nm)
|
|
404 |
class ExtInPkgImporter(iu.DirOwner):
|
|
405 |
def __init__(self, path, prefix):
|
|
406 |
iu.DirOwner.__init__(self, path)
|
|
407 |
self.prefix = prefix
|
|
408 |
def getmod(self, nm):
|
|
409 |
return iu.DirOwner.getmod(self, self.prefix+'.'+nm)
|
|
410 |
|
|
411 |
#XXX this should also get moved out
|
|
412 |
##iu._globalownertypes.insert(0, PYZOwner)
|
|
413 |
##iu.ImportManager().install()
|