0
|
1 |
#
|
|
2 |
# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
|
|
3 |
# All rights reserved.
|
|
4 |
# This component and the accompanying materials are made available
|
|
5 |
# under the terms of "Eclipse Public License v1.0"
|
|
6 |
# which accompanies this distribution, and is available
|
|
7 |
# at the URL "http://www.eclipse.org/legal/epl-v10.html".
|
|
8 |
#
|
|
9 |
# Initial Contributors:
|
|
10 |
# Nokia Corporation - initial contribution.
|
|
11 |
#
|
|
12 |
# Contributors:
|
|
13 |
#
|
|
14 |
# Description:
|
|
15 |
#
|
3
|
16 |
import shutil
|
0
|
17 |
|
|
18 |
import zipfile,zlib, StringIO, os, logging
|
|
19 |
import datetime
|
|
20 |
import tempfile
|
|
21 |
|
|
22 |
try:
|
|
23 |
from cElementTree import ElementTree
|
|
24 |
except ImportError:
|
|
25 |
try:
|
|
26 |
from elementtree import ElementTree
|
|
27 |
except ImportError:
|
|
28 |
try:
|
|
29 |
from xml.etree import cElementTree as ElementTree
|
|
30 |
except ImportError:
|
|
31 |
from xml.etree import ElementTree
|
|
32 |
|
3
|
33 |
from cone.public import api, utils, persistence, exceptions, parsecontext
|
0
|
34 |
from cone.public.api import Resource, Storage, Configuration, Folder
|
|
35 |
from cone.storage import metadata, common
|
|
36 |
from cone.confml import persistentconfml
|
|
37 |
|
|
38 |
|
|
39 |
class ZipException(exceptions.StorageException):
|
|
40 |
def __init__(self,value):
|
|
41 |
self.value = value
|
|
42 |
def __str__(self):
|
|
43 |
return repr(self.value)
|
|
44 |
|
|
45 |
|
|
46 |
class ZipStorage(common.StorageBase):
|
|
47 |
"""
|
|
48 |
A storage for zip file
|
|
49 |
"""
|
3
|
50 |
def __init__(self, path, mode, **kwargs):
|
0
|
51 |
"""
|
|
52 |
Open the given filename object as a cpf zipfile
|
|
53 |
"""
|
|
54 |
self.mode = mode
|
|
55 |
self.persistentmodule = persistentconfml
|
|
56 |
self.compression = zipfile.ZIP_DEFLATED
|
|
57 |
self.modified = False
|
5
|
58 |
self.cache = {}
|
0
|
59 |
self.logger = logging.getLogger('cone')
|
|
60 |
self.logger.debug("ZipStorage path %s open in mode %s" % (path,self.mode))
|
|
61 |
try:
|
|
62 |
# If opening the file in read/append mode check that the given file is a zipfile
|
|
63 |
if self.get_mode(mode) != self.MODE_WRITE:
|
|
64 |
if os.path.exists(path) and not zipfile.is_zipfile(path):
|
3
|
65 |
raise ZipException("The file %s is not a zip file!" % path)
|
|
66 |
# If creating a new file make sure that the path to the file exists
|
|
67 |
elif self.get_mode(mode) in (self.MODE_APPEND, self.MODE_WRITE):
|
|
68 |
dirname = os.path.dirname(path)
|
|
69 |
if dirname != '' and not os.path.exists(dirname):
|
|
70 |
os.makedirs(dirname)
|
0
|
71 |
self.zipfile = zipfile.ZipFile(path,self.mode,self.compression)
|
|
72 |
except IOError,e:
|
|
73 |
raise ZipException("ZipFile open error: %s" % e)
|
3
|
74 |
super(ZipStorage, self).__init__(path, mode)
|
0
|
75 |
|
|
76 |
def _zippath(self, path):
|
|
77 |
"""
|
|
78 |
Convert a norm path to zipfile path
|
|
79 |
"""
|
|
80 |
normpath = utils.resourceref.norm(path)
|
|
81 |
return normpath.lstrip('.')
|
|
82 |
|
|
83 |
@classmethod
|
|
84 |
def supported_storage(cls,path):
|
|
85 |
"""
|
|
86 |
Class method for determing if the given clas supports a storage by given path.
|
|
87 |
E.g. foo.zip, foo.cpd, foo/bar, http://foo.com/
|
|
88 |
@param path:
|
|
89 |
@return: Boolean value. True if the storage of the path is supported. False if not.
|
|
90 |
"""
|
|
91 |
if utils.resourceref.get_ext(path) == "zip" or \
|
|
92 |
utils.resourceref.get_ext(path) == "cpf":
|
|
93 |
return True
|
|
94 |
else:
|
|
95 |
return False
|
|
96 |
|
|
97 |
def open_resource(self,path,mode="r"):
|
|
98 |
strio = None
|
|
99 |
path = utils.resourceref.remove_begin_slash(path)
|
|
100 |
fullpath = utils.resourceref.join_refs([self.get_current_path(),path])
|
|
101 |
try:
|
|
102 |
if self.get_mode(mode) == self.MODE_READ:
|
|
103 |
if not self.is_resource(fullpath):
|
|
104 |
raise exceptions.NotResource("Resource is not found %s" % fullpath)
|
|
105 |
bytes = self.zipfile.read(fullpath)
|
|
106 |
strio = StringIO.StringIO(bytes)
|
|
107 |
elif self.get_mode(mode) == self.MODE_APPEND:
|
|
108 |
if not self.is_resource(fullpath):
|
|
109 |
raise exceptions.NotResource("Resource is not found %s" % fullpath)
|
|
110 |
bytes = self.zipfile.read(fullpath)
|
|
111 |
# delete the "old" resource
|
|
112 |
self.delete_resource(fullpath)
|
|
113 |
strio = StringIO.StringIO(bytes)
|
|
114 |
strio.seek(0, os.SEEK_END)
|
|
115 |
|
|
116 |
elif self.get_mode(mode) == self.MODE_WRITE:
|
|
117 |
if self.is_resource(fullpath):
|
|
118 |
# delete the "old" resource
|
|
119 |
self.delete_resource(fullpath)
|
|
120 |
# Create a new string buffer because the resource is overwritten
|
|
121 |
strio = StringIO.StringIO()
|
|
122 |
else:
|
|
123 |
raise ZipException("Unrecognized mode %s" % mode)
|
|
124 |
res = ZipFileResource(self,fullpath,mode,strio)
|
|
125 |
self.__opened__(res)
|
|
126 |
return res
|
|
127 |
except KeyError:
|
|
128 |
raise exceptions.NotResource(path)
|
|
129 |
|
|
130 |
def is_resource(self,path):
|
|
131 |
path = utils.resourceref.remove_begin_slash(path)
|
|
132 |
files = self.zipfile.namelist()
|
|
133 |
try:
|
|
134 |
i = files.index(path)
|
|
135 |
return True
|
|
136 |
except ValueError:
|
|
137 |
return False
|
|
138 |
|
|
139 |
def is_dir(self,path):
|
|
140 |
"""
|
|
141 |
Get an array of files in a folder
|
|
142 |
"""
|
|
143 |
try:
|
|
144 |
zinfo = self.zipfile.getinfo(utils.resourceref.add_end_slash(path))
|
|
145 |
return True
|
|
146 |
except KeyError:
|
|
147 |
return False
|
|
148 |
|
3
|
149 |
def list_resources(self, path, **kwargs):
|
0
|
150 |
"""
|
|
151 |
Get an array of files in a folder
|
|
152 |
"""
|
|
153 |
path = utils.resourceref.remove_begin_slash(path)
|
|
154 |
fullpath = utils.resourceref.join_refs([self.get_current_path(),path])
|
|
155 |
fullpath = self._zippath(fullpath)
|
|
156 |
retarray = []
|
|
157 |
filelist = self.zipfile.namelist()
|
|
158 |
for name in filelist:
|
|
159 |
(filepath,filename) = os.path.split(name)
|
|
160 |
curname = utils.resourceref.replace_dir(name, self.get_current_path(),'')
|
|
161 |
# return directories only if specified
|
3
|
162 |
if kwargs.get('empty_folders', False) == True or not self.is_dir(name):
|
0
|
163 |
# Skip the filename if it is marked as deleted
|
|
164 |
if self.__has_open__(name) and self.__get_open__(name)[-1].get_mode() == api.Storage.MODE_DELETE:
|
|
165 |
continue
|
|
166 |
if filepath == fullpath:
|
|
167 |
retarray.append(curname)
|
3
|
168 |
elif kwargs.get('recurse', False) and filepath.startswith(fullpath):
|
0
|
169 |
retarray.append(curname)
|
|
170 |
#retarray = sorted(utils.distinct_array(retarray))
|
|
171 |
return retarray
|
|
172 |
|
|
173 |
def import_resources(self,paths,storage,empty_folders=False):
|
|
174 |
for path in paths:
|
3
|
175 |
path = utils.resourceref.remove_begin_slash(utils.resourceref.norm(path))
|
0
|
176 |
if not storage.is_resource(path) and empty_folders==False:
|
|
177 |
logging.getLogger('cone').warning("The given path is not a Resource in the storage %s! Ignoring from export!" % path)
|
|
178 |
continue
|
|
179 |
if storage.is_resource(path):
|
|
180 |
wres = self.open_resource(path,'wb')
|
|
181 |
res = storage.open_resource(path,"rb")
|
|
182 |
wres.write(res.read())
|
|
183 |
wres.close()
|
|
184 |
res.close()
|
|
185 |
|
|
186 |
elif storage.is_folder(path) and empty_folders:
|
|
187 |
self.create_folder(path)
|
|
188 |
|
|
189 |
def export_resources(self,paths,storage,empty_folders=False):
|
|
190 |
|
|
191 |
for path in paths:
|
|
192 |
if not self.is_resource(path) and empty_folders==False:
|
|
193 |
logging.getLogger('cone').warning("The given path is not a Resource in this storage %s! Ignoring from export!" % path)
|
|
194 |
continue
|
|
195 |
if self.is_resource(path):
|
|
196 |
wres = storage.open_resource(path,'wb')
|
|
197 |
res = self.open_resource(path,"rb")
|
|
198 |
wres.write(res.read())
|
|
199 |
wres.close()
|
|
200 |
res.close()
|
|
201 |
|
|
202 |
if self.is_folder(path) and empty_folders:
|
|
203 |
storage.create_folder(path)
|
|
204 |
|
|
205 |
|
|
206 |
def close_resource(self, res):
|
|
207 |
"""
|
|
208 |
Close the given resource instance. Normally this is called by the Resource object
|
|
209 |
in its own close.
|
|
210 |
@param res: the resource object to close.
|
|
211 |
"""
|
|
212 |
try:
|
|
213 |
self.__closed__(res)
|
|
214 |
if self.get_mode(self.mode) != api.Storage.MODE_READ and \
|
|
215 |
(res.get_mode() == api.Storage.MODE_WRITE or res.get_mode() == api.Storage.MODE_APPEND):
|
|
216 |
self.zipfile.writestr(res.path,res.getvalue())
|
|
217 |
except KeyError,e:
|
|
218 |
raise exceptions.StorageException("No such %s open resource! %s" % (res.path,e))
|
|
219 |
|
|
220 |
|
|
221 |
def save_resource(self, res):
|
|
222 |
"""
|
|
223 |
Flush the changes of a given resource instance. Normally this is called by the Resource object
|
|
224 |
in its own save.
|
|
225 |
@param res: the resource to the resource to save.
|
|
226 |
"""
|
|
227 |
if not self.__has_resource__(res):
|
|
228 |
raise exceptions.NotResource("No such %s open resource!" % res.path)
|
|
229 |
else:
|
|
230 |
self.zipfile.writestr(res.path,res.getvalue())
|
|
231 |
|
|
232 |
def delete_resource(self,path):
|
|
233 |
"""
|
|
234 |
Delete the given resource from storage
|
|
235 |
@param res : Resource object to the resource
|
|
236 |
raises a NotSupportedException exception if delete operation is not supported by the storage
|
|
237 |
"""
|
|
238 |
# First close all open resources
|
|
239 |
for res in self.__get_open__(path):
|
|
240 |
self.__closed__(res)
|
|
241 |
self.modified = True
|
|
242 |
self.zipfile.filelist.remove(self.zipfile.NameToInfo[path])
|
|
243 |
del self.zipfile.NameToInfo[path]
|
|
244 |
|
|
245 |
|
|
246 |
def create_folder(self,path):
|
|
247 |
"""
|
|
248 |
Create a folder entry to a path
|
|
249 |
@param path : path to the folder
|
|
250 |
"""
|
|
251 |
fullpath = utils.resourceref.join_refs([self.get_current_path(),path])
|
|
252 |
fullpath = utils.resourceref.add_end_slash(self._zippath(fullpath))
|
|
253 |
if self.is_folder(fullpath):
|
|
254 |
# delete the "old" resource
|
|
255 |
self.delete_resource(fullpath)
|
|
256 |
now = datetime.datetime.now()
|
|
257 |
zinfo = zipfile.ZipInfo(fullpath,
|
|
258 |
(now.year,now.month, now.day,
|
|
259 |
now.hour, now.minute, now.second)
|
|
260 |
)
|
|
261 |
# set an external attribute for directory entry
|
|
262 |
zinfo.external_attr = 0x10
|
|
263 |
zinfo.extract_version = 10
|
|
264 |
self.zipfile.writestr(zinfo,'')
|
|
265 |
|
|
266 |
def delete_folder(self,path):
|
|
267 |
"""
|
|
268 |
Delete a folder entry from a path. The path must be empty.
|
|
269 |
@param path : path to the folder
|
|
270 |
"""
|
|
271 |
pass
|
|
272 |
|
|
273 |
def is_folder(self,path):
|
|
274 |
"""
|
|
275 |
Check if the given path is an existing folder in the storage
|
|
276 |
@param path : path to the folder
|
|
277 |
"""
|
|
278 |
fullpath = utils.resourceref.join_refs([self.get_current_path(),path])
|
|
279 |
folderpath = self._zippath(fullpath)
|
|
280 |
return self.is_dir(folderpath)
|
|
281 |
|
|
282 |
def close(self):
|
|
283 |
if self.zipfile:
|
|
284 |
super(ZipStorage,self).close()
|
|
285 |
self.zipfile.close()
|
|
286 |
# Recreate the zip file if the zip has been modified to make a zip without
|
|
287 |
# duplicate local file entries
|
|
288 |
else:
|
|
289 |
raise exceptions.StorageException('Storage %s has been already closed!' % self.path)
|
|
290 |
|
5
|
291 |
if self.modified or self.cache:
|
|
292 |
logging.getLogger('cone').debug("Recreating the ZIP output file")
|
|
293 |
oldfile = None
|
|
294 |
newzipfile = None
|
|
295 |
fh, tmp_path = tempfile.mkstemp(suffix='.zip')
|
|
296 |
shutil.move(self.path, tmp_path)
|
|
297 |
oldfile = zipfile.ZipFile(tmp_path,"r")
|
|
298 |
newzipfile = zipfile.ZipFile(self.path,"w",self.compression)
|
|
299 |
for fileinfo in oldfile.infolist():
|
|
300 |
if fileinfo.filename not in self.cache.keys():
|
|
301 |
newzipfile.writestr(fileinfo, oldfile.read(fileinfo.filename))
|
|
302 |
for filename in sorted(self.cache.keys()):
|
|
303 |
logging.getLogger('cone').debug("Adding pre-cached file %s." % filename)
|
|
304 |
newzipfile.write(self.cache[filename], arcname=filename)
|
|
305 |
if oldfile: oldfile.close()
|
|
306 |
if newzipfile: newzipfile.close()
|
|
307 |
os.close(fh)
|
|
308 |
os.unlink(tmp_path)
|
|
309 |
logging.getLogger('cone').debug("Recreating the ZIP output file completed.")
|
|
310 |
|
|
311 |
self.zipfile = None
|
|
312 |
|
0
|
313 |
def unload(self, path, object):
|
|
314 |
"""
|
|
315 |
Dump a given object to the storage (reference is fetched from the object)
|
|
316 |
@param object: The object to dump to the storage, which is expected to be an instance
|
|
317 |
of Base class.
|
|
318 |
"""
|
|
319 |
# Add the current path in front of the given path
|
|
320 |
path = utils.resourceref.join_refs([self.get_current_path(), path])
|
|
321 |
if not isinstance(object, api.Configuration):
|
|
322 |
raise exceptions.StorageException("Cannot dump object type %s" % object.__class__)
|
|
323 |
if self.get_mode(self.mode) != api.Storage.MODE_READ:
|
|
324 |
res = self.open_resource(path,"wb")
|
|
325 |
data = "%s" % self.persistentmodule.dumps(object)
|
|
326 |
res.write(data)
|
|
327 |
res.close()
|
|
328 |
return
|
|
329 |
|
|
330 |
def load(self, path):
|
|
331 |
"""
|
|
332 |
Load an from a reference.
|
|
333 |
"""
|
|
334 |
# Add the current path in front of the given path
|
|
335 |
path = utils.resourceref.join_refs([self.get_current_path(), path])
|
|
336 |
if not utils.resourceref.get_ext(path) == "confml":
|
|
337 |
raise exceptions.StorageException("Cannot load reference type %s" % utils.resourceref.get_ext(path))
|
|
338 |
if self.is_resource(path):
|
|
339 |
res = self.open_resource(path,"r")
|
|
340 |
# read the resource with persistentmodule
|
3
|
341 |
parsecontext.get_confml_context().current_file = path
|
0
|
342 |
try:
|
|
343 |
obj = self.persistentmodule.loads(res.read())
|
|
344 |
obj.set_path(path)
|
|
345 |
res.close()
|
|
346 |
return obj
|
|
347 |
except exceptions.ParseError,e:
|
3
|
348 |
parsecontext.get_confml_context().handle_exception(e)
|
|
349 |
#logging.getLogger('cone').error("Resource %s parsing failed with exception: %s" % (path,e))
|
0
|
350 |
# returning an empty config in case of xml parsing failure.
|
|
351 |
return api.Configuration(path)
|
|
352 |
else:
|
|
353 |
raise exceptions.NotResource("No such %s resource!" % path)
|
|
354 |
|
|
355 |
|
|
356 |
class ZipFileResource(Resource):
|
|
357 |
def __init__(self,storage,path,mode,handle):
|
|
358 |
Resource.__init__(self,storage,path,mode)
|
|
359 |
self.handle = handle
|
|
360 |
|
|
361 |
def read(self,bytes=0):
|
|
362 |
if bytes == 0:
|
|
363 |
return self.handle.read()
|
|
364 |
else:
|
|
365 |
return self.handle.read(bytes)
|
|
366 |
|
|
367 |
def write(self,string):
|
|
368 |
if self.get_mode() == api.Storage.MODE_READ:
|
|
369 |
raise exceptions.StorageException("Writing attempted to %s in read-only mode." % self.path)
|
|
370 |
else:
|
|
371 |
self.handle.write(string)
|
|
372 |
|
|
373 |
def truncate(self,size=0):
|
|
374 |
raise exceptions.NotSupportedException()
|
|
375 |
|
|
376 |
def save(self):
|
|
377 |
self.storage.save_resource(self)
|
|
378 |
|
|
379 |
def close(self):
|
|
380 |
self.storage.close_resource(self)
|
|
381 |
self.handle.close()
|
|
382 |
|
|
383 |
def get_size(self):
|
|
384 |
if self.get_mode() == api.Storage.MODE_WRITE:
|
|
385 |
raise exceptions.StorageException("Reading resource size attempted to %s in write-only mode." % self.path)
|
|
386 |
return len(self.handle.getvalue())
|
|
387 |
|
|
388 |
def getvalue(self):
|
|
389 |
return self.handle.getvalue()
|
|
390 |
|
|
391 |
def get_content_info(self):
|
|
392 |
if self.content_info == None:
|
3
|
393 |
self.content_info = api.make_content_info(self, self.handle.getvalue())
|
0
|
394 |
|
5
|
395 |
return self.content_info
|