|
1 # |
|
2 # Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
3 # All rights reserved. |
|
4 # This component and the accompanying materials are made available |
|
5 # under the terms of "Eclipse Public License v1.0" |
|
6 # which accompanies this distribution, and is available |
|
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 # |
|
9 # Initial Contributors: |
|
10 # Nokia Corporation - initial contribution. |
|
11 # |
|
12 # Contributors: |
|
13 # |
|
14 # Description: |
|
15 # |
|
16 |
|
17 import zipfile,zlib, StringIO, os, logging |
|
18 import datetime |
|
19 import tempfile |
|
20 |
|
21 try: |
|
22 from cElementTree import ElementTree |
|
23 except ImportError: |
|
24 try: |
|
25 from elementtree import ElementTree |
|
26 except ImportError: |
|
27 try: |
|
28 from xml.etree import cElementTree as ElementTree |
|
29 except ImportError: |
|
30 from xml.etree import ElementTree |
|
31 |
|
32 from cone.public import api, utils, persistence, exceptions |
|
33 from cone.public.api import Resource, Storage, Configuration, Folder |
|
34 from cone.storage import metadata, common |
|
35 from cone.confml import persistentconfml |
|
36 |
|
37 |
|
38 class ZipException(exceptions.StorageException): |
|
39 def __init__(self,value): |
|
40 self.value = value |
|
41 def __str__(self): |
|
42 return repr(self.value) |
|
43 |
|
44 |
|
45 class ZipStorage(common.StorageBase): |
|
46 """ |
|
47 A storage for zip file |
|
48 """ |
|
49 TEMP_FILE = '_temp_%i.zip' % os.getpid() |
|
50 def __init__(self, path ,mode, **kwargs): |
|
51 """ |
|
52 Open the given filename object as a cpf zipfile |
|
53 """ |
|
54 self.mode = mode |
|
55 self.persistentmodule = persistentconfml |
|
56 self.compression = zipfile.ZIP_DEFLATED |
|
57 self.modified = False |
|
58 self.logger = logging.getLogger('cone') |
|
59 self.logger.debug("ZipStorage path %s open in mode %s" % (path,self.mode)) |
|
60 try: |
|
61 # If opening the file in read/append mode check that the given file is a zipfile |
|
62 if self.get_mode(mode) != self.MODE_WRITE: |
|
63 if os.path.exists(path) and not zipfile.is_zipfile(path): |
|
64 raise ZipException("The file %s is not a zip file!" % path) |
|
65 self.zipfile = zipfile.ZipFile(path,self.mode,self.compression) |
|
66 except IOError,e: |
|
67 raise ZipException("ZipFile open error: %s" % e) |
|
68 super(ZipStorage, self).__init__(path) |
|
69 |
|
70 def _zippath(self, path): |
|
71 """ |
|
72 Convert a norm path to zipfile path |
|
73 """ |
|
74 normpath = utils.resourceref.norm(path) |
|
75 return normpath.lstrip('.') |
|
76 |
|
77 @classmethod |
|
78 def supported_storage(cls,path): |
|
79 """ |
|
80 Class method for determing if the given clas supports a storage by given path. |
|
81 E.g. foo.zip, foo.cpd, foo/bar, http://foo.com/ |
|
82 @param path: |
|
83 @return: Boolean value. True if the storage of the path is supported. False if not. |
|
84 """ |
|
85 if utils.resourceref.get_ext(path) == "zip" or \ |
|
86 utils.resourceref.get_ext(path) == "cpf": |
|
87 return True |
|
88 else: |
|
89 return False |
|
90 |
|
91 def open_resource(self,path,mode="r"): |
|
92 strio = None |
|
93 path = utils.resourceref.remove_begin_slash(path) |
|
94 fullpath = utils.resourceref.join_refs([self.get_current_path(),path]) |
|
95 try: |
|
96 if self.get_mode(mode) == self.MODE_READ: |
|
97 if not self.is_resource(fullpath): |
|
98 raise exceptions.NotResource("Resource is not found %s" % fullpath) |
|
99 bytes = self.zipfile.read(fullpath) |
|
100 strio = StringIO.StringIO(bytes) |
|
101 elif self.get_mode(mode) == self.MODE_APPEND: |
|
102 if not self.is_resource(fullpath): |
|
103 raise exceptions.NotResource("Resource is not found %s" % fullpath) |
|
104 bytes = self.zipfile.read(fullpath) |
|
105 # delete the "old" resource |
|
106 self.delete_resource(fullpath) |
|
107 strio = StringIO.StringIO(bytes) |
|
108 strio.seek(0, os.SEEK_END) |
|
109 |
|
110 elif self.get_mode(mode) == self.MODE_WRITE: |
|
111 if self.is_resource(fullpath): |
|
112 # delete the "old" resource |
|
113 self.delete_resource(fullpath) |
|
114 # Create a new string buffer because the resource is overwritten |
|
115 strio = StringIO.StringIO() |
|
116 else: |
|
117 raise ZipException("Unrecognized mode %s" % mode) |
|
118 res = ZipFileResource(self,fullpath,mode,strio) |
|
119 self.__opened__(res) |
|
120 return res |
|
121 except KeyError: |
|
122 raise exceptions.NotResource(path) |
|
123 |
|
124 def is_resource(self,path): |
|
125 path = utils.resourceref.remove_begin_slash(path) |
|
126 files = self.zipfile.namelist() |
|
127 try: |
|
128 i = files.index(path) |
|
129 return True |
|
130 except ValueError: |
|
131 return False |
|
132 |
|
133 def is_dir(self,path): |
|
134 """ |
|
135 Get an array of files in a folder |
|
136 """ |
|
137 try: |
|
138 zinfo = self.zipfile.getinfo(utils.resourceref.add_end_slash(path)) |
|
139 return True |
|
140 except KeyError: |
|
141 return False |
|
142 |
|
143 def list_resources(self,path,recurse=False, empty_folders=False): |
|
144 """ |
|
145 Get an array of files in a folder |
|
146 """ |
|
147 path = utils.resourceref.remove_begin_slash(path) |
|
148 fullpath = utils.resourceref.join_refs([self.get_current_path(),path]) |
|
149 fullpath = self._zippath(fullpath) |
|
150 retarray = [] |
|
151 filelist = self.zipfile.namelist() |
|
152 for name in filelist: |
|
153 (filepath,filename) = os.path.split(name) |
|
154 curname = utils.resourceref.replace_dir(name, self.get_current_path(),'') |
|
155 # return directories only if specified |
|
156 if empty_folders == True or not self.is_dir(name): |
|
157 # Skip the filename if it is marked as deleted |
|
158 if self.__has_open__(name) and self.__get_open__(name)[-1].get_mode() == api.Storage.MODE_DELETE: |
|
159 continue |
|
160 if filepath == fullpath: |
|
161 retarray.append(curname) |
|
162 elif recurse and filepath.startswith(fullpath): |
|
163 retarray.append(curname) |
|
164 #retarray = sorted(utils.distinct_array(retarray)) |
|
165 return retarray |
|
166 |
|
167 def import_resources(self,paths,storage,empty_folders=False): |
|
168 for path in paths: |
|
169 if not storage.is_resource(path) and empty_folders==False: |
|
170 logging.getLogger('cone').warning("The given path is not a Resource in the storage %s! Ignoring from export!" % path) |
|
171 continue |
|
172 if storage.is_resource(path): |
|
173 wres = self.open_resource(path,'wb') |
|
174 res = storage.open_resource(path,"rb") |
|
175 wres.write(res.read()) |
|
176 wres.close() |
|
177 res.close() |
|
178 |
|
179 elif storage.is_folder(path) and empty_folders: |
|
180 self.create_folder(path) |
|
181 |
|
182 def export_resources(self,paths,storage,empty_folders=False): |
|
183 |
|
184 for path in paths: |
|
185 if not self.is_resource(path) and empty_folders==False: |
|
186 logging.getLogger('cone').warning("The given path is not a Resource in this storage %s! Ignoring from export!" % path) |
|
187 continue |
|
188 if self.is_resource(path): |
|
189 wres = storage.open_resource(path,'wb') |
|
190 res = self.open_resource(path,"rb") |
|
191 wres.write(res.read()) |
|
192 wres.close() |
|
193 res.close() |
|
194 |
|
195 if self.is_folder(path) and empty_folders: |
|
196 storage.create_folder(path) |
|
197 |
|
198 |
|
199 def close_resource(self, res): |
|
200 """ |
|
201 Close the given resource instance. Normally this is called by the Resource object |
|
202 in its own close. |
|
203 @param res: the resource object to close. |
|
204 """ |
|
205 try: |
|
206 self.__closed__(res) |
|
207 if self.get_mode(self.mode) != api.Storage.MODE_READ and \ |
|
208 (res.get_mode() == api.Storage.MODE_WRITE or res.get_mode() == api.Storage.MODE_APPEND): |
|
209 self.zipfile.writestr(res.path,res.getvalue()) |
|
210 except KeyError,e: |
|
211 raise exceptions.StorageException("No such %s open resource! %s" % (res.path,e)) |
|
212 |
|
213 |
|
214 def save_resource(self, res): |
|
215 """ |
|
216 Flush the changes of a given resource instance. Normally this is called by the Resource object |
|
217 in its own save. |
|
218 @param res: the resource to the resource to save. |
|
219 """ |
|
220 if not self.__has_resource__(res): |
|
221 raise exceptions.NotResource("No such %s open resource!" % res.path) |
|
222 else: |
|
223 self.zipfile.writestr(res.path,res.getvalue()) |
|
224 |
|
225 def delete_resource(self,path): |
|
226 """ |
|
227 Delete the given resource from storage |
|
228 @param res : Resource object to the resource |
|
229 raises a NotSupportedException exception if delete operation is not supported by the storage |
|
230 """ |
|
231 # First close all open resources |
|
232 for res in self.__get_open__(path): |
|
233 self.__closed__(res) |
|
234 self.modified = True |
|
235 self.zipfile.filelist.remove(self.zipfile.NameToInfo[path]) |
|
236 del self.zipfile.NameToInfo[path] |
|
237 |
|
238 |
|
239 def create_folder(self,path): |
|
240 """ |
|
241 Create a folder entry to a path |
|
242 @param path : path to the folder |
|
243 """ |
|
244 fullpath = utils.resourceref.join_refs([self.get_current_path(),path]) |
|
245 fullpath = utils.resourceref.add_end_slash(self._zippath(fullpath)) |
|
246 if self.is_folder(fullpath): |
|
247 # delete the "old" resource |
|
248 self.delete_resource(fullpath) |
|
249 now = datetime.datetime.now() |
|
250 zinfo = zipfile.ZipInfo(fullpath, |
|
251 (now.year,now.month, now.day, |
|
252 now.hour, now.minute, now.second) |
|
253 ) |
|
254 # set an external attribute for directory entry |
|
255 zinfo.external_attr = 0x10 |
|
256 zinfo.extract_version = 10 |
|
257 self.zipfile.writestr(zinfo,'') |
|
258 |
|
259 def delete_folder(self,path): |
|
260 """ |
|
261 Delete a folder entry from a path. The path must be empty. |
|
262 @param path : path to the folder |
|
263 """ |
|
264 pass |
|
265 |
|
266 def is_folder(self,path): |
|
267 """ |
|
268 Check if the given path is an existing folder in the storage |
|
269 @param path : path to the folder |
|
270 """ |
|
271 fullpath = utils.resourceref.join_refs([self.get_current_path(),path]) |
|
272 folderpath = self._zippath(fullpath) |
|
273 return self.is_dir(folderpath) |
|
274 |
|
275 def close(self): |
|
276 if self.zipfile: |
|
277 super(ZipStorage,self).close() |
|
278 self.zipfile.close() |
|
279 # Recreate the zip file if the zip has been modified to make a zip without |
|
280 # duplicate local file entries |
|
281 if self.modified: |
|
282 oldfile = None |
|
283 newzipfile = None |
|
284 tmp_path = os.path.join(tempfile.gettempdir(), self.TEMP_FILE) |
|
285 os.rename(self.path, tmp_path) |
|
286 oldfile = zipfile.ZipFile(tmp_path,"r") |
|
287 newzipfile = zipfile.ZipFile(self.path,"w",self.compression) |
|
288 for fileinfo in oldfile.infolist(): |
|
289 newzipfile.writestr(fileinfo, oldfile.read(fileinfo.filename)) |
|
290 if oldfile: oldfile.close() |
|
291 if newzipfile: newzipfile.close() |
|
292 os.unlink(tmp_path) |
|
293 self.zipfile = None |
|
294 else: |
|
295 raise exceptions.StorageException('Storage %s has been already closed!' % self.path) |
|
296 |
|
297 def unload(self, path, object): |
|
298 """ |
|
299 Dump a given object to the storage (reference is fetched from the object) |
|
300 @param object: The object to dump to the storage, which is expected to be an instance |
|
301 of Base class. |
|
302 """ |
|
303 # Add the current path in front of the given path |
|
304 path = utils.resourceref.join_refs([self.get_current_path(), path]) |
|
305 if not isinstance(object, api.Configuration): |
|
306 raise exceptions.StorageException("Cannot dump object type %s" % object.__class__) |
|
307 if self.get_mode(self.mode) != api.Storage.MODE_READ: |
|
308 res = self.open_resource(path,"wb") |
|
309 data = "%s" % self.persistentmodule.dumps(object) |
|
310 res.write(data) |
|
311 res.close() |
|
312 return |
|
313 |
|
314 def load(self, path): |
|
315 """ |
|
316 Load an from a reference. |
|
317 """ |
|
318 # Add the current path in front of the given path |
|
319 path = utils.resourceref.join_refs([self.get_current_path(), path]) |
|
320 if not utils.resourceref.get_ext(path) == "confml": |
|
321 raise exceptions.StorageException("Cannot load reference type %s" % utils.resourceref.get_ext(path)) |
|
322 if self.is_resource(path): |
|
323 res = self.open_resource(path,"r") |
|
324 # read the resource with persistentmodule |
|
325 try: |
|
326 obj = self.persistentmodule.loads(res.read()) |
|
327 obj.set_path(path) |
|
328 res.close() |
|
329 return obj |
|
330 except exceptions.ParseError,e: |
|
331 logging.getLogger('cone').error("Resource %s parsing failed with exception: %s" % (path,e)) |
|
332 # returning an empty config in case of xml parsing failure. |
|
333 return api.Configuration(path) |
|
334 else: |
|
335 raise exceptions.NotResource("No such %s resource!" % path) |
|
336 |
|
337 |
|
338 class ZipFileResource(Resource): |
|
339 def __init__(self,storage,path,mode,handle): |
|
340 Resource.__init__(self,storage,path,mode) |
|
341 self.handle = handle |
|
342 |
|
343 def read(self,bytes=0): |
|
344 if bytes == 0: |
|
345 return self.handle.read() |
|
346 else: |
|
347 return self.handle.read(bytes) |
|
348 |
|
349 def write(self,string): |
|
350 if self.get_mode() == api.Storage.MODE_READ: |
|
351 raise exceptions.StorageException("Writing attempted to %s in read-only mode." % self.path) |
|
352 else: |
|
353 self.handle.write(string) |
|
354 |
|
355 def truncate(self,size=0): |
|
356 raise exceptions.NotSupportedException() |
|
357 |
|
358 def save(self): |
|
359 self.storage.save_resource(self) |
|
360 |
|
361 def close(self): |
|
362 self.storage.close_resource(self) |
|
363 self.handle.close() |
|
364 |
|
365 def get_size(self): |
|
366 if self.get_mode() == api.Storage.MODE_WRITE: |
|
367 raise exceptions.StorageException("Reading resource size attempted to %s in write-only mode." % self.path) |
|
368 return len(self.handle.getvalue()) |
|
369 |
|
370 def getvalue(self): |
|
371 return self.handle.getvalue() |
|
372 |
|
373 def get_content_info(self): |
|
374 if self.content_info == None: |
|
375 self.content_info = utils.make_content_info(self, self.handle.getvalue()) |
|
376 |
|
377 return self.content_info |