|
1 """Manage shelves of pickled objects. |
|
2 |
|
3 A "shelf" is a persistent, dictionary-like object. The difference |
|
4 with dbm databases is that the values (not the keys!) in a shelf can |
|
5 be essentially arbitrary Python objects -- anything that the "pickle" |
|
6 module can handle. This includes most class instances, recursive data |
|
7 types, and objects containing lots of shared sub-objects. The keys |
|
8 are ordinary strings. |
|
9 |
|
10 To summarize the interface (key is a string, data is an arbitrary |
|
11 object): |
|
12 |
|
13 import shelve |
|
14 d = shelve.open(filename) # open, with (g)dbm filename -- no suffix |
|
15 |
|
16 d[key] = data # store data at key (overwrites old data if |
|
17 # using an existing key) |
|
18 data = d[key] # retrieve a COPY of the data at key (raise |
|
19 # KeyError if no such key) -- NOTE that this |
|
20 # access returns a *copy* of the entry! |
|
21 del d[key] # delete data stored at key (raises KeyError |
|
22 # if no such key) |
|
23 flag = d.has_key(key) # true if the key exists; same as "key in d" |
|
24 list = d.keys() # a list of all existing keys (slow!) |
|
25 |
|
26 d.close() # close it |
|
27 |
|
28 Dependent on the implementation, closing a persistent dictionary may |
|
29 or may not be necessary to flush changes to disk. |
|
30 |
|
31 Normally, d[key] returns a COPY of the entry. This needs care when |
|
32 mutable entries are mutated: for example, if d[key] is a list, |
|
33 d[key].append(anitem) |
|
34 does NOT modify the entry d[key] itself, as stored in the persistent |
|
35 mapping -- it only modifies the copy, which is then immediately |
|
36 discarded, so that the append has NO effect whatsoever. To append an |
|
37 item to d[key] in a way that will affect the persistent mapping, use: |
|
38 data = d[key] |
|
39 data.append(anitem) |
|
40 d[key] = data |
|
41 |
|
42 To avoid the problem with mutable entries, you may pass the keyword |
|
43 argument writeback=True in the call to shelve.open. When you use: |
|
44 d = shelve.open(filename, writeback=True) |
|
45 then d keeps a cache of all entries you access, and writes them all back |
|
46 to the persistent mapping when you call d.close(). This ensures that |
|
47 such usage as d[key].append(anitem) works as intended. |
|
48 |
|
49 However, using keyword argument writeback=True may consume vast amount |
|
50 of memory for the cache, and it may make d.close() very slow, if you |
|
51 access many of d's entries after opening it in this way: d has no way to |
|
52 check which of the entries you access are mutable and/or which ones you |
|
53 actually mutate, so it must cache, and write back at close, all of the |
|
54 entries that you access. You can call d.sync() to write back all the |
|
55 entries in the cache, and empty the cache (d.sync() also synchronizes |
|
56 the persistent dictionary on disk, if feasible). |
|
57 """ |
|
58 |
|
59 # Try using cPickle and cStringIO if available. |
|
60 |
|
61 try: |
|
62 from cPickle import Pickler, Unpickler |
|
63 except ImportError: |
|
64 from pickle import Pickler, Unpickler |
|
65 |
|
66 try: |
|
67 from cStringIO import StringIO |
|
68 except ImportError: |
|
69 from StringIO import StringIO |
|
70 |
|
71 import UserDict |
|
72 import warnings |
|
73 |
|
74 __all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"] |
|
75 |
|
76 class Shelf(UserDict.DictMixin): |
|
77 """Base class for shelf implementations. |
|
78 |
|
79 This is initialized with a dictionary-like object. |
|
80 See the module's __doc__ string for an overview of the interface. |
|
81 """ |
|
82 |
|
83 def __init__(self, dict, protocol=None, writeback=False): |
|
84 self.dict = dict |
|
85 if protocol is None: |
|
86 protocol = 0 |
|
87 self._protocol = protocol |
|
88 self.writeback = writeback |
|
89 self.cache = {} |
|
90 |
|
91 def keys(self): |
|
92 return self.dict.keys() |
|
93 |
|
94 def __len__(self): |
|
95 return len(self.dict) |
|
96 |
|
97 def has_key(self, key): |
|
98 return self.dict.has_key(key) |
|
99 |
|
100 def __contains__(self, key): |
|
101 return self.dict.has_key(key) |
|
102 |
|
103 def get(self, key, default=None): |
|
104 if self.dict.has_key(key): |
|
105 return self[key] |
|
106 return default |
|
107 |
|
108 def __getitem__(self, key): |
|
109 try: |
|
110 value = self.cache[key] |
|
111 except KeyError: |
|
112 f = StringIO(self.dict[key]) |
|
113 value = Unpickler(f).load() |
|
114 if self.writeback: |
|
115 self.cache[key] = value |
|
116 return value |
|
117 |
|
118 def __setitem__(self, key, value): |
|
119 if self.writeback: |
|
120 self.cache[key] = value |
|
121 f = StringIO() |
|
122 p = Pickler(f, self._protocol) |
|
123 p.dump(value) |
|
124 self.dict[key] = f.getvalue() |
|
125 |
|
126 def __delitem__(self, key): |
|
127 del self.dict[key] |
|
128 try: |
|
129 del self.cache[key] |
|
130 except KeyError: |
|
131 pass |
|
132 |
|
133 def close(self): |
|
134 self.sync() |
|
135 try: |
|
136 self.dict.close() |
|
137 except AttributeError: |
|
138 pass |
|
139 self.dict = 0 |
|
140 |
|
141 def __del__(self): |
|
142 if not hasattr(self, 'writeback'): |
|
143 # __init__ didn't succeed, so don't bother closing |
|
144 return |
|
145 self.close() |
|
146 |
|
147 def sync(self): |
|
148 if self.writeback and self.cache: |
|
149 self.writeback = False |
|
150 for key, entry in self.cache.iteritems(): |
|
151 self[key] = entry |
|
152 self.writeback = True |
|
153 self.cache = {} |
|
154 if hasattr(self.dict, 'sync'): |
|
155 self.dict.sync() |
|
156 |
|
157 |
|
158 class BsdDbShelf(Shelf): |
|
159 """Shelf implementation using the "BSD" db interface. |
|
160 |
|
161 This adds methods first(), next(), previous(), last() and |
|
162 set_location() that have no counterpart in [g]dbm databases. |
|
163 |
|
164 The actual database must be opened using one of the "bsddb" |
|
165 modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or |
|
166 bsddb.rnopen) and passed to the constructor. |
|
167 |
|
168 See the module's __doc__ string for an overview of the interface. |
|
169 """ |
|
170 |
|
171 def __init__(self, dict, protocol=None, writeback=False): |
|
172 Shelf.__init__(self, dict, protocol, writeback) |
|
173 |
|
174 def set_location(self, key): |
|
175 (key, value) = self.dict.set_location(key) |
|
176 f = StringIO(value) |
|
177 return (key, Unpickler(f).load()) |
|
178 |
|
179 def next(self): |
|
180 (key, value) = self.dict.next() |
|
181 f = StringIO(value) |
|
182 return (key, Unpickler(f).load()) |
|
183 |
|
184 def previous(self): |
|
185 (key, value) = self.dict.previous() |
|
186 f = StringIO(value) |
|
187 return (key, Unpickler(f).load()) |
|
188 |
|
189 def first(self): |
|
190 (key, value) = self.dict.first() |
|
191 f = StringIO(value) |
|
192 return (key, Unpickler(f).load()) |
|
193 |
|
194 def last(self): |
|
195 (key, value) = self.dict.last() |
|
196 f = StringIO(value) |
|
197 return (key, Unpickler(f).load()) |
|
198 |
|
199 |
|
200 class DbfilenameShelf(Shelf): |
|
201 """Shelf implementation using the "anydbm" generic dbm interface. |
|
202 |
|
203 This is initialized with the filename for the dbm database. |
|
204 See the module's __doc__ string for an overview of the interface. |
|
205 """ |
|
206 |
|
207 def __init__(self, filename, flag='c', protocol=None, writeback=False): |
|
208 import anydbm |
|
209 Shelf.__init__(self, anydbm.open(filename, flag), protocol, writeback) |
|
210 |
|
211 |
|
212 def open(filename, flag='c', protocol=None, writeback=False): |
|
213 """Open a persistent dictionary for reading and writing. |
|
214 |
|
215 The filename parameter is the base filename for the underlying |
|
216 database. As a side-effect, an extension may be added to the |
|
217 filename and more than one file may be created. The optional flag |
|
218 parameter has the same interpretation as the flag parameter of |
|
219 anydbm.open(). The optional protocol parameter specifies the |
|
220 version of the pickle protocol (0, 1, or 2). |
|
221 |
|
222 See the module's __doc__ string for an overview of the interface. |
|
223 """ |
|
224 |
|
225 return DbfilenameShelf(filename, flag, protocol, writeback) |