author | Deepak Modgil <Deepak.Modgil@Nokia.com> |
Fri, 03 Apr 2009 17:19:34 +0100 | |
changeset 0 | ae805ac0140d |
permissions | -rw-r--r-- |
0
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
1 |
""" Python 'utf-16' Codec |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
2 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
3 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
4 |
Written by Marc-Andre Lemburg (mal@lemburg.com). |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
5 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
6 |
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
7 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
8 |
""" |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
9 |
import codecs, sys |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
10 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
11 |
### Codec APIs |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
12 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
13 |
encode = codecs.utf_16_encode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
14 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
15 |
def decode(input, errors='strict'): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
16 |
return codecs.utf_16_decode(input, errors, True) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
17 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
18 |
class IncrementalEncoder(codecs.IncrementalEncoder): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
19 |
def __init__(self, errors='strict'): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
20 |
codecs.IncrementalEncoder.__init__(self, errors) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
21 |
self.encoder = None |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
22 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
23 |
def encode(self, input, final=False): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
24 |
if self.encoder is None: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
25 |
result = codecs.utf_16_encode(input, self.errors)[0] |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
26 |
if sys.byteorder == 'little': |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
27 |
self.encoder = codecs.utf_16_le_encode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
28 |
else: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
29 |
self.encoder = codecs.utf_16_be_encode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
30 |
return result |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
31 |
return self.encoder(input, self.errors)[0] |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
32 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
33 |
def reset(self): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
34 |
codecs.IncrementalEncoder.reset(self) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
35 |
self.encoder = None |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
36 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
37 |
class IncrementalDecoder(codecs.BufferedIncrementalDecoder): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
38 |
def __init__(self, errors='strict'): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
39 |
codecs.BufferedIncrementalDecoder.__init__(self, errors) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
40 |
self.decoder = None |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
41 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
42 |
def _buffer_decode(self, input, errors, final): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
43 |
if self.decoder is None: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
44 |
(output, consumed, byteorder) = \ |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
45 |
codecs.utf_16_ex_decode(input, errors, 0, final) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
46 |
if byteorder == -1: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
47 |
self.decoder = codecs.utf_16_le_decode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
48 |
elif byteorder == 1: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
49 |
self.decoder = codecs.utf_16_be_decode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
50 |
elif consumed >= 2: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
51 |
raise UnicodeError("UTF-16 stream does not start with BOM") |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
52 |
return (output, consumed) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
53 |
return self.decoder(input, self.errors, final) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
54 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
55 |
def reset(self): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
56 |
codecs.BufferedIncrementalDecoder.reset(self) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
57 |
self.decoder = None |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
58 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
59 |
class StreamWriter(codecs.StreamWriter): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
60 |
def __init__(self, stream, errors='strict'): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
61 |
self.bom_written = False |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
62 |
codecs.StreamWriter.__init__(self, stream, errors) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
63 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
64 |
def encode(self, input, errors='strict'): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
65 |
self.bom_written = True |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
66 |
result = codecs.utf_16_encode(input, errors) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
67 |
if sys.byteorder == 'little': |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
68 |
self.encode = codecs.utf_16_le_encode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
69 |
else: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
70 |
self.encode = codecs.utf_16_be_encode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
71 |
return result |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
72 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
73 |
class StreamReader(codecs.StreamReader): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
74 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
75 |
def reset(self): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
76 |
codecs.StreamReader.reset(self) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
77 |
try: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
78 |
del self.decode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
79 |
except AttributeError: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
80 |
pass |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
81 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
82 |
def decode(self, input, errors='strict'): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
83 |
(object, consumed, byteorder) = \ |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
84 |
codecs.utf_16_ex_decode(input, errors, 0, False) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
85 |
if byteorder == -1: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
86 |
self.decode = codecs.utf_16_le_decode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
87 |
elif byteorder == 1: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
88 |
self.decode = codecs.utf_16_be_decode |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
89 |
elif consumed>=2: |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
90 |
raise UnicodeError,"UTF-16 stream does not start with BOM" |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
91 |
return (object, consumed) |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
92 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
93 |
### encodings module API |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
94 |
|
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
95 |
def getregentry(): |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
96 |
return codecs.CodecInfo( |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
97 |
name='utf-16', |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
98 |
encode=encode, |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
99 |
decode=decode, |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
100 |
incrementalencoder=IncrementalEncoder, |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
101 |
incrementaldecoder=IncrementalDecoder, |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
102 |
streamreader=StreamReader, |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
103 |
streamwriter=StreamWriter, |
ae805ac0140d
DP tools release version Revision: 200912
Deepak Modgil <Deepak.Modgil@Nokia.com>
parents:
diff
changeset
|
104 |
) |