author | martin.trojer@nokia.com |
Fri, 31 Jul 2009 15:01:17 +0100 | |
changeset 1 | 2fb8b9db1c86 |
permissions | -rw-r--r-- |
1
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
1 |
# -*- coding: iso-8859-1 -*- |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
2 |
""" Codec for the Punicode encoding, as specified in RFC 3492 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
3 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
4 |
Written by Martin v. Löwis. |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
5 |
""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
6 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
7 |
import codecs |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
8 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
9 |
##################### Encoding ##################################### |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
10 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
11 |
def segregate(str): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
12 |
"""3.1 Basic code point segregation""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
13 |
base = [] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
14 |
extended = {} |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
15 |
for c in str: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
16 |
if ord(c) < 128: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
17 |
base.append(c) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
18 |
else: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
19 |
extended[c] = 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
20 |
extended = extended.keys() |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
21 |
extended.sort() |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
22 |
return "".join(base).encode("ascii"),extended |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
23 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
24 |
def selective_len(str, max): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
25 |
"""Return the length of str, considering only characters below max.""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
26 |
res = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
27 |
for c in str: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
28 |
if ord(c) < max: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
29 |
res += 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
30 |
return res |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
31 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
32 |
def selective_find(str, char, index, pos): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
33 |
"""Return a pair (index, pos), indicating the next occurrence of |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
34 |
char in str. index is the position of the character considering |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
35 |
only ordinals up to and including char, and pos is the position in |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
36 |
the full string. index/pos is the starting position in the full |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
37 |
string.""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
38 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
39 |
l = len(str) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
40 |
while 1: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
41 |
pos += 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
42 |
if pos == l: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
43 |
return (-1, -1) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
44 |
c = str[pos] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
45 |
if c == char: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
46 |
return index+1, pos |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
47 |
elif c < char: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
48 |
index += 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
49 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
50 |
def insertion_unsort(str, extended): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
51 |
"""3.2 Insertion unsort coding""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
52 |
oldchar = 0x80 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
53 |
result = [] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
54 |
oldindex = -1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
55 |
for c in extended: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
56 |
index = pos = -1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
57 |
char = ord(c) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
58 |
curlen = selective_len(str, char) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
59 |
delta = (curlen+1) * (char - oldchar) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
60 |
while 1: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
61 |
index,pos = selective_find(str,c,index,pos) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
62 |
if index == -1: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
63 |
break |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
64 |
delta += index - oldindex |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
65 |
result.append(delta-1) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
66 |
oldindex = index |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
67 |
delta = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
68 |
oldchar = char |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
69 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
70 |
return result |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
71 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
72 |
def T(j, bias): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
73 |
# Punycode parameters: tmin = 1, tmax = 26, base = 36 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
74 |
res = 36 * (j + 1) - bias |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
75 |
if res < 1: return 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
76 |
if res > 26: return 26 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
77 |
return res |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
78 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
79 |
digits = "abcdefghijklmnopqrstuvwxyz0123456789" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
80 |
def generate_generalized_integer(N, bias): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
81 |
"""3.3 Generalized variable-length integers""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
82 |
result = [] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
83 |
j = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
84 |
while 1: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
85 |
t = T(j, bias) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
86 |
if N < t: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
87 |
result.append(digits[N]) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
88 |
return result |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
89 |
result.append(digits[t + ((N - t) % (36 - t))]) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
90 |
N = (N - t) // (36 - t) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
91 |
j += 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
92 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
93 |
def adapt(delta, first, numchars): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
94 |
if first: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
95 |
delta //= 700 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
96 |
else: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
97 |
delta //= 2 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
98 |
delta += delta // numchars |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
99 |
# ((base - tmin) * tmax) // 2 == 455 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
100 |
divisions = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
101 |
while delta > 455: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
102 |
delta = delta // 35 # base - tmin |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
103 |
divisions += 36 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
104 |
bias = divisions + (36 * delta // (delta + 38)) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
105 |
return bias |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
106 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
107 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
108 |
def generate_integers(baselen, deltas): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
109 |
"""3.4 Bias adaptation""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
110 |
# Punycode parameters: initial bias = 72, damp = 700, skew = 38 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
111 |
result = [] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
112 |
bias = 72 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
113 |
for points, delta in enumerate(deltas): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
114 |
s = generate_generalized_integer(delta, bias) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
115 |
result.extend(s) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
116 |
bias = adapt(delta, points==0, baselen+points+1) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
117 |
return "".join(result) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
118 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
119 |
def punycode_encode(text): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
120 |
base, extended = segregate(text) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
121 |
base = base.encode("ascii") |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
122 |
deltas = insertion_unsort(text, extended) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
123 |
extended = generate_integers(len(base), deltas) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
124 |
if base: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
125 |
return base + "-" + extended |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
126 |
return extended |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
127 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
128 |
##################### Decoding ##################################### |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
129 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
130 |
def decode_generalized_number(extended, extpos, bias, errors): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
131 |
"""3.3 Generalized variable-length integers""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
132 |
result = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
133 |
w = 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
134 |
j = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
135 |
while 1: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
136 |
try: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
137 |
char = ord(extended[extpos]) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
138 |
except IndexError: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
139 |
if errors == "strict": |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
140 |
raise UnicodeError, "incomplete punicode string" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
141 |
return extpos + 1, None |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
142 |
extpos += 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
143 |
if 0x41 <= char <= 0x5A: # A-Z |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
144 |
digit = char - 0x41 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
145 |
elif 0x30 <= char <= 0x39: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
146 |
digit = char - 22 # 0x30-26 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
147 |
elif errors == "strict": |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
148 |
raise UnicodeError("Invalid extended code point '%s'" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
149 |
% extended[extpos]) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
150 |
else: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
151 |
return extpos, None |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
152 |
t = T(j, bias) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
153 |
result += digit * w |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
154 |
if digit < t: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
155 |
return extpos, result |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
156 |
w = w * (36 - t) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
157 |
j += 1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
158 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
159 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
160 |
def insertion_sort(base, extended, errors): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
161 |
"""3.2 Insertion unsort coding""" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
162 |
char = 0x80 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
163 |
pos = -1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
164 |
bias = 72 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
165 |
extpos = 0 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
166 |
while extpos < len(extended): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
167 |
newpos, delta = decode_generalized_number(extended, extpos, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
168 |
bias, errors) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
169 |
if delta is None: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
170 |
# There was an error in decoding. We can't continue because |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
171 |
# synchronization is lost. |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
172 |
return base |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
173 |
pos += delta+1 |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
174 |
char += pos // (len(base) + 1) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
175 |
if char > 0x10FFFF: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
176 |
if errors == "strict": |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
177 |
raise UnicodeError, ("Invalid character U+%x" % char) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
178 |
char = ord('?') |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
179 |
pos = pos % (len(base) + 1) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
180 |
base = base[:pos] + unichr(char) + base[pos:] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
181 |
bias = adapt(delta, (extpos == 0), len(base)) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
182 |
extpos = newpos |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
183 |
return base |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
184 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
185 |
def punycode_decode(text, errors): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
186 |
pos = text.rfind("-") |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
187 |
if pos == -1: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
188 |
base = "" |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
189 |
extended = text |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
190 |
else: |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
191 |
base = text[:pos] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
192 |
extended = text[pos+1:] |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
193 |
base = unicode(base, "ascii", errors) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
194 |
extended = extended.upper() |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
195 |
return insertion_sort(base, extended, errors) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
196 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
197 |
### Codec APIs |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
198 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
199 |
class Codec(codecs.Codec): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
200 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
201 |
def encode(self,input,errors='strict'): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
202 |
res = punycode_encode(input) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
203 |
return res, len(input) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
204 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
205 |
def decode(self,input,errors='strict'): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
206 |
if errors not in ('strict', 'replace', 'ignore'): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
207 |
raise UnicodeError, "Unsupported error handling "+errors |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
208 |
res = punycode_decode(input, errors) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
209 |
return res, len(input) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
210 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
211 |
class IncrementalEncoder(codecs.IncrementalEncoder): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
212 |
def encode(self, input, final=False): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
213 |
return punycode_encode(input) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
214 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
215 |
class IncrementalDecoder(codecs.IncrementalDecoder): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
216 |
def decode(self, input, final=False): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
217 |
if self.errors not in ('strict', 'replace', 'ignore'): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
218 |
raise UnicodeError, "Unsupported error handling "+self.errors |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
219 |
return punycode_decode(input, self.errors) |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
220 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
221 |
class StreamWriter(Codec,codecs.StreamWriter): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
222 |
pass |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
223 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
224 |
class StreamReader(Codec,codecs.StreamReader): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
225 |
pass |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
226 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
227 |
### encodings module API |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
228 |
|
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
229 |
def getregentry(): |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
230 |
return codecs.CodecInfo( |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
231 |
name='punycode', |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
232 |
encode=Codec().encode, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
233 |
decode=Codec().decode, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
234 |
incrementalencoder=IncrementalEncoder, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
235 |
incrementaldecoder=IncrementalDecoder, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
236 |
streamwriter=StreamWriter, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
237 |
streamreader=StreamReader, |
2fb8b9db1c86
Initial QEMU (symbian-qemu-0.9.1-12) import
martin.trojer@nokia.com
parents:
diff
changeset
|
238 |
) |