1 """ Unicode Mapping Parser and Codec Generator.

     2 

     3 This script parses Unicode mapping files as available from the Unicode

     4 site (ftp://ftp.unicode.org/Public/MAPPINGS/) and creates Python codec

     5 modules from them. The codecs use the standard character mapping codec

     6 to actually apply the mapping.

     7 

     8 Synopsis: gencodec.py dir codec_prefix

     9 

    10 All files in dir are scanned and those producing non-empty mappings

    11 will be written to <codec_prefix><mapname>.py with <mapname> being the

    12 first part of the map's filename ('a' in a.b.c.txt) converted to

    13 lowercase with hyphens replaced by underscores.

    14 

    15 The tool also writes marshalled versions of the mapping tables to the

    16 same location (with .mapping extension).

    17 

    18 Written by Marc-Andre Lemburg (mal@lemburg.com).

    19 

    20 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.

    21 (c) Copyright Guido van Rossum, 2000.

    22 

    23 Table generation:

    24 (c) Copyright Marc-Andre Lemburg, 2005.

    25     Licensed to PSF under a Contributor Agreement.

    26 

    27 """#"

    28 

    29 import re, os, marshal, codecs

    30 

    31 # Maximum allowed size of charmap tables

    32 MAX_TABLE_SIZE = 8192

    33 

    34 # Standard undefined Unicode code point

    35 UNI_UNDEFINED = unichr(0xFFFE)

    36 

    37 mapRE = re.compile('((?:0x[0-9a-fA-F]+\+?)+)'

    38                    '\s+'

    39                    '((?:(?:0x[0-9a-fA-Z]+|<[A-Za-z]+>)\+?)*)'

    40                    '\s*'

    41                    '(#.+)?')

    42 

    43 def parsecodes(codes,

    44                len=len, filter=filter,range=range):

    45 

    46     """ Converts code combinations to either a single code integer

    47         or a tuple of integers.

    48 

    49         meta-codes (in angular brackets, e.g. <LR> and <RL>) are

    50         ignored.

    51 

    52         Empty codes or illegal ones are returned as None.

    53 

    54     """

    55     if not codes:

    56         return None

    57     l = codes.split('+')

    58     if len(l) == 1:

    59         return int(l[0],16)

    60     for i in range(len(l)):

    61         try:

    62             l[i] = int(l[i],16)

    63         except ValueError:

    64             l[i] = None

    65     l = filter(lambda x: x is not None, l)

    66     if len(l) == 1:

    67         return l[0]

    68     else:

    69         return tuple(l)

    70 

    71 def readmap(filename):

    72 

    73     f = open(filename,'r')

    74     lines = f.readlines()

    75     f.close()

    76     enc2uni = {}

    77     identity = []

    78     unmapped = range(256)

    79 

    80     # UTC mapping tables per convention don't include the identity

    81     # mappings for code points 0x00 - 0x1F and 0x7F, unless these are

    82     # explicitly mapped to different characters or undefined

    83     for i in range(32) + [127]:

    84         identity.append(i)

    85         unmapped.remove(i)

    86         enc2uni[i] = (i, 'CONTROL CHARACTER')

    87 

    88     for line in lines:

    89         line = line.strip()

    90         if not line or line[0] == '#':

    91             continue

    92         m = mapRE.match(line)

    93         if not m:

    94             #print '* not matched: %s' % repr(line)

    95             continue

    96         enc,uni,comment = m.groups()

    97         enc = parsecodes(enc)

    98         uni = parsecodes(uni)

    99         if comment is None:

   100             comment = ''

   101         else:

   102             comment = comment[1:].strip()

   103         if enc < 256:

   104             if enc in unmapped:

   105                 unmapped.remove(enc)

   106             if enc == uni:

   107                 identity.append(enc)

   108             enc2uni[enc] = (uni,comment)

   109         else:

   110             enc2uni[enc] = (uni,comment)

   111 

   112     # If there are more identity-mapped entries than unmapped entries,

   113     # it pays to generate an identity dictionary first, and add explicit

   114     # mappings to None for the rest

   115     if len(identity) >= len(unmapped):

   116         for enc in unmapped:

   117             enc2uni[enc] = (None, "")

   118         enc2uni['IDENTITY'] = 256

   119 

   120     return enc2uni

   121 

   122 def hexrepr(t, precision=4):

   123 

   124     if t is None:

   125         return 'None'

   126     try:

   127         len(t)

   128     except:

   129         return '0x%0*X' % (precision, t)

   130     try:

   131         return '(' + ', '.join(['0x%0*X' % (precision, item)

   132                                 for item in t]) + ')'

   133     except TypeError, why:

   134         print '* failed to convert %r: %s' % (t, why)

   135         raise

   136 

   137 def python_mapdef_code(varname, map, comments=1, precisions=(2, 4)):

   138 

   139     l = []

   140     append = l.append

   141     if map.has_key("IDENTITY"):

   142         append("%s = codecs.make_identity_dict(range(%d))" %

   143                (varname, map["IDENTITY"]))

   144         append("%s.update({" % varname)

   145         splits = 1

   146         del map["IDENTITY"]

   147         identity = 1

   148     else:

   149         append("%s = {" % varname)

   150         splits = 0

   151         identity = 0

   152 

   153     mappings = map.items()

   154     mappings.sort()

   155     i = 0

   156     key_precision, value_precision = precisions

   157     for mapkey, mapvalue in mappings:

   158         mapcomment = ''

   159         if isinstance(mapkey, tuple):

   160             (mapkey, mapcomment) = mapkey

   161         if isinstance(mapvalue, tuple):

   162             (mapvalue, mapcomment) = mapvalue

   163         if mapkey is None:

   164             continue

   165         if (identity and

   166             mapkey == mapvalue and

   167             mapkey < 256):

   168             # No need to include identity mappings, since these

   169             # are already set for the first 256 code points.

   170             continue

   171         key = hexrepr(mapkey, key_precision)

   172         value = hexrepr(mapvalue, value_precision)

   173         if mapcomment and comments:

   174             append('    %s: %s,\t#  %s' % (key, value, mapcomment))

   175         else:

   176             append('    %s: %s,' % (key, value))

   177         i += 1

   178         if i == 4096:

   179             # Split the definition into parts to that the Python

   180             # parser doesn't dump core

   181             if splits == 0:

   182                 append('}')

   183             else:

   184                 append('})')

   185             append('%s.update({' % varname)

   186             i = 0

   187             splits = splits + 1

   188     if splits == 0:

   189         append('}')

   190     else:

   191         append('})')

   192 

   193     return l

   194 

   195 def python_tabledef_code(varname, map, comments=1, key_precision=2):

   196 

   197     l = []

   198     append = l.append

   199     append('%s = (' % varname)

   200 

   201     # Analyze map and create table dict

   202     mappings = map.items()

   203     mappings.sort()

   204     table = {}

   205     maxkey = 0

   206     if map.has_key('IDENTITY'):

   207         for key in range(256):

   208             table[key] = (key, '')

   209         maxkey = 255

   210         del map['IDENTITY']

   211     for mapkey, mapvalue in mappings:

   212         mapcomment = ''

   213         if isinstance(mapkey, tuple):

   214             (mapkey, mapcomment) = mapkey

   215         if isinstance(mapvalue, tuple):

   216             (mapvalue, mapcomment) = mapvalue

   217         if mapkey is None:

   218             continue

   219         table[mapkey] = (mapvalue, mapcomment)

   220         if mapkey > maxkey:

   221             maxkey = mapkey

   222     if maxkey > MAX_TABLE_SIZE:

   223         # Table too large

   224         return None

   225 

   226     # Create table code

   227     for key in range(maxkey + 1):

   228         if key not in table:

   229             mapvalue = None

   230             mapcomment = 'UNDEFINED'

   231         else:

   232             mapvalue, mapcomment = table[key]

   233         if mapvalue is None:

   234             mapchar = UNI_UNDEFINED

   235         else:

   236             if isinstance(mapvalue, tuple):

   237                 # 1-n mappings not supported

   238                 return None

   239             else:

   240                 mapchar = unichr(mapvalue)

   241         if mapcomment and comments:

   242             append('    %r\t#  %s -> %s' % (mapchar,

   243                                             hexrepr(key, key_precision),

   244                                             mapcomment))

   245         else:

   246             append('    %r' % mapchar)

   247 

   248     append(')')

   249     return l

   250 

   251 def codegen(name, map, encodingname, comments=1):

   252 

   253     """ Returns Python source for the given map.

   254 

   255         Comments are included in the source, if comments is true (default).

   256 

   257     """

   258     # Generate code

   259     decoding_map_code = python_mapdef_code(

   260         'decoding_map',

   261         map,

   262         comments=comments)

   263     decoding_table_code = python_tabledef_code(

   264         'decoding_table',

   265         map,

   266         comments=comments)

   267     encoding_map_code = python_mapdef_code(

   268         'encoding_map',

   269         codecs.make_encoding_map(map),

   270         comments=comments,

   271         precisions=(4, 2))

   272 

   273     if decoding_table_code:

   274         suffix = 'table'

   275     else:

   276         suffix = 'map'

   277 

   278     l = [

   279         '''\

   280 """ Python Character Mapping Codec %s generated from '%s' with gencodec.py.

   281 

   282 """#"

   283 

   284 import codecs

   285 

   286 ### Codec APIs

   287 

   288 class Codec(codecs.Codec):

   289 

   290     def encode(self,input,errors='strict'):

   291         return codecs.charmap_encode(input,errors,encoding_%s)

   292 

   293     def decode(self,input,errors='strict'):

   294         return codecs.charmap_decode(input,errors,decoding_%s)

   295 ''' % (encodingname, name, suffix, suffix)]

   296     l.append('''\

   297 class IncrementalEncoder(codecs.IncrementalEncoder):

   298     def encode(self, input, final=False):

   299         return codecs.charmap_encode(input,self.errors,encoding_%s)[0]

   300 

   301 class IncrementalDecoder(codecs.IncrementalDecoder):

   302     def decode(self, input, final=False):

   303         return codecs.charmap_decode(input,self.errors,decoding_%s)[0]''' %

   304         (suffix, suffix))

   305 

   306     l.append('''

   307 class StreamWriter(Codec,codecs.StreamWriter):

   308     pass

   309 

   310 class StreamReader(Codec,codecs.StreamReader):

   311     pass

   312 

   313 ### encodings module API

   314 

   315 def getregentry():

   316     return codecs.CodecInfo(

   317         name=%r,

   318         encode=Codec().encode,

   319         decode=Codec().decode,

   320         incrementalencoder=IncrementalEncoder,

   321         incrementaldecoder=IncrementalDecoder,

   322         streamreader=StreamReader,

   323         streamwriter=StreamWriter,

   324     )

   325 ''' % encodingname.replace('_', '-'))

   326 

   327     # Add decoding table or map (with preference to the table)

   328     if not decoding_table_code:

   329         l.append('''

   330 ### Decoding Map

   331 ''')

   332         l.extend(decoding_map_code)

   333     else:

   334         l.append('''

   335 ### Decoding Table

   336 ''')

   337         l.extend(decoding_table_code)

   338 

   339     # Add encoding map

   340     if decoding_table_code:

   341         l.append('''

   342 ### Encoding table

   343 encoding_table=codecs.charmap_build(decoding_table)

   344 ''')

   345     else:

   346         l.append('''

   347 ### Encoding Map

   348 ''')

   349         l.extend(encoding_map_code)

   350 

   351     # Final new-line

   352     l.append('')

   353 

   354     return '\n'.join(l).expandtabs()

   355 

   356 def pymap(name,map,pyfile,encodingname,comments=1):

   357 

   358     code = codegen(name,map,encodingname,comments)

   359     f = open(pyfile,'w')

   360     f.write(code)

   361     f.close()

   362 

   363 def marshalmap(name,map,marshalfile):

   364 

   365     d = {}

   366     for e,(u,c) in map.items():

   367         d[e] = (u,c)

   368     f = open(marshalfile,'wb')

   369     marshal.dump(d,f)

   370     f.close()

   371 

   372 def convertdir(dir, dirprefix='', nameprefix='', comments=1):

   373 

   374     mapnames = os.listdir(dir)

   375     for mapname in mapnames:

   376         mappathname = os.path.join(dir, mapname)

   377         if not os.path.isfile(mappathname):

   378             continue

   379         name = os.path.split(mapname)[1]

   380         name = name.replace('-','_')

   381         name = name.split('.')[0]

   382         name = name.lower()

   383         name = nameprefix + name

   384         codefile = name + '.py'

   385         marshalfile = name + '.mapping'

   386         print 'converting %s to %s and %s' % (mapname,

   387                                               dirprefix + codefile,

   388                                               dirprefix + marshalfile)

   389         try:

   390             map = readmap(os.path.join(dir,mapname))

   391             if not map:

   392                 print '* map is empty; skipping'

   393             else:

   394                 pymap(mappathname, map, dirprefix + codefile,name,comments)

   395                 marshalmap(mappathname, map, dirprefix + marshalfile)

   396         except ValueError, why:

   397             print '* conversion failed: %s' % why

   398             raise

   399 

   400 def rewritepythondir(dir, dirprefix='', comments=1):

   401 

   402     mapnames = os.listdir(dir)

   403     for mapname in mapnames:

   404         if not mapname.endswith('.mapping'):

   405             continue

   406         name = mapname[:-len('.mapping')]

   407         codefile = name + '.py'

   408         print 'converting %s to %s' % (mapname,

   409                                        dirprefix + codefile)

   410         try:

   411             map = marshal.load(open(os.path.join(dir,mapname),

   412                                'rb'))

   413             if not map:

   414                 print '* map is empty; skipping'

   415             else:

   416                 pymap(mapname, map, dirprefix + codefile,name,comments)

   417         except ValueError, why:

   418             print '* conversion failed: %s' % why

   419 

   420 if __name__ == '__main__':

   421 

   422     import sys

   423     if 1:

   424         apply(convertdir,tuple(sys.argv[1:]))

   425     else:

   426         apply(rewritepythondir,tuple(sys.argv[1:]))