1 """Conversion pipeline templates.

     2 

     3 The problem:

     4 ------------

     5 

     6 Suppose you have some data that you want to convert to another format,

     7 such as from GIF image format to PPM image format.  Maybe the

     8 conversion involves several steps (e.g. piping it through compress or

     9 uuencode).  Some of the conversion steps may require that their input

    10 is a disk file, others may be able to read standard input; similar for

    11 their output.  The input to the entire conversion may also be read

    12 from a disk file or from an open file, and similar for its output.

    13 

    14 The module lets you construct a pipeline template by sticking one or

    15 more conversion steps together.  It will take care of creating and

    16 removing temporary files if they are necessary to hold intermediate

    17 data.  You can then use the template to do conversions from many

    18 different sources to many different destinations.  The temporary

    19 file names used are different each time the template is used.

    20 

    21 The templates are objects so you can create templates for many

    22 different conversion steps and store them in a dictionary, for

    23 instance.

    24 

    25 

    26 Directions:

    27 -----------

    28 

    29 To create a template:

    30     t = Template()

    31 

    32 To add a conversion step to a template:

    33    t.append(command, kind)

    34 where kind is a string of two characters: the first is '-' if the

    35 command reads its standard input or 'f' if it requires a file; the

    36 second likewise for the output. The command must be valid /bin/sh

    37 syntax.  If input or output files are required, they are passed as

    38 $IN and $OUT; otherwise, it must be  possible to use the command in

    39 a pipeline.

    40 

    41 To add a conversion step at the beginning:

    42    t.prepend(command, kind)

    43 

    44 To convert a file to another file using a template:

    45   sts = t.copy(infile, outfile)

    46 If infile or outfile are the empty string, standard input is read or

    47 standard output is written, respectively.  The return value is the

    48 exit status of the conversion pipeline.

    49 

    50 To open a file for reading or writing through a conversion pipeline:

    51    fp = t.open(file, mode)

    52 where mode is 'r' to read the file, or 'w' to write it -- just like

    53 for the built-in function open() or for os.popen().

    54 

    55 To create a new template object initialized to a given one:

    56    t2 = t.clone()

    57 

    58 For an example, see the function test() at the end of the file.

    59 """                                     # '

    60 

    61 

    62 import re

    63 

    64 import os

    65 import tempfile

    66 import string

    67 

    68 __all__ = ["Template"]

    69 

    70 # Conversion step kinds

    71 

    72 FILEIN_FILEOUT = 'ff'                   # Must read & write real files

    73 STDIN_FILEOUT  = '-f'                   # Must write a real file

    74 FILEIN_STDOUT  = 'f-'                   # Must read a real file

    75 STDIN_STDOUT   = '--'                   # Normal pipeline element

    76 SOURCE         = '.-'                   # Must be first, writes stdout

    77 SINK           = '-.'                   # Must be last, reads stdin

    78 

    79 stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \

    80              SOURCE, SINK]

    81 

    82 

    83 class Template:

    84     """Class representing a pipeline template."""

    85 

    86     def __init__(self):

    87         """Template() returns a fresh pipeline template."""

    88         self.debugging = 0

    89         self.reset()

    90 

    91     def __repr__(self):

    92         """t.__repr__() implements repr(t)."""

    93         return '<Template instance, steps=%r>' % (self.steps,)

    94 

    95     def reset(self):

    96         """t.reset() restores a pipeline template to its initial state."""

    97         self.steps = []

    98 

    99     def clone(self):

   100         """t.clone() returns a new pipeline template with identical

   101         initial state as the current one."""

   102         t = Template()

   103         t.steps = self.steps[:]

   104         t.debugging = self.debugging

   105         return t

   106 

   107     def debug(self, flag):

   108         """t.debug(flag) turns debugging on or off."""

   109         self.debugging = flag

   110 

   111     def append(self, cmd, kind):

   112         """t.append(cmd, kind) adds a new step at the end."""

   113         if type(cmd) is not type(''):

   114             raise TypeError, \

   115                   'Template.append: cmd must be a string'

   116         if kind not in stepkinds:

   117             raise ValueError, \

   118                   'Template.append: bad kind %r' % (kind,)

   119         if kind == SOURCE:

   120             raise ValueError, \

   121                   'Template.append: SOURCE can only be prepended'

   122         if self.steps and self.steps[-1][1] == SINK:

   123             raise ValueError, \

   124                   'Template.append: already ends with SINK'

   125         if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):

   126             raise ValueError, \

   127                   'Template.append: missing $IN in cmd'

   128         if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):

   129             raise ValueError, \

   130                   'Template.append: missing $OUT in cmd'

   131         self.steps.append((cmd, kind))

   132 

   133     def prepend(self, cmd, kind):

   134         """t.prepend(cmd, kind) adds a new step at the front."""

   135         if type(cmd) is not type(''):

   136             raise TypeError, \

   137                   'Template.prepend: cmd must be a string'

   138         if kind not in stepkinds:

   139             raise ValueError, \

   140                   'Template.prepend: bad kind %r' % (kind,)

   141         if kind == SINK:

   142             raise ValueError, \

   143                   'Template.prepend: SINK can only be appended'

   144         if self.steps and self.steps[0][1] == SOURCE:

   145             raise ValueError, \

   146                   'Template.prepend: already begins with SOURCE'

   147         if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):

   148             raise ValueError, \

   149                   'Template.prepend: missing $IN in cmd'

   150         if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):

   151             raise ValueError, \

   152                   'Template.prepend: missing $OUT in cmd'

   153         self.steps.insert(0, (cmd, kind))

   154 

   155     def open(self, file, rw):

   156         """t.open(file, rw) returns a pipe or file object open for

   157         reading or writing; the file is the other end of the pipeline."""

   158         if rw == 'r':

   159             return self.open_r(file)

   160         if rw == 'w':

   161             return self.open_w(file)

   162         raise ValueError, \

   163               'Template.open: rw must be \'r\' or \'w\', not %r' % (rw,)

   164 

   165     def open_r(self, file):

   166         """t.open_r(file) and t.open_w(file) implement

   167         t.open(file, 'r') and t.open(file, 'w') respectively."""

   168         if not self.steps:

   169             return open(file, 'r')

   170         if self.steps[-1][1] == SINK:

   171             raise ValueError, \

   172                   'Template.open_r: pipeline ends width SINK'

   173         cmd = self.makepipeline(file, '')

   174         return os.popen(cmd, 'r')

   175 

   176     def open_w(self, file):

   177         if not self.steps:

   178             return open(file, 'w')

   179         if self.steps[0][1] == SOURCE:

   180             raise ValueError, \

   181                   'Template.open_w: pipeline begins with SOURCE'

   182         cmd = self.makepipeline('', file)

   183         return os.popen(cmd, 'w')

   184 

   185     def copy(self, infile, outfile):

   186         return os.system(self.makepipeline(infile, outfile))

   187 

   188     def makepipeline(self, infile, outfile):

   189         cmd = makepipeline(infile, self.steps, outfile)

   190         if self.debugging:

   191             print cmd

   192             cmd = 'set -x; ' + cmd

   193         return cmd

   194 

   195 

   196 def makepipeline(infile, steps, outfile):

   197     # Build a list with for each command:

   198     # [input filename or '', command string, kind, output filename or '']

   199 

   200     list = []

   201     for cmd, kind in steps:

   202         list.append(['', cmd, kind, ''])

   203     #

   204     # Make sure there is at least one step

   205     #

   206     if not list:

   207         list.append(['', 'cat', '--', ''])

   208     #

   209     # Take care of the input and output ends

   210     #

   211     [cmd, kind] = list[0][1:3]

   212     if kind[0] == 'f' and not infile:

   213         list.insert(0, ['', 'cat', '--', ''])

   214     list[0][0] = infile

   215     #

   216     [cmd, kind] = list[-1][1:3]

   217     if kind[1] == 'f' and not outfile:

   218         list.append(['', 'cat', '--', ''])

   219     list[-1][-1] = outfile

   220     #

   221     # Invent temporary files to connect stages that need files

   222     #

   223     garbage = []

   224     for i in range(1, len(list)):

   225         lkind = list[i-1][2]

   226         rkind = list[i][2]

   227         if lkind[1] == 'f' or rkind[0] == 'f':

   228             (fd, temp) = tempfile.mkstemp()

   229             os.close(fd)

   230             garbage.append(temp)

   231             list[i-1][-1] = list[i][0] = temp

   232     #

   233     for item in list:

   234         [inf, cmd, kind, outf] = item

   235         if kind[1] == 'f':

   236             cmd = 'OUT=' + quote(outf) + '; ' + cmd

   237         if kind[0] == 'f':

   238             cmd = 'IN=' + quote(inf) + '; ' + cmd

   239         if kind[0] == '-' and inf:

   240             cmd = cmd + ' <' + quote(inf)

   241         if kind[1] == '-' and outf:

   242             cmd = cmd + ' >' + quote(outf)

   243         item[1] = cmd

   244     #

   245     cmdlist = list[0][1]

   246     for item in list[1:]:

   247         [cmd, kind] = item[1:3]

   248         if item[0] == '':

   249             if 'f' in kind:

   250                 cmd = '{ ' + cmd + '; }'

   251             cmdlist = cmdlist + ' |\n' + cmd

   252         else:

   253             cmdlist = cmdlist + '\n' + cmd

   254     #

   255     if garbage:

   256         rmcmd = 'rm -f'

   257         for file in garbage:

   258             rmcmd = rmcmd + ' ' + quote(file)

   259         trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'

   260         cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd

   261     #

   262     return cmdlist

   263 

   264 

   265 # Reliably quote a string as a single argument for /bin/sh

   266 

   267 _safechars = string.ascii_letters + string.digits + '!@%_-+=:,./' # Safe unquoted

   268 _funnychars = '"`$\\'                           # Unsafe inside "double quotes"

   269 

   270 def quote(file):

   271     for c in file:

   272         if c not in _safechars:

   273             break

   274     else:

   275         return file

   276     if '\'' not in file:

   277         return '\'' + file + '\''

   278     res = ''

   279     for c in file:

   280         if c in _funnychars:

   281             c = '\\' + c

   282         res = res + c

   283     return '"' + res + '"'

   284 

   285 

   286 # Small test program and example

   287 

   288 def test():

   289     print 'Testing...'

   290     t = Template()

   291     t.append('togif $IN $OUT', 'ff')

   292     t.append('giftoppm', '--')

   293     t.append('ppmtogif >$OUT', '-f')

   294     t.append('fromgif $IN $OUT', 'ff')

   295     t.debug(1)

   296     FILE = '/usr/local/images/rgb/rogues/guido.rgb'

   297     t.copy(FILE, '@temp')

   298     print 'Done.'