1 # -*- coding: iso-8859-1 -*-

     2 """A lexical analyzer class for simple shell-like syntaxes."""

     3 

     4 # Module and documentation by Eric S. Raymond, 21 Dec 1998

     5 # Input stacking and error message cleanup added by ESR, March 2000

     6 # push_source() and pop_source() made explicit by ESR, January 2001.

     7 # Posix compliance, split(), string arguments, and

     8 # iterator interface by Gustavo Niemeyer, April 2003.

     9 

    10 import os.path

    11 import sys

    12 from collections import deque

    13 

    14 try:

    15     from cStringIO import StringIO

    16 except ImportError:

    17     from StringIO import StringIO

    18 

    19 __all__ = ["shlex", "split"]

    20 

    21 class shlex:

    22     "A lexical analyzer class for simple shell-like syntaxes."

    23     def __init__(self, instream=None, infile=None, posix=False):

    24         if isinstance(instream, basestring):

    25             instream = StringIO(instream)

    26         if instream is not None:

    27             self.instream = instream

    28             self.infile = infile

    29         else:

    30             self.instream = sys.stdin

    31             self.infile = None

    32         self.posix = posix

    33         if posix:

    34             self.eof = None

    35         else:

    36             self.eof = ''

    37         self.commenters = '#'

    38         self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'

    39                           'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')

    40         if self.posix:

    41             self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'

    42                                'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')

    43         self.whitespace = ' \t\r\n'

    44         self.whitespace_split = False

    45         self.quotes = '\'"'

    46         self.escape = '\\'

    47         self.escapedquotes = '"'

    48         self.state = ' '

    49         self.pushback = deque()

    50         self.lineno = 1

    51         self.debug = 0

    52         self.token = ''

    53         self.filestack = deque()

    54         self.source = None

    55         if self.debug:

    56             print 'shlex: reading from %s, line %d' \

    57                   % (self.instream, self.lineno)

    58 

    59     def push_token(self, tok):

    60         "Push a token onto the stack popped by the get_token method"

    61         if self.debug >= 1:

    62             print "shlex: pushing token " + repr(tok)

    63         self.pushback.appendleft(tok)

    64 

    65     def push_source(self, newstream, newfile=None):

    66         "Push an input source onto the lexer's input source stack."

    67         if isinstance(newstream, basestring):

    68             newstream = StringIO(newstream)

    69         self.filestack.appendleft((self.infile, self.instream, self.lineno))

    70         self.infile = newfile

    71         self.instream = newstream

    72         self.lineno = 1

    73         if self.debug:

    74             if newfile is not None:

    75                 print 'shlex: pushing to file %s' % (self.infile,)

    76             else:

    77                 print 'shlex: pushing to stream %s' % (self.instream,)

    78 

    79     def pop_source(self):

    80         "Pop the input source stack."

    81         self.instream.close()

    82         (self.infile, self.instream, self.lineno) = self.filestack.popleft()

    83         if self.debug:

    84             print 'shlex: popping to %s, line %d' \

    85                   % (self.instream, self.lineno)

    86         self.state = ' '

    87 

    88     def get_token(self):

    89         "Get a token from the input stream (or from stack if it's nonempty)"

    90         if self.pushback:

    91             tok = self.pushback.popleft()

    92             if self.debug >= 1:

    93                 print "shlex: popping token " + repr(tok)

    94             return tok

    95         # No pushback.  Get a token.

    96         raw = self.read_token()

    97         # Handle inclusions

    98         if self.source is not None:

    99             while raw == self.source:

   100                 spec = self.sourcehook(self.read_token())

   101                 if spec:

   102                     (newfile, newstream) = spec

   103                     self.push_source(newstream, newfile)

   104                 raw = self.get_token()

   105         # Maybe we got EOF instead?

   106         while raw == self.eof:

   107             if not self.filestack:

   108                 return self.eof

   109             else:

   110                 self.pop_source()

   111                 raw = self.get_token()

   112         # Neither inclusion nor EOF

   113         if self.debug >= 1:

   114             if raw != self.eof:

   115                 print "shlex: token=" + repr(raw)

   116             else:

   117                 print "shlex: token=EOF"

   118         return raw

   119 

   120     def read_token(self):

   121         quoted = False

   122         escapedstate = ' '

   123         while True:

   124             nextchar = self.instream.read(1)

   125             if nextchar == '\n':

   126                 self.lineno = self.lineno + 1

   127             if self.debug >= 3:

   128                 print "shlex: in state", repr(self.state), \

   129                       "I see character:", repr(nextchar)

   130             if self.state is None:

   131                 self.token = ''        # past end of file

   132                 break

   133             elif self.state == ' ':

   134                 if not nextchar:

   135                     self.state = None  # end of file

   136                     break

   137                 elif nextchar in self.whitespace:

   138                     if self.debug >= 2:

   139                         print "shlex: I see whitespace in whitespace state"

   140                     if self.token or (self.posix and quoted):

   141                         break   # emit current token

   142                     else:

   143                         continue

   144                 elif nextchar in self.commenters:

   145                     self.instream.readline()

   146                     self.lineno = self.lineno + 1

   147                 elif self.posix and nextchar in self.escape:

   148                     escapedstate = 'a'

   149                     self.state = nextchar

   150                 elif nextchar in self.wordchars:

   151                     self.token = nextchar

   152                     self.state = 'a'

   153                 elif nextchar in self.quotes:

   154                     if not self.posix:

   155                         self.token = nextchar

   156                     self.state = nextchar

   157                 elif self.whitespace_split:

   158                     self.token = nextchar

   159                     self.state = 'a'

   160                 else:

   161                     self.token = nextchar

   162                     if self.token or (self.posix and quoted):

   163                         break   # emit current token

   164                     else:

   165                         continue

   166             elif self.state in self.quotes:

   167                 quoted = True

   168                 if not nextchar:      # end of file

   169                     if self.debug >= 2:

   170                         print "shlex: I see EOF in quotes state"

   171                     # XXX what error should be raised here?

   172                     raise ValueError, "No closing quotation"

   173                 if nextchar == self.state:

   174                     if not self.posix:

   175                         self.token = self.token + nextchar

   176                         self.state = ' '

   177                         break

   178                     else:

   179                         self.state = 'a'

   180                 elif self.posix and nextchar in self.escape and \

   181                      self.state in self.escapedquotes:

   182                     escapedstate = self.state

   183                     self.state = nextchar

   184                 else:

   185                     self.token = self.token + nextchar

   186             elif self.state in self.escape:

   187                 if not nextchar:      # end of file

   188                     if self.debug >= 2:

   189                         print "shlex: I see EOF in escape state"

   190                     # XXX what error should be raised here?

   191                     raise ValueError, "No escaped character"

   192                 # In posix shells, only the quote itself or the escape

   193                 # character may be escaped within quotes.

   194                 if escapedstate in self.quotes and \

   195                    nextchar != self.state and nextchar != escapedstate:

   196                     self.token = self.token + self.state

   197                 self.token = self.token + nextchar

   198                 self.state = escapedstate

   199             elif self.state == 'a':

   200                 if not nextchar:

   201                     self.state = None   # end of file

   202                     break

   203                 elif nextchar in self.whitespace:

   204                     if self.debug >= 2:

   205                         print "shlex: I see whitespace in word state"

   206                     self.state = ' '

   207                     if self.token or (self.posix and quoted):

   208                         break   # emit current token

   209                     else:

   210                         continue

   211                 elif nextchar in self.commenters:

   212                     self.instream.readline()

   213                     self.lineno = self.lineno + 1

   214                     if self.posix:

   215                         self.state = ' '

   216                         if self.token or (self.posix and quoted):

   217                             break   # emit current token

   218                         else:

   219                             continue

   220                 elif self.posix and nextchar in self.quotes:

   221                     self.state = nextchar

   222                 elif self.posix and nextchar in self.escape:

   223                     escapedstate = 'a'

   224                     self.state = nextchar

   225                 elif nextchar in self.wordchars or nextchar in self.quotes \

   226                     or self.whitespace_split:

   227                     self.token = self.token + nextchar

   228                 else:

   229                     self.pushback.appendleft(nextchar)

   230                     if self.debug >= 2:

   231                         print "shlex: I see punctuation in word state"

   232                     self.state = ' '

   233                     if self.token:

   234                         break   # emit current token

   235                     else:

   236                         continue

   237         result = self.token

   238         self.token = ''

   239         if self.posix and not quoted and result == '':

   240             result = None

   241         if self.debug > 1:

   242             if result:

   243                 print "shlex: raw token=" + repr(result)

   244             else:

   245                 print "shlex: raw token=EOF"

   246         return result

   247 

   248     def sourcehook(self, newfile):

   249         "Hook called on a filename to be sourced."

   250         if newfile[0] == '"':

   251             newfile = newfile[1:-1]

   252         # This implements cpp-like semantics for relative-path inclusion.

   253         if isinstance(self.infile, basestring) and not os.path.isabs(newfile):

   254             newfile = os.path.join(os.path.dirname(self.infile), newfile)

   255         return (newfile, open(newfile, "r"))

   256 

   257     def error_leader(self, infile=None, lineno=None):

   258         "Emit a C-compiler-like, Emacs-friendly error-message leader."

   259         if infile is None:

   260             infile = self.infile

   261         if lineno is None:

   262             lineno = self.lineno

   263         return "\"%s\", line %d: " % (infile, lineno)

   264 

   265     def __iter__(self):

   266         return self

   267 

   268     def next(self):

   269         token = self.get_token()

   270         if token == self.eof:

   271             raise StopIteration

   272         return token

   273 

   274 def split(s, comments=False, posix=True):

   275     lex = shlex(s, posix=posix)

   276     lex.whitespace_split = True

   277     if not comments:

   278         lex.commenters = ''

   279     return list(lex)

   280 

   281 if __name__ == '__main__':

   282     if len(sys.argv) == 1:

   283         lexer = shlex()

   284     else:

   285         file = sys.argv[1]

   286         lexer = shlex(open(file), file)

   287     while 1:

   288         tt = lexer.get_token()

   289         if tt:

   290             print "Token: " + repr(tt)

   291         else:

   292             break