1 // Copyright (c) 2005, Google Inc.

     2 // All rights reserved.

     3 //

     4 // Redistribution and use in source and binary forms, with or without

     5 // modification, are permitted provided that the following conditions are

     6 // met:

     7 //

     8 //     * Redistributions of source code must retain the above copyright

     9 // notice, this list of conditions and the following disclaimer.

    10 //     * Redistributions in binary form must reproduce the above

    11 // copyright notice, this list of conditions and the following disclaimer

    12 // in the documentation and/or other materials provided with the

    13 // distribution.

    14 //     * Neither the name of Google Inc. nor the names of its

    15 // contributors may be used to endorse or promote products derived from

    16 // this software without specific prior written permission.

    17 //

    18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

    19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

    20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

    21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

    22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

    23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

    24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

    25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

    26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

    27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

    28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

    29 //

    30 // Author: Sanjay Ghemawat

    31 //

    32 // Regular-expression based scanner for parsing an input stream.

    33 //

    34 // Example 1: parse a sequence of "var = number" entries from input:

    35 //

    36 //      Scanner scanner(input);

    37 //      string var;

    38 //      int number;

    39 //      scanner.SetSkipExpression("\\s+"); // Skip any white space we encounter

    40 //      while (scanner.Consume("(\\w+) = (\\d+)", &var, &number)) {

    41 //        ...;

    42 //      }

    43 

    44 #ifndef _PCRE_SCANNER_H

    45 #define _PCRE_SCANNER_H

    46 

    47 #include <assert.h>

    48 #include <string>

    49 #include <vector>

    50 

    51 #include <pcrecpp.h>

    52 #include <pcre_stringpiece.h>

    53 

    54 namespace pcrecpp {

    55 

    56 class PCRECPP_EXP_DEFN Scanner {

    57  public:

    58   Scanner();

    59   explicit Scanner(const std::string& input);

    60   ~Scanner();

    61 

    62   // Return current line number.  The returned line-number is

    63   // one-based.  I.e. it returns 1 + the number of consumed newlines.

    64   //

    65   // Note: this method may be slow.  It may take time proportional to

    66   // the size of the input.

    67   int LineNumber() const;

    68 

    69   // Return the byte-offset that the scanner is looking in the

    70   // input data;

    71   int Offset() const;

    72 

    73   // Return true iff the start of the remaining input matches "re"

    74   bool LookingAt(const RE& re) const;

    75 

    76   // Return true iff all of the following are true

    77   //    a. the start of the remaining input matches "re",

    78   //    b. if any arguments are supplied, matched sub-patterns can be

    79   //       parsed and stored into the arguments.

    80   // If it returns true, it skips over the matched input and any

    81   // following input that matches the "skip" regular expression.

    82   bool Consume(const RE& re,

    83                const Arg& arg0 = RE::no_arg,

    84                const Arg& arg1 = RE::no_arg,

    85                const Arg& arg2 = RE::no_arg

    86                // TODO: Allow more arguments?

    87                );

    88 

    89   // Set the "skip" regular expression.  If after consuming some data,

    90   // a prefix of the input matches this RE, it is automatically

    91   // skipped.  For example, a programming language scanner would use

    92   // a skip RE that matches white space and comments.

    93   //

    94   //    scanner.SetSkipExpression("\\s+|//.*|/[*](.|\n)*?[*]/");

    95   //

    96   // Skipping repeats as long as it succeeds.  We used to let people do

    97   // this by writing "(...)*" in the regular expression, but that added

    98   // up to lots of recursive calls within the pcre library, so now we

    99   // control repetition explicitly via the function call API.

   100   //

   101   // You can pass NULL for "re" if you do not want any data to be skipped.

   102   void Skip(const char* re);   // DEPRECATED; does *not* repeat

   103   void SetSkipExpression(const char* re);

   104 

   105   // Temporarily pause "skip"ing. This

   106   //   Skip("Foo"); code ; DisableSkip(); code; EnableSkip()

   107   // is similar to

   108   //   Skip("Foo"); code ; Skip(NULL); code ; Skip("Foo");

   109   // but avoids creating/deleting new RE objects.

   110   void DisableSkip();

   111 

   112   // Reenable previously paused skipping.  Any prefix of the input

   113   // that matches the skip pattern is immediately dropped.

   114   void EnableSkip();

   115 

   116   /***** Special wrappers around SetSkip() for some common idioms *****/

   117 

   118   // Arranges to skip whitespace, C comments, C++ comments.

   119   // The overall RE is a disjunction of the following REs:

   120   //    \\s                     whitespace

   121   //    //.*\n                  C++ comment

   122   //    /[*](.|\n)*?[*]/        C comment (x*? means minimal repetitions of x)

   123   // We get repetition via the semantics of SetSkipExpression, not by using *

   124   void SkipCXXComments() {

   125     SetSkipExpression("\\s|//.*\n|/[*](?:\n|.)*?[*]/");

   126   }

   127 

   128   void set_save_comments(bool comments) {

   129     save_comments_ = comments;

   130   }

   131 

   132   bool save_comments() {

   133     return save_comments_;

   134   }

   135 

   136   // Append to vector ranges the comments found in the

   137   // byte range [start,end] (inclusive) of the input data.

   138   // Only comments that were extracted entirely within that

   139   // range are returned: no range splitting of atomically-extracted

   140   // comments is performed.

   141   void GetComments(int start, int end, std::vector<StringPiece> *ranges);

   142 

   143   // Append to vector ranges the comments added

   144   // since the last time this was called. This

   145   // functionality is provided for efficiency when

   146   // interleaving scanning with parsing.

   147   void GetNextComments(std::vector<StringPiece> *ranges);

   148 

   149  private:

   150   std::string   data_;          // All the input data

   151   StringPiece   input_;         // Unprocessed input

   152   RE*           skip_;          // If non-NULL, RE for skipping input

   153   bool          should_skip_;   // If true, use skip_

   154   bool          skip_repeat_;   // If true, repeat skip_ as long as it works

   155   bool          save_comments_; // If true, aggregate the skip expression

   156 

   157   // the skipped comments

   158   // TODO: later consider requiring that the StringPieces be added

   159   // in order by their start position

   160   std::vector<StringPiece> *comments_;

   161 

   162   // the offset into comments_ that has been returned by GetNextComments

   163   int           comments_offset_;

   164 

   165   // helper function to consume *skip_ and honour

   166   // save_comments_

   167   void ConsumeSkip();

   168 };

   169 

   170 }   // namespace pcrecpp

   171 

   172 #endif /* _PCRE_SCANNER_H */