1 // Boost tokenizer.hpp  -----------------------------------------------------//

     2 

     3 // © Copyright Jeremy Siek and John R. Bandela 2001. 

     4 

     5 // Distributed under the Boost Software License, Version 1.0. (See

     6 // accompanying file LICENSE_1_0.txt or copy at

     7 // http://www.boost.org/LICENSE_1_0.txt)

     8 

     9 // See http://www.boost.org/libs/tokenizer for documenation

    10 

    11 // Revision History:

    12 // 03 Jul 2003   John Bandela

    13 //      Converted to new iterator adapter

    14 // 02 Feb 2002   Jeremy Siek

    15 //      Removed tabs and a little cleanup.

    16 

    17 #ifndef BOOST_TOKENIZER_JRB070303_HPP_

    18 #define BOOST_TOKENIZER_JRB070303_HPP_

    19 

    20 #include <boost/token_iterator.hpp>

    21 

    22 namespace boost {

    23 

    24   

    25   //===========================================================================

    26   // A container-view of a tokenized "sequence"

    27   template <

    28     typename TokenizerFunc = char_delimiters_separator<char>, 

    29     typename Iterator = std::string::const_iterator,

    30     typename Type = std::string

    31   >

    32   class tokenizer {

    33   private:

    34     typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;

    35         

    36     // It seems that MSVC does not like the unqualified use of iterator,

    37     // Thus we use iter internally when it is used unqualified and

    38     // the users of this class will always qualify iterator.     

    39     typedef typename TGen::type iter;

    40     

    41   public:

    42     

    43     typedef iter iterator;

    44     typedef iter const_iterator;

    45     typedef Type value_type;

    46     typedef value_type& reference;

    47     typedef const value_type& const_reference;

    48     typedef value_type* pointer;

    49     typedef const pointer const_pointer;

    50     typedef void size_type;

    51     typedef void difference_type;

    52 

    53     tokenizer(Iterator first, Iterator last,

    54               const TokenizerFunc& f = TokenizerFunc()) 

    55       : first_(first), last_(last), f_(f) { }

    56         

    57     template <typename Container>

    58     tokenizer(const Container& c)

    59       : first_(c.begin()), last_(c.end()), f_() { }

    60     

    61     template <typename Container>

    62     tokenizer(const Container& c,const TokenizerFunc& f)

    63       : first_(c.begin()), last_(c.end()), f_(f) { }

    64     

    65     void assign(Iterator first, Iterator last){

    66       first_ = first;

    67       last_ = last;

    68     }

    69     

    70     void assign(Iterator first, Iterator last, const TokenizerFunc& f){

    71       assign(first,last);

    72       f_ = f;

    73     }

    74     

    75     template <typename Container>

    76     void assign(const Container& c){

    77       assign(c.begin(),c.end());

    78     }

    79     

    80     

    81     template <typename Container>

    82     void assign(const Container& c, const TokenizerFunc& f){

    83       assign(c.begin(),c.end(),f);

    84     }

    85     

    86     iter begin() const { return iter(f_,first_,last_); }

    87     iter end() const { return iter(f_,last_,last_); }

    88         

    89   private:

    90     Iterator first_;

    91     Iterator last_;

    92     TokenizerFunc f_;

    93   };

    94 

    95 

    96 } // namespace boost

    97 

    98 #endif