// Boost token_functions.hpp  ------------------------------------------------//

// Copyright John R. Bandela 2001. 

// Distributed under the Boost Software License, Version 1.0. (See

// accompanying file LICENSE_1_0.txt or copy at

// http://www.boost.org/LICENSE_1_0.txt)

// See http://www.boost.org/libs/tokenizer/ for documentation.

// Revision History:

// 01 Oct 2004   Joaquín M López Muñoz

//      Workaround for a problem with string::assign in msvc-stlport

// 06 Apr 2004   John Bandela

//      Fixed a bug involving using char_delimiter with a true input iterator

// 28 Nov 2003   Robert Zeh and John Bandela

//      Converted into "fast" functions that avoid using += when

//      the supplied iterator isn't an input_iterator; based on

//      some work done at Archelon and a version that was checked into

//      the boost CVS for a short period of time.

// 20 Feb 2002   John Maddock

//      Removed using namespace std declarations and added

//      workaround for BOOST_NO_STDC_NAMESPACE (the library

//      can be safely mixed with regex).

// 06 Feb 2002   Jeremy Siek

//      Added char_separator.

// 02 Feb 2002   Jeremy Siek

//      Removed tabs and a little cleanup.

#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_

#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_

#include <vector>

#include <stdexcept>

#include <string>

#include <cctype>

#include <algorithm> // for find_if

#include <boost/config.hpp>

#include <boost/assert.hpp>

#include <boost/detail/workaround.hpp>

#include <boost/mpl/if.hpp>

//

// the following must not be macros if we are to prefix them

// with std:: (they shouldn't be macros anyway...)

//

#ifdef ispunct

#  undef ispunct

#endif

#ifdef isspace

#  undef isspace

#endif

//

// fix namespace problems:

//

#ifdef BOOST_NO_STDC_NAMESPACE

namespace std{

 using ::ispunct;

 using ::isspace;

}

#endif

namespace boost{

  //===========================================================================

  // The escaped_list_separator class. Which is a model of TokenizerFunction

  // An escaped list is a super-set of what is commonly known as a comma 

  // separated value (csv) list.It is separated into fields by a comma or 

  // other character. If the delimiting character is inside quotes, then it is

  // counted as a regular character.To allow for embedded quotes in a field,

  // there can be escape sequences using the \ much like C. 

  // The role of the comma, the quotation mark, and the escape 

  // character (backslash \), can be assigned to other characters.

  struct escaped_list_error : public std::runtime_error{

    escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }

  };

// The out of the box GCC 2.95 on cygwin does not have a char_traits class.

// MSVC does not like the following typename

#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300

  template <class Char, 

    class Traits = typename std::basic_string<Char>::traits_type >

#else

  template <class Char, 

    class Traits = std::basic_string<Char>::traits_type >

#endif

  class escaped_list_separator {

  private:

    typedef std::basic_string<Char,Traits> string_type;

    struct char_eq {

      Char e_;

      char_eq(Char e):e_(e) { }

      bool operator()(Char c) {

        return Traits::eq(e_,c);

      }

    };

    string_type  escape_;

    string_type  c_;

    string_type  quote_;      

    bool last_;

    bool is_escape(Char e) {

      char_eq f(e);

      return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();

    }

    bool is_c(Char e) {

      char_eq f(e);

      return std::find_if(c_.begin(),c_.end(),f)!=c_.end();

    }

    bool is_quote(Char e) {

      char_eq f(e);

      return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();

    }

    template <typename iterator, typename Token>

    void do_escape(iterator& next,iterator end,Token& tok) {

      if (++next == end)

        throw escaped_list_error(std::string("cannot end with escape"));

      if (Traits::eq(*next,'n')) {

        tok+='\n';

        return;

      }

      else if (is_quote(*next)) {

        tok+=*next;

        return;

      }

      else if (is_c(*next)) {

        tok+=*next;

        return;

      }

      else if (is_escape(*next)) {

        tok+=*next;

        return;

      }

      else

        throw escaped_list_error(std::string("unknown escape sequence"));

    }

    public:

    explicit escaped_list_separator(Char  e = '\\',

                                    Char c = ',',Char  q = '\"')

      : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }

    escaped_list_separator(string_type e, string_type c, string_type q)

      : escape_(e), c_(c), quote_(q), last_(false) { }

    void reset() {last_=false;}

    template <typename InputIterator, typename Token>

    bool operator()(InputIterator& next,InputIterator end,Token& tok) {

      bool bInQuote = false;

      tok = Token();

      if (next == end) {

        if (last_) {

          last_ = false;

          return true;

        }

        else

          return false;

      }

      last_ = false;

      for (;next != end;++next) {

        if (is_escape(*next)) {

          do_escape(next,end,tok);

        }

        else if (is_c(*next)) {

          if (!bInQuote) {

            // If we are not in quote, then we are done

            ++next;

            // The last character was a c, that means there is

            // 1 more blank field

            last_ = true; 

            return true;

          }

          else tok+=*next;

        }

        else if (is_quote(*next)) {

          bInQuote=!bInQuote;

        }

        else {

          tok += *next;

        }

      }

      return true;

    }

  };

  //===========================================================================

  // The classes here are used by offset_separator and char_separator to implement

  // faster assigning of tokens using assign instead of +=

  namespace tokenizer_detail {

  // The assign_or_plus_equal struct contains functions that implement

  // assign, +=, and clearing based on the iterator type.  The

  // generic case does nothing for plus_equal and clearing, while

  // passing through the call for assign.

  //

  // When an input iterator is being used, the situation is reversed.

  // The assign method does nothing, plus_equal invokes operator +=,

  // and the clearing method sets the supplied token to the default

  // token constructor's result.

  //

  template<class IteratorTag>

  struct assign_or_plus_equal {

    template<class Iterator, class Token>

    static void assign(Iterator b, Iterator e, Token &t) {

#if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\

    BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\

    defined(_STLP_DEBUG) &&\

    (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))

    // Problem with string::assign for msvc-stlport in debug mode: the

    // linker tries to import the templatized version of this memfun,

    // which is obviously not exported.

    // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.

      t = Token();

      while(b != e) t += *b++;

#else

      t.assign(b, e);

#endif

    }

    template<class Token, class Value> 

    static void plus_equal(Token &, const Value &) {

  }

    // If we are doing an assign, there is no need for the

    // the clear. 

    //

    template<class Token>

    static void clear(Token &) {

    }

  };

  template <>

  struct assign_or_plus_equal<std::input_iterator_tag> {

    template<class Iterator, class Token>

    static void assign(Iterator b, Iterator e, Token &t) {

    }

    template<class Token, class Value> 

    static void plus_equal(Token &t, const Value &v) {

      t += v;

    }

    template<class Token>

    static void clear(Token &t) {

      t = Token();

    }

  };

  template<class Iterator>

  struct pointer_iterator_category{

    typedef std::random_access_iterator_tag type;

  };

  template<class Iterator>

  struct class_iterator_category{

    typedef typename Iterator::iterator_category type;

  };

  // This portably gets the iterator_tag without partial template specialization

  template<class Iterator>

    struct get_iterator_category{

    typedef typename mpl::if_<is_pointer<Iterator>,

      pointer_iterator_category<Iterator>,

      class_iterator_category<Iterator>

    >::type cat;

    typedef typename cat::type iterator_category;

  };

}

  //===========================================================================

  // The offset_separator class, which is a model of TokenizerFunction.

  // Offset breaks a string into tokens based on a range of offsets

  class offset_separator {

  private:

    std::vector<int> offsets_;

    unsigned int current_offset_;

    bool wrap_offsets_;

    bool return_partial_last_;

  public:

    template <typename Iter>

    offset_separator(Iter begin, Iter end, bool wrap_offsets = true,

                     bool return_partial_last = true)

      : offsets_(begin,end), current_offset_(0),

        wrap_offsets_(wrap_offsets),

        return_partial_last_(return_partial_last) { }

    offset_separator()

      : offsets_(1,1), current_offset_(),

        wrap_offsets_(true), return_partial_last_(true) { }

    void reset() {

      current_offset_ = 0;

    }

    template <typename InputIterator, typename Token>

    bool operator()(InputIterator& next, InputIterator end, Token& tok)

    {

      typedef tokenizer_detail::assign_or_plus_equal<

#if     !defined(BOOST_MSVC) || BOOST_MSVC > 1300

        typename

#endif

        tokenizer_detail::get_iterator_category<

        InputIterator>::iterator_category> assigner;

      BOOST_ASSERT(!offsets_.empty());

      assigner::clear(tok);

      InputIterator start(next);

      if (next == end)

        return false;

      if (current_offset_ == offsets_.size())

        if (wrap_offsets_)

          current_offset_=0;

        else

          return false;

      int c = offsets_[current_offset_];

      int i = 0;

      for (; i < c; ++i) {

        if (next == end)break;

        assigner::plus_equal(tok,*next++);

      }

      assigner::assign(start,next,tok);

      if (!return_partial_last_)

        if (i < (c-1) )

          return false;

      ++current_offset_;

      return true;

    }

  };

  //===========================================================================

  // The char_separator class breaks a sequence of characters into

  // tokens based on the character delimiters (very much like bad old

  // strtok). A delimiter character can either be kept or dropped. A

  // kept delimiter shows up as an output token, whereas a dropped

  // delimiter does not.

  // This class replaces the char_delimiters_separator class. The

  // constructor for the char_delimiters_separator class was too

  // confusing and needed to be deprecated. However, because of the

  // default arguments to the constructor, adding the new constructor

  // would cause ambiguity, so instead I deprecated the whole class.

  // The implementation of the class was also simplified considerably.

  enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };

  // The out of the box GCC 2.95 on cygwin does not have a char_traits class.

#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300

  template <typename Char, 

    typename Traits = typename std::basic_string<Char>::traits_type >

#else

  template <typename Char, 

    typename Traits = std::basic_string<Char>::traits_type >

#endif

  class char_separator

  {

    typedef std::basic_string<Char,Traits> string_type;

  public:

    explicit 

    char_separator(const Char* dropped_delims,

                   const Char* kept_delims = 0,

                   empty_token_policy empty_tokens = drop_empty_tokens)

      : m_dropped_delims(dropped_delims),

        m_use_ispunct(false),

        m_use_isspace(false),

        m_empty_tokens(empty_tokens),

        m_output_done(false)

    {

      // Borland workaround

      if (kept_delims)

        m_kept_delims = kept_delims;

    }

                // use ispunct() for kept delimiters and isspace for dropped.

    explicit

    char_separator()

      : m_use_ispunct(true), 

        m_use_isspace(true), 

        m_empty_tokens(drop_empty_tokens) { }

    void reset() { }

    template <typename InputIterator, typename Token>

    bool operator()(InputIterator& next, InputIterator end, Token& tok)

    {

      typedef tokenizer_detail::assign_or_plus_equal<

#if     !defined(BOOST_MSVC) || BOOST_MSVC > 1300

        typename

#endif

        tokenizer_detail::get_iterator_category<

        InputIterator>::iterator_category> assigner;

      assigner::clear(tok);

      // skip past all dropped_delims

      if (m_empty_tokens == drop_empty_tokens)

        for (; next != end  && is_dropped(*next); ++next)

          { }

      InputIterator start(next);

      if (m_empty_tokens == drop_empty_tokens) {

        if (next == end)

          return false;

        // if we are on a kept_delims move past it and stop

        if (is_kept(*next)) {

          assigner::plus_equal(tok,*next);

          ++next;

        } else

          // append all the non delim characters

          for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)

            assigner::plus_equal(tok,*next);

      } 

      else { // m_empty_tokens == keep_empty_tokens

        // Handle empty token at the end

        if (next == end)

          if (m_output_done == false) {

            m_output_done = true;

            assigner::assign(start,next,tok);

            return true;

          } else

            return false;

        if (is_kept(*next)) {

          if (m_output_done == false)

            m_output_done = true;

          else {

            assigner::plus_equal(tok,*next);

            ++next;

            m_output_done = false;

          }

        } 

        else if (m_output_done == false && is_dropped(*next)) {

          m_output_done = true;

        } 

        else {

          if (is_dropped(*next))

            start=++next;

          for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)

            assigner::plus_equal(tok,*next);

          m_output_done = true;

        }

      }

      assigner::assign(start,next,tok);

      return true;

    }

  private:

    string_type m_kept_delims;

    string_type m_dropped_delims;

    bool m_use_ispunct;

    bool m_use_isspace;

    empty_token_policy m_empty_tokens;

    bool m_output_done;

    bool is_kept(Char E) const

    {  

      if (m_kept_delims.length())

        return m_kept_delims.find(E) != string_type::npos;

      else if (m_use_ispunct) {

        return std::ispunct(E) != 0;

      } else

        return false;

    }

    bool is_dropped(Char E) const

    {

      if (m_dropped_delims.length())

        return m_dropped_delims.find(E) != string_type::npos;

      else if (m_use_isspace) {

        return std::isspace(E) != 0;

      } else

        return false;

    }

  };

  //===========================================================================

  // The following class is DEPRECATED, use class char_separators instead.

  //

  // The char_delimiters_separator class, which is a model of

  // TokenizerFunction.  char_delimiters_separator breaks a string

  // into tokens based on character delimiters. There are 2 types of

  // delimiters. returnable delimiters can be returned as

  // tokens. These are often punctuation. nonreturnable delimiters

  // cannot be returned as tokens. These are often whitespace

  // The out of the box GCC 2.95 on cygwin does not have a char_traits class.

#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300

  template <class Char, 

    class Traits = typename std::basic_string<Char>::traits_type >

#else

  template <class Char, 

    class Traits = std::basic_string<Char>::traits_type >

#endif