Revert last code drop.
// Boost token_functions.hpp ------------------------------------------------//
// Copyright John R. Bandela 2001.
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/tokenizer/ for documentation.
// Revision History:
// 01 Oct 2004 Joaquín M López Muñoz
// Workaround for a problem with string::assign in msvc-stlport
// 06 Apr 2004 John Bandela
// Fixed a bug involving using char_delimiter with a true input iterator
// 28 Nov 2003 Robert Zeh and John Bandela
// Converted into "fast" functions that avoid using += when
// the supplied iterator isn't an input_iterator; based on
// some work done at Archelon and a version that was checked into
// the boost CVS for a short period of time.
// 20 Feb 2002 John Maddock
// Removed using namespace std declarations and added
// workaround for BOOST_NO_STDC_NAMESPACE (the library
// can be safely mixed with regex).
// 06 Feb 2002 Jeremy Siek
// Added char_separator.
// 02 Feb 2002 Jeremy Siek
// Removed tabs and a little cleanup.
#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
#include <vector>
#include <stdexcept>
#include <string>
#include <cctype>
#include <algorithm> // for find_if
#include <boost/config.hpp>
#include <boost/assert.hpp>
#include <boost/detail/workaround.hpp>
#include <boost/mpl/if.hpp>
//
// the following must not be macros if we are to prefix them
// with std:: (they shouldn't be macros anyway...)
//
#ifdef ispunct
# undef ispunct
#endif
#ifdef isspace
# undef isspace
#endif
//
// fix namespace problems:
//
#ifdef BOOST_NO_STDC_NAMESPACE
namespace std{
using ::ispunct;
using ::isspace;
}
#endif
namespace boost{
//===========================================================================
// The escaped_list_separator class. Which is a model of TokenizerFunction
// An escaped list is a super-set of what is commonly known as a comma
// separated value (csv) list.It is separated into fields by a comma or
// other character. If the delimiting character is inside quotes, then it is
// counted as a regular character.To allow for embedded quotes in a field,
// there can be escape sequences using the \ much like C.
// The role of the comma, the quotation mark, and the escape
// character (backslash \), can be assigned to other characters.
struct escaped_list_error : public std::runtime_error{
escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
};
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
// MSVC does not like the following typename
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
template <class Char,
class Traits = typename std::basic_string<Char>::traits_type >
#else
template <class Char,
class Traits = std::basic_string<Char>::traits_type >
#endif
class escaped_list_separator {
private:
typedef std::basic_string<Char,Traits> string_type;
struct char_eq {
Char e_;
char_eq(Char e):e_(e) { }
bool operator()(Char c) {
return Traits::eq(e_,c);
}
};
string_type escape_;
string_type c_;
string_type quote_;
bool last_;
bool is_escape(Char e) {
char_eq f(e);
return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
}
bool is_c(Char e) {
char_eq f(e);
return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
}
bool is_quote(Char e) {
char_eq f(e);
return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
}
template <typename iterator, typename Token>
void do_escape(iterator& next,iterator end,Token& tok) {
if (++next == end)
throw escaped_list_error(std::string("cannot end with escape"));
if (Traits::eq(*next,'n')) {
tok+='\n';
return;
}
else if (is_quote(*next)) {
tok+=*next;
return;
}
else if (is_c(*next)) {
tok+=*next;
return;
}
else if (is_escape(*next)) {
tok+=*next;
return;
}
else
throw escaped_list_error(std::string("unknown escape sequence"));
}
public:
explicit escaped_list_separator(Char e = '\\',
Char c = ',',Char q = '\"')
: escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
escaped_list_separator(string_type e, string_type c, string_type q)
: escape_(e), c_(c), quote_(q), last_(false) { }
void reset() {last_=false;}
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next,InputIterator end,Token& tok) {
bool bInQuote = false;
tok = Token();
if (next == end) {
if (last_) {
last_ = false;
return true;
}
else
return false;
}
last_ = false;
for (;next != end;++next) {
if (is_escape(*next)) {
do_escape(next,end,tok);
}
else if (is_c(*next)) {
if (!bInQuote) {
// If we are not in quote, then we are done
++next;
// The last character was a c, that means there is
// 1 more blank field
last_ = true;
return true;
}
else tok+=*next;
}
else if (is_quote(*next)) {
bInQuote=!bInQuote;
}
else {
tok += *next;
}
}
return true;
}
};
//===========================================================================
// The classes here are used by offset_separator and char_separator to implement
// faster assigning of tokens using assign instead of +=
namespace tokenizer_detail {
// The assign_or_plus_equal struct contains functions that implement
// assign, +=, and clearing based on the iterator type. The
// generic case does nothing for plus_equal and clearing, while
// passing through the call for assign.
//
// When an input iterator is being used, the situation is reversed.
// The assign method does nothing, plus_equal invokes operator +=,
// and the clearing method sets the supplied token to the default
// token constructor's result.
//
template<class IteratorTag>
struct assign_or_plus_equal {
template<class Iterator, class Token>
static void assign(Iterator b, Iterator e, Token &t) {
#if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\
BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\
defined(_STLP_DEBUG) &&\
(defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))
// Problem with string::assign for msvc-stlport in debug mode: the
// linker tries to import the templatized version of this memfun,
// which is obviously not exported.
// See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.
t = Token();
while(b != e) t += *b++;
#else
t.assign(b, e);
#endif
}
template<class Token, class Value>
static void plus_equal(Token &, const Value &) {
}
// If we are doing an assign, there is no need for the
// the clear.
//
template<class Token>
static void clear(Token &) {
}
};
template <>
struct assign_or_plus_equal<std::input_iterator_tag> {
template<class Iterator, class Token>
static void assign(Iterator b, Iterator e, Token &t) {
}
template<class Token, class Value>
static void plus_equal(Token &t, const Value &v) {
t += v;
}
template<class Token>
static void clear(Token &t) {
t = Token();
}
};
template<class Iterator>
struct pointer_iterator_category{
typedef std::random_access_iterator_tag type;
};
template<class Iterator>
struct class_iterator_category{
typedef typename Iterator::iterator_category type;
};
// This portably gets the iterator_tag without partial template specialization
template<class Iterator>
struct get_iterator_category{
typedef typename mpl::if_<is_pointer<Iterator>,
pointer_iterator_category<Iterator>,
class_iterator_category<Iterator>
>::type cat;
typedef typename cat::type iterator_category;
};
}
//===========================================================================
// The offset_separator class, which is a model of TokenizerFunction.
// Offset breaks a string into tokens based on a range of offsets
class offset_separator {
private:
std::vector<int> offsets_;
unsigned int current_offset_;
bool wrap_offsets_;
bool return_partial_last_;
public:
template <typename Iter>
offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
bool return_partial_last = true)
: offsets_(begin,end), current_offset_(0),
wrap_offsets_(wrap_offsets),
return_partial_last_(return_partial_last) { }
offset_separator()
: offsets_(1,1), current_offset_(),
wrap_offsets_(true), return_partial_last_(true) { }
void reset() {
current_offset_ = 0;
}
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end, Token& tok)
{
typedef tokenizer_detail::assign_or_plus_equal<
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
typename
#endif
tokenizer_detail::get_iterator_category<
InputIterator>::iterator_category> assigner;
BOOST_ASSERT(!offsets_.empty());
assigner::clear(tok);
InputIterator start(next);
if (next == end)
return false;
if (current_offset_ == offsets_.size())
if (wrap_offsets_)
current_offset_=0;
else
return false;
int c = offsets_[current_offset_];
int i = 0;
for (; i < c; ++i) {
if (next == end)break;
assigner::plus_equal(tok,*next++);
}
assigner::assign(start,next,tok);
if (!return_partial_last_)
if (i < (c-1) )
return false;
++current_offset_;
return true;
}
};
//===========================================================================
// The char_separator class breaks a sequence of characters into
// tokens based on the character delimiters (very much like bad old
// strtok). A delimiter character can either be kept or dropped. A
// kept delimiter shows up as an output token, whereas a dropped
// delimiter does not.
// This class replaces the char_delimiters_separator class. The
// constructor for the char_delimiters_separator class was too
// confusing and needed to be deprecated. However, because of the
// default arguments to the constructor, adding the new constructor
// would cause ambiguity, so instead I deprecated the whole class.
// The implementation of the class was also simplified considerably.
enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
template <typename Char,
typename Traits = typename std::basic_string<Char>::traits_type >
#else
template <typename Char,
typename Traits = std::basic_string<Char>::traits_type >
#endif
class char_separator
{
typedef std::basic_string<Char,Traits> string_type;
public:
explicit
char_separator(const Char* dropped_delims,
const Char* kept_delims = 0,
empty_token_policy empty_tokens = drop_empty_tokens)
: m_dropped_delims(dropped_delims),
m_use_ispunct(false),
m_use_isspace(false),
m_empty_tokens(empty_tokens),
m_output_done(false)
{
// Borland workaround
if (kept_delims)
m_kept_delims = kept_delims;
}
// use ispunct() for kept delimiters and isspace for dropped.
explicit
char_separator()
: m_use_ispunct(true),
m_use_isspace(true),
m_empty_tokens(drop_empty_tokens) { }
void reset() { }
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end, Token& tok)
{
typedef tokenizer_detail::assign_or_plus_equal<
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
typename
#endif
tokenizer_detail::get_iterator_category<
InputIterator>::iterator_category> assigner;
assigner::clear(tok);
// skip past all dropped_delims
if (m_empty_tokens == drop_empty_tokens)
for (; next != end && is_dropped(*next); ++next)
{ }
InputIterator start(next);
if (m_empty_tokens == drop_empty_tokens) {
if (next == end)
return false;
// if we are on a kept_delims move past it and stop
if (is_kept(*next)) {
assigner::plus_equal(tok,*next);
++next;
} else
// append all the non delim characters
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
assigner::plus_equal(tok,*next);
}
else { // m_empty_tokens == keep_empty_tokens
// Handle empty token at the end
if (next == end)
if (m_output_done == false) {
m_output_done = true;
assigner::assign(start,next,tok);
return true;
} else
return false;
if (is_kept(*next)) {
if (m_output_done == false)
m_output_done = true;
else {
assigner::plus_equal(tok,*next);
++next;
m_output_done = false;
}
}
else if (m_output_done == false && is_dropped(*next)) {
m_output_done = true;
}
else {
if (is_dropped(*next))
start=++next;
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
assigner::plus_equal(tok,*next);
m_output_done = true;
}
}
assigner::assign(start,next,tok);
return true;
}
private:
string_type m_kept_delims;
string_type m_dropped_delims;
bool m_use_ispunct;
bool m_use_isspace;
empty_token_policy m_empty_tokens;
bool m_output_done;
bool is_kept(Char E) const
{
if (m_kept_delims.length())
return m_kept_delims.find(E) != string_type::npos;
else if (m_use_ispunct) {
return std::ispunct(E) != 0;
} else
return false;
}
bool is_dropped(Char E) const
{
if (m_dropped_delims.length())
return m_dropped_delims.find(E) != string_type::npos;
else if (m_use_isspace) {
return std::isspace(E) != 0;
} else
return false;
}
};
//===========================================================================
// The following class is DEPRECATED, use class char_separators instead.
//
// The char_delimiters_separator class, which is a model of
// TokenizerFunction. char_delimiters_separator breaks a string
// into tokens based on character delimiters. There are 2 types of
// delimiters. returnable delimiters can be returned as
// tokens. These are often punctuation. nonreturnable delimiters
// cannot be returned as tokens. These are often whitespace
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
template <class Char,
class Traits = typename std::basic_string<Char>::traits_type >
#else
template <class Char,
class Traits = std::basic_string<Char>::traits_type >
#endif
class char_delimiters_separator {
private:
typedef std::basic_string<Char,Traits> string_type;
string_type returnable_;
string_type nonreturnable_;
bool return_delims_;
bool no_ispunct_;
bool no_isspace_;
bool is_ret(Char E)const
{
if (returnable_.length())
return returnable_.find(E) != string_type::npos;
else{
if (no_ispunct_) {return false;}
else{
int r = std::ispunct(E);
return r != 0;
}
}
}
bool is_nonret(Char E)const
{
if (nonreturnable_.length())
return nonreturnable_.find(E) != string_type::npos;
else{
if (no_isspace_) {return false;}
else{
int r = std::isspace(E);
return r != 0;
}
}
}
public:
explicit char_delimiters_separator(bool return_delims = false,
const Char* returnable = 0,
const Char* nonreturnable = 0)
: returnable_(returnable ? returnable : string_type().c_str()),
nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
return_delims_(return_delims), no_ispunct_(returnable!=0),
no_isspace_(nonreturnable!=0) { }
void reset() { }
public:
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end,Token& tok) {
tok = Token();
// skip past all nonreturnable delims
// skip past the returnable only if we are not returning delims
for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
&& !return_delims_ ) );++next) { }
if (next == end) {
return false;
}
// if we are to return delims and we are one a returnable one
// move past it and stop
if (is_ret(*next) && return_delims_) {
tok+=*next;
++next;
}
else
// append all the non delim characters
for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
tok+=*next;
return true;
}
};
} //namespace boost
#endif