2

我boost::tokenizer用来读取类似 CSV 的文件。我将令牌存储在std::vector. 它运作良好，但我只想boost::iterator为每个令牌存储一个。

我试过：

#include <string>
#include <boost/tokenizer.hpp>
#include <boost/range/iterator_range.hpp>

typedef std::string::const_iterator string_iter;
typedef boost::iterator_range<string_iter> string_view;

int main(){
    std::string line;

    std::vector<string_view> contents;

    boost::tokenizer<boost::escaped_list_separator<char>, string_iter, string_view> tok(line.begin(), line.end());
    contents.assing(tok.begin(), tok.end());
}

但它无法编译：

/usr/include/boost/token_functions.hpp：在 'bool boost::escaped_list_separator::operator()(InputIterator&, InputIterator, Token&) 的实例化中 [with InputIterator = __gnu_cxx::__normal_iterator >; 令牌 = boost::iterator_range<__gnu_cxx::__normal_iterator >>; 字符 = 字符；Traits = std::char_traits]': /usr/include/boost/token_iterator.hpp:70:11: 需要来自'void boost::token_iterator::initialize() [with TokenizerFunc = boost::escaped_list_separator; 迭代器 = __gnu_cxx::__normal_iterator >; 类型 = boost::iterator_range<__gnu_cxx::__normal_iterator >>]' /usr/include/boost/token_iterator.hpp:77:63: 来自 'boost::token_iterator::token_iterator(TokenizerFunc, Iterator, Iterator) [with TokenizerFunc = boost::escaped_list_separator; 迭代器 = __gnu_cxx::__normal_iterator >; 类型 = boost::iterator_range<__gnu_cxx::__normal_iterator >>]' /usr/include/boost/tokenizer.hpp:86:53: 来自 'boost::tokenizer::iter boost::tokenizer::begin() 常量[与 TokenizerFunc = boost::escaped_list_separator; 迭代器 = __gnu_cxx::__normal_iterator >; 类型 = boost::iterator_range<__gnu_cxx::__normal_iterator >>; boost::tokenizer::iter = boost::token_iterator, __gnu_cxx::__normal_iterator >, boost::iterator_range<__gnu_cxx::__normal_iterator >> >]' /home/wichtounet/dev/gooda-to-afdo-converter/src/ gooda_reader.cpp:58:37: 从这里需要 /usr/include/boost/token_functions.hpp:187:16: 错误: 'tok += (& next)->__gnu_cxx::__normal_iterator 中的 'operator+=' 不匹配<_Iterator, _Container>::operator* >()' /usr/include/boost/token_functions.hpp:193:11: 错误：'tok += (& next)->__gnu_cxx::__normal_iterator<_Iterator, _Container>::operator* >()' /usr/include/boost/token_functions.hpp 中的 'operator+=' 不匹配：在实例化中'void boost::escaped_list_separator::do_escape(iterator&, iterator, Token&) [with iterator = __gnu_cxx::__normal_iterator >; 令牌 = boost::iterator_range<__gnu_cxx::__normal_iterator >>; 字符 = 字符；Traits = std::char_traits]': /usr/include/boost/token_functions.hpp:176:11: 来自'bool boost::escaped_list_separator::operator()(InputIterator&, InputIterator, Token&) [with InputIterator = __gnu_cxx: :__normal_iterator >; 令牌 = boost::iterator_range<__gnu_cxx::__normal_iterator >>; 字符 = 字符；特征 = std::char_traits]' /usr/include/boost/token_iterator.hpp:70:11: 'void boost::token_iterator::initialize() [with TokenizerFunc = boost::escaped_list_separator; 迭代器 = __gnu_cxx::__normal_iterator >; 类型 = boost::iterator_range<__gnu_cxx::__normal_iterator >>]' /usr/include/boost/token_iterator.hpp:77:63: 来自 'boost::token_iterator::token_iterator(TokenizerFunc, Iterator, Iterator) [with TokenizerFunc = boost::escaped_list_separator; 迭代器 = __gnu_cxx::__normal_iterator >; 类型 = boost::iterator_range<__gnu_cxx::__normal_iterator >>]' /usr/include/boost/tokenizer.hpp:86:53: 来自 'boost::tokenizer::iter boost::tokenizer::begin() 常量[与 TokenizerFunc = boost::escaped_list_separator; 迭代器 = __gnu_cxx::__normal_iterator >; 类型 = boost::iterator_range<__gnu_cxx::__normal_iterator >>; 升压::分词器::

我也只是尝试使用 自己计算两个迭代器boost::token_iterator，但到目前为止我还没有成功。

是否有解决方案只获取每个令牌的迭代器范围而不是字符串以节省一些性能？

4

2 回答 2

3

啊! 你需要一个包括：

#include <iostream>
#include <boost/tokenizer.hpp>
#include <boost/range/iterator_range.hpp>
#include <string>

int main()
{
    std::string line;
    typedef std::string::const_iterator string_iter;
    typedef boost::iterator_range<string_iter> string_view;

    boost::tokenizer<boost::escaped_list_separator<char>, string_iter, string_view> tok(line.begin(), line.end());
}

编译得很好

于 2012-11-08T20:25:21.797 回答
3

这是行不通的。需要一个类型（第三tokenizer个模板参数），它可以附加标记器函数的结果。具体来说，它必须提供 operator += ( tokenizer<...>::iterator::value_type )。下面的代码片段应该让你更进一步，虽然我不确定它是否值得努力......

#include <string>
#include <boost/tokenizer.hpp>
#include <boost/range/iterator_range.hpp>
#include <iostream>
#include <cstddef>

typedef std::string::const_iterator string_iter;
typedef boost::iterator_range<string_iter> string_view;

// a constant size character buffer, skips anything beyond CSize...
template< std::size_t CSize >
class assignable_view {
   std::size_t m_size;
   char m_buffer[CSize];

   friend std::ostream& operator << (std::ostream& p_out, assignable_view const & p_view)
   {
      if (p_view.m_size > 0u) {
         std::copy(p_view.m_buffer, p_view.m_buffer + p_view.m_size, std::ostream_iterator<char>(p_out));
      }
      return p_out;
   }

public:
   template <class TIter>
   void operator += (TIter p_input) 
   {
      if (m_size < CSize) {
         m_buffer[m_size++] = p_input;
      }   
   }   
   assignable_view() 
      : m_size(0u) {}
};

int main(){
    std::string line
        = "Field 1,\"putting quotes around fields, allows commas\",Field 3";

    std::vector<string_view> contents;

    boost::tokenizer<
       boost::escaped_list_separator<char>, 
       string_iter, 
       assignable_view<11>    
    > tok(line.begin(), line.end());

    for (auto const & t_s : tok) {
       std::cout << t_s << std::endl;
    }
    //contents.assing(tok.begin(), tok.end());
}
于 2012-11-10T10:13:30.763 回答