2

I am reading lines of a text file into a variable, myline and then I am trying to tokenize these lines using istringstream. However, it seems that I am losing random characters from original text file string.

cout<< myline << buff << flush; //print original text file line
istringstream iss(myline);
string sub;
while (iss >> sub) {
cout << "[" << sub << "]" << endl;
}

If you look at my output, you can see that I have the correct string from the text file, but when I use istringstream and then print the individual tokens (seen within [] brackets), some of the tokens are prematurely truncated.

#include <iostream>
[#include]
[<iostream]
#include <sstream>
[#include]
[<sstream>]
using namespace std;
[using]
[namespace]
[st]

int main()
[int]
[main(]
{
    string str("   SOME  LONG    STRING\twith\nSPACES    ");
[string]
[str("]
[SOME]
[LONG]
[STRING\twith\nSPACES]

    istringstream iss(str);
[istringstream]
[iss(str);]

    string s;
[strin]
    while (iss >> s) {
[while]
[(iss]
[>>]
        cout << "[" << s << "]" << endl;
[cout]
[<<]
["["]
[<<]
[s]
[<<]
["]"]
[<<]
[e]
    }
    return 0;
[retur]
}

Does anybody have any ideas what I'm doing wrong? Thanks in advance!

EDIT: Here's a version of the code that will compile fully. You can run it with any text file

#include <cstring>
#include <cstdio>
#include <iostream>
#include <iomanip>
#include <string>
#include <sstream>
#include <vector>
#include <algorithm>
#include <iterator>

using namespace std;

class MyFileReader {

public:
    //constructor
    MyFileReader(const char* p);

    //destructor
    ~MyFileReader();

    //getLine()
    int getLine(char *buffer, int size);

    //getCurrentLineNumber()
    int getCurrentLineNumber();

    void tokenizeLine(vector<string>& vec);

    FILE * pFile;

};

    //constructor
    MyFileReader::MyFileReader(const char* p) {
        pFile = fopen(p, "r");
    }

    //destructor
    MyFileReader::~MyFileReader() {
        fclose(pFile);
    }

    //getLine()
    int MyFileReader::getLine(char *buffer, int size){
        char *out = fgets(buffer, size, pFile);
        if (out==NULL) {
            return -1;
        }
        char *pch = strpbrk(out,"\n");
        if (pch != NULL) {
            return 1;
        }
        else {
            return 0;
        }

    }

    int MyFileReader::getCurrentLineNumber() {
        static int mynumber=2;
        return mynumber++;
    }

    //tokenizeLine
    void MyFileReader::tokenizeLine(vector<string>& vec) {
        string myline("");
        char buff[10];
        while (1) {
            int result = getLine(buff, sizeof(buff));
            if (result == -1 ) {
                if (myline.length() > 0) 
                    cout << myline << flush;
            break;
            }
            else if (result == 0) {
                myline += buff;
            }
            else if (result == 1) {
                cout<< myline << buff << flush;
                istringstream iss(myline);
                string sub;
                while (iss >> sub) {
                    cout << "[" << sub << "]" << endl;
                } 
                myline = "";
            }
            else {
                printf("PANIC");
            }
            }
            return;
        }

    int main(int argc, char **argv) {
    vector<string> v;

    const char *filename = argv[1];
    MyFileReader f(filename);
    f.tokenizeLine(v);
    return 0;

    }

To generate the output above, I ran it on:

#include <iostream>
#include <sstream>
using namespace std;

int main()
{
    string str("   SOME  LONG    STRING\twith\nSPACES    ");

    istringstream iss(str);

    string s;
    while (iss >> s) {
        cout << "[" << s << "]" << endl;
    }
    return 0;
}
4

2 回答 2

1

错误在这里：

else if (result == 1) {
            cout<< myline << buff << flush;
            istringstream iss(myline);
            string sub;
            while (iss >> sub) {
                cout << "[" << sub << "]" << endl;
            } 
            myline = "";
        }

如果result == 1, 这意味着buff 包含 \n，并不意味着它只包含\n。即，如果缓冲区包含\n. 因此，如果该行恰好有 n*10 (sizeof buffer) 个字符，则您的代码可以工作，否则，一行的最后一个字符不会被复制到myline而是被删除。

快速解决方法是：

    else if (result == 1) {
            myline += buff; // copy the rest of the line into `myline`
            cout<< myline << flush; // buff now is part of myline
            istringstream iss(myline);
            string sub;
            while (iss >> sub) {
                cout << "[" << sub << "]" << endl;
            } 
            myline = "";
        }

尽管您可能需要考虑\n从缓冲区中删除，例如：

int MyFileReader::getLine(char *buffer, int size){
    char *out = fgets(buffer, size, pFile);
    if (out==NULL) {
        return -1;
    }
    //char *pch = strpbrk(out,"\n");
    char *pch = strchr(out,'\n'); // no need to search for a string
    if (pch != NULL) {
        *pch = '\0'; // drop the '\n'
        return 1;
    }
    else {
        return 0;
    }

}

不过，您必须更改cout<< myline << flush;为cout<< myline << endl;。

除了这个错误，请考虑使用ifstream：

#include <iostream>
#include <fstream>
#include <sstream>
using namespace std;

int main()
{
    ifstream file("test.txt");
    if(!file)
    {
        /* error */
    }else
    {
        string line;
        while(getline(file, line))
        {
            istringstream iss(line);

            string s;
            while (iss >> s) {
                cout << "[" << s << "]" << endl;
            }
        }
    }
}
于 2013-05-03T22:29:01.460 回答
1

您的行缓冲区只有 10 个字节长。这还不够长，不足以容纳一条完整的线。

void MyFileReader::tokenizeLine(vector<string>& vec) {
        string myline("");
        char buff[10];// this is too short
....

编辑

正如 Dyp 正确指出，当您在输入文件中检测到 \n 时，您的附加逻辑不正确。

于 2013-05-03T22:30:24.120 回答