hey DaWei, no i didnt know there was a standard tokenizer - could you give a bit more details?
well im only testing this on Windows at the moment - im only using conio for keeping the console open while i test....
well im trying to write a tokenizer class:
doesnt seem to be working properly, here it is:
class Tokenizer {
public:
//constructors
Tokenizer(const char *file);
Tokenizer(const char *file,istream in);
//destructor
~Tokenizer();
//operators
string operator++();
string operator--();
private:
//member data
ifstream mIn;
char *mFile;
protected:
//
};
//std dependencies
#include <iostream>
using namespace std;
//local dependencies
#include "Tokenizer.h"
//constructor 1
Tokenizer::Tokenizer(const char *file) {
//read in passed file
mIn.open(file,ios::in);
if(!mIn) {
cerr << "error: could not open file for reading!";
exit(1);
}
}
//constructor2 - use own ifstream object
Tokenizer::Tokenizer(const char *file,istream in) {
//
}
//destructor
Tokenizer::~Tokenizer() {
//close stream
mIn.close();
}
//get next token from stream
string Tokenizer::operator++() {
//data
string token;
//skip all whitespace
while(mIn.peek() == ' ') {
mIn.seekg(1,ios::cur);
}
//get token
char ch;
while(mIn.get(ch)) {
//get token
if(ch == ' ' || ch == '\n') {
//got token
//check if it is a valid token
if(token == "") {
return "kNull";
}
return token;
} else {
token.push_back(ch);
}
}
//didnt get a token
return "kNull";
}
//get previous token from the stream
string Tokenizer::operator--() {
//
return "";
}
it doesnt seem to print the last character, im testing by running this:
int main() {
//
Tokenizer tokenizer(inputFile);
cout << endl << tokenizer++;
cout << endl << tokenizer++;
cout << endl << tokenizer++;
cout << endl << tokenizer++;
...
it returns kNull for the last token... im really battling...
i want to be able to do something like this:
while(tokenizer++ != "atoken") {
cout << endl << tokenizer++;
}
but not working...
damn i need way more practice!
if any better suggestions/ideas please shout out! im really starting to understand the importance of practical programming, ive never really done any practical programming - and it all makes sense that "practise makes perfect"....