Create a class called tokenizer that tokenizes a string. - C++ STL

C++ examples for STL:string

  1. HOME
  2. C++
  3. STL
  4. string

Description

Create a class called tokenizer that tokenizes a string.

Demo Code

#include <iostream>
#include <string>
using namespace std;

class tokenizer {
   string s;/*from   www . ja  va2s.  c  om*/
   string::size_type startidx;
   string::size_type endidx;
   public:
   tokenizer(const string &str) {
      s = str;
      startidx = 0;
   }
   string get_token(const string &delims);
};

string tokenizer::get_token(const string &delims) {
   if(startidx == string::npos)
      return string("");
   endidx = s.find_first_of(delims, startidx);
   string tok(s.substr(startidx, endidx-startidx));
   startidx = s.find_first_not_of(delims, endidx);
   return tok;
}
int main()
{
   // Strings to be tokenized.
   string strA("test test test I have 123 321 four, (*&(&*(&*five, six tokens. ");
   string strB("I might have more tokens!\nDo You?");
   string delimiters(" ,.!?\n");
   string token;
   tokenizer tokA(strA);
   tokenizer tokB(strB);
   cout << "The tokens in strA:\n";
   token = tokA.get_token(delimiters);

   while(token != "") {
      cout << token << endl;
      token = tokA.get_token(delimiters);
   }
   cout << endl;
   cout << "The tokens in strB:\n";
   token = tokB.get_token(delimiters);
   while(token != "") {
      cout << token << endl;
      token = tokB.get_token(delimiters);
   }
   return 0;
}

Result


Related Tutorials