27 void split(
const String & buffer, std::vector<spear::Word> & tokens);
33 static bool initialize(
const std::string & dataPath);
35 static bool initialized;
39 String concatenate(
const std::vector<String> & tokens,
41 unsigned int end)
const;
static String normalize(const String &s)
Definition: Tokenizer.cc:51
Reference counting pointer class This file contains the code for the classes and class templates maki...
RCIPtr< Tokenizer > TokenizerPtr
Definition: Tokenizer.h:44
Definition: Tokenizer.h:21
#define String
Definition: Wide.h:36
void split(const String &buffer, std::vector< spear::Word > &tokens)
Definition: Tokenizer.cc:84
Tokenizer(const std::string &dataPath)
Definition: Tokenizer.cc:44