Java Utililty Methods String Tokenize

List<Integer> tokenEndIndices = new ArrayList<Integer>();
int fromIndex = 0;
while ((fromIndex = string.indexOf(tokenSeparator, fromIndex)) != -1) {
    tokenEndIndices.add(fromIndex++);
if ((tokenEndIndices.isEmpty()) || (string.length() > tokenEndIndices.get(tokenEndIndices.size() - 1) + 1))
    tokenEndIndices.add(string.length());
fromIndex = -tokenSeparator.length();
...

return Arrays.asList(value.replaceAll("[^a-zA-Z0-9]", " ").split("[\\s@&.?$+-]+"));

String t = removeFillers(vbt);
Vector v = new Vector();
StringTokenizer st = new StringTokenizer(t, " ");
int knt = 0;
String retstr = "";
while (st.hasMoreTokens()) {
    String val = st.nextToken();
    val = val.toLowerCase();
...

tokenTypes = new ArrayList<>();
generateTokenTypes(tree);
return tokenTypes;

for (Iterator i = mUnitTypeMap.keySet().iterator(); i.hasNext();) {
    String key = (String) i.next();
    if (typeToken.equals(key)) {
        return (Integer) mUnitTypeMap.get(key);
throw new IllegalArgumentException("Type " + typeToken + " is not a known device type");

java.util.List<String> list = new java.util.ArrayList<String>();
if (_token.contains(".") == true) {
    String[] tokenArray = _token.split("\\.");
    list.add(tokenArray[0]);
    list.add(tokenArray[1]);
return list;

if (posTagSet == null) {
    initializePOSTagSet();
return posTagSet.contains(token);

return stringFunctions.contains(token);

StringTokenizer sentenceToken = new StringTokenizer(sentence, " .;:");
while (sentenceToken.hasMoreTokens()) {
    if (sentenceToken.nextToken().equals(searchWord))
        return true;
return false;

if (s == null)
    return 0;
int max = 0;
StringTokenizer st = new StringTokenizer(s);
while (st.hasMoreTokens()) {
    int tokenLength = st.nextToken().length();
    if (tokenLength > max)
        max = tokenLength;
...