Java String Tokenize Tokenize(String sent)

Description

Tokenize

License

Open Source License

Declaration

public static ArrayList<String> Tokenize(String sent) 

Method Source Code

//package com.java2s;
//License from project: Open Source License 

import java.util.ArrayList;

public class Main {
    public static final String SPLIT_TOKENS = "[!\"#$%&'()*+,./:;<=>?\\[\\]^`{|}~\\s]";

    public static ArrayList<String> Tokenize(String sent) {
        ArrayList<String> result = new ArrayList<String>();
        String tokens[] = sent.split(SPLIT_TOKENS);
        //ArrayList<String> tokens = ST.extractTokens(sent, true);
        for (String token : tokens) {
            token = token.trim().toLowerCase();
            if (token.length() == 0)
                continue;
            result.add(token);/*w  w w . j  a va2 s . c o  m*/
        }
        return result;
    }
}

Related

1. tokenize(String s, char separator)
2. tokenize(String s, char separator, int maxTokens)
3. tokenize(String s, char separator, int maxTokens)
4. tokenize(String s, String delimiters)
5. tokenize(String s1)
6. tokenize(String sentence)
7. tokenize(String source, char separator)
8. tokenize(String source, char separator)
9. tokenize(String str, char delim)