Java String Tokenize tokenizeExpression(String s)

Description

Crude expression tokenizer - will not work for complex expresions but may suffice for most

License

Apache License

Parameter

Parameter	Description
s	non-null expression

Return

non-null array of tokens

Declaration

public static String[] tokenizeExpression(String s) 

Method Source Code

//package com.java2s;
//License from project: Apache License 

import java.util.*;
import java.util.List;

public class Main {
    public static final int NOT_ACCUMULATING = 0;
    public static final int ACCUM_VARIABLE = 1;
    public static final int ACCUM_NUMBER = 2;
    public static final int ACCUM_PUNCT = 3;

    /**/*from  w w w  .  j  a va  2 s  . co m*/
     * Crude expression tokenizer - will not work for complex expresions
     * but may suffice for most
     * @param s non-null expression
     * @return non-null array of tokens
     */
    public static String[] tokenizeExpression(String s) {
        String[] ret;
        List tokens = new ArrayList();
        int SLength = s.length();
        StringBuffer accum = new StringBuffer(SLength);
        int accumType = NOT_ACCUMULATING;

        for (int i = 0; i < SLength; i++) {
            char c = s.charAt(i);
            if (accumType == NOT_ACCUMULATING) {
                if (Character.isWhitespace(c))
                    continue;
                accumType = findAcumType(c);
                accum.append(c);
            } else {
                switch (accumType) {
                case ACCUM_VARIABLE:
                    if (Character.isJavaIdentifierPart(c)) {
                        accum.append(c);
                    } else {
                        saveaccumulatedToken(accum, tokens);
                        accumType = findAcumType(c);
                        if (accumType != NOT_ACCUMULATING) {
                            accum.append(c);
                        }
                    }
                    break;
                case ACCUM_NUMBER:
                    if (Character.isDigit(c) || c == '.') {
                        accum.append(c);
                    } else {
                        saveaccumulatedToken(accum, tokens);
                        accumType = findAcumType(c);
                        if (accumType != NOT_ACCUMULATING) {
                            accum.append(c);
                        }
                    }
                    break;
                case ACCUM_PUNCT:
                    if (isCompatablePunctuation(accum, c)) {
                        accum.append(c);
                    } else {
                        saveaccumulatedToken(accum, tokens);
                        accumType = findAcumType(c);
                        if (accumType != NOT_ACCUMULATING) {
                            accum.append(c);
                        }
                    }
                    break;
                }
            }

        }
        if (accum.length() > 0) {
            tokens.add(accum.toString());
        }

        ret = new String[tokens.size()];
        tokens.toArray(ret);
        return (ret);
    }

    private static int findAcumType(char pC) {
        int accumType;
        if (Character.isWhitespace(pC))
            return NOT_ACCUMULATING;
        if (Character.isJavaIdentifierStart(pC))
            return ACCUM_VARIABLE;
        if (Character.isDigit(pC))
            return ACCUM_NUMBER;

        return ACCUM_PUNCT;
    }

    private static void saveaccumulatedToken(StringBuffer pAccum, List pTokens) {
        if (pAccum.length() > 0) {
            pTokens.add(pAccum.toString());
            pAccum.setLength(0);
        }
    }

    private static boolean isCompatablePunctuation(StringBuffer accumType, char c) {
        if (Character.isDigit(c) || Character.isWhitespace(c) || Character.isJavaIdentifierPart(c))
            return false;
        if (c == '=' && (accumType.equals("<") || accumType.equals(">") || accumType.equals("=")))
            return true;
        return false; // Todo be smarter

    }

    /**{ method
     @name toString
     @function convert a char to a string
     @param c the char
     @return the String
     }*/
    public static String toString(char c) {
        StringBuffer s = new StringBuffer();
        s.append(c);
        return (s.toString());
    }
}

Related

1. tokenize(String value)
2. tokenize(String[] stringArray, String delimiter)
3. tokenizeAutocomplete(String text, int minSize)
4. tokenizeClassName(final String className)
5. tokenizeDirectiveCall(String data)
6. tokenizeFilter(CharSequence filter)
7. tokenizeLine(String input)
8. tokenizeLine(String line)
9. tokenizeName(String name)