Example usage for java.io StreamTokenizer TT

Introduction

In this page you can find the example usage for java.io StreamTokenizer TT_EOF.

Prototype

int TT_EOF

To view the source code for java.io StreamTokenizer TT_EOF.

Click Source Link

Document

A constant indicating that the end of the stream has been read.

Usage

From source file:com.rapidminer.tools.Tools.java

/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
    if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
        throw new IOException("unexpected end of line " + tokenizer.lineno());
    }// w  w w .j av  a 2 s .  com

    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
        throw new IOException("unexpected end of file in line " + tokenizer.lineno());
    } else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
        tokenizer.ttype = '?';
    }
}

From source file:com.jcraft.weirdx.XColormap.java

static void init(Map<String, Color> table) {
    if (_rgbtxt == null)
        return;/*from ww  w . j  a  va2  s  .c om*/
    StringBuffer foo = new StringBuffer();
    for (int i = 0; i < _rgbtxt.length; i++) {
        foo.append(_rgbtxt[i]);
    }
    rgbtxt = foo.toString();
    _rgbtxt = null;
    foo = null;

    try {
        InputStream is = new ByteArrayInputStream(RGBTXT.rgbtxt.getBytes());
        //      StreamTokenizer st=new StreamTokenizer(is);
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        StreamTokenizer st = new StreamTokenizer(br);

        st.ordinaryChar('!');
        //    st.ordinaryChar('\n');
        //    st.ordinaryChar('\t');
        //String token=null;
        char c;
        int r, g, b;
        byte[] buf = new byte[1024];
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            //System.out.println("type="+st.ttype+", "+st.sval);
            if (st.ttype == '!') {
                //     while((c=(char)is.read())!='\n');
                while ((c = (char) br.read()) != '\n')
                    ;
                continue;
            }
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                r = (int) st.nval;
                st.nextToken();
                g = (int) st.nval;
                st.nextToken();
                b = (int) st.nval;
                //System.out.print("r, g, b="+r+", "+g+", "+b);
                int i = 0;
                //     while((c=(char)is.read())!='\n'){
                while ((c = (char) br.read()) != '\n') {
                    if (c == '\t')
                        continue;
                    if (c == ' ')
                        continue;
                    if ('A' <= c && c <= 'Z') {
                        c = (char) ('a' + c - 'A');
                    }
                    buf[i] = (byte) c;
                    i++;
                }
                table.put(new String(buf, 0, i), new Color(r, g, b));
                //System.out.println(" -> "+new String(buf, 0, i));
                continue;
            }
        }
        st = null;
        buf = null;
        //    table.put("slategrey", rgbTable.get("slate grey"));
        //    table.put("Black", rgbTable.get("black"));
        //    table.put("White", rgbTable.get("white"));
    } catch (Exception e) {
        //System.out.println(e);
    }
}

From source file:org.apache.openjpa.jdbc.kernel.SQLStoreQuery.java

/**
 * Utility method to substitute '?num' for parameters in the given SQL
 * statement, and fill-in the order of the parameter tokens
 *///from   ww  w . ja  v  a 2s.c om
public static String substituteParams(String sql, List<Integer> paramOrder) throws IOException {
    // if there's no "?" parameter marker, then we don't need to
    // perform the parsing process
    if (sql.indexOf("?") == -1)
        return sql;

    paramOrder.clear();
    StreamTokenizer tok = new StreamTokenizer(new StringReader(sql));
    tok.resetSyntax();
    tok.quoteChar('\'');
    tok.wordChars('0', '9');
    tok.wordChars('?', '?');

    StringBuilder buf = new StringBuilder(sql.length());
    for (int ttype; (ttype = tok.nextToken()) != StreamTokenizer.TT_EOF;) {
        switch (ttype) {
        case StreamTokenizer.TT_WORD:
            // a token is a positional parameter if it starts with
            // a "?" and the rest of the token are all numbers
            if (tok.sval.startsWith("?")) {
                buf.append("?");
                String pIndex = tok.sval.substring(1);
                if (pIndex.length() > 0) {
                    paramOrder.add(Integer.valueOf(pIndex));
                } else { // or nothing
                    paramOrder.add(paramOrder.size() + 1);
                }
            } else
                buf.append(tok.sval);
            break;
        case '\'':
            buf.append('\'');
            if (tok.sval != null) {
                buf.append(tok.sval);
                buf.append('\'');
            }
            break;
        default:
            buf.append((char) ttype);
        }
    }
    return buf.toString();
}

From source file:org.apache.qpid.server.security.access.config.PlainConfiguration.java

@Override
public RuleSet load() {
    RuleSet ruleSet = super.load();

    File file = getFile();/*from   w  ww .  j a v a 2  s.  c om*/
    FileReader fileReader = null;

    try {
        if (_logger.isDebugEnabled()) {
            _logger.debug("About to load ACL file " + file);
        }

        fileReader = new FileReader(file);
        _st = new StreamTokenizer(new BufferedReader(fileReader));
        _st.resetSyntax(); // setup the tokenizer

        _st.commentChar(COMMENT); // single line comments
        _st.eolIsSignificant(true); // return EOL as a token
        _st.ordinaryChar('='); // equals is a token
        _st.ordinaryChar(CONTINUATION); // continuation character (when followed by EOL)
        _st.quoteChar('"'); // double quote
        _st.quoteChar('\''); // single quote
        _st.whitespaceChars('\u0000', '\u0020'); // whitespace (to be ignored) TODO properly
        _st.wordChars('a', 'z'); // unquoted token characters [a-z]
        _st.wordChars('A', 'Z'); // [A-Z]
        _st.wordChars('0', '9'); // [0-9]
        _st.wordChars('_', '_'); // underscore
        _st.wordChars('-', '-'); // dash
        _st.wordChars('.', '.'); // dot
        _st.wordChars('*', '*'); // star
        _st.wordChars('@', '@'); // at
        _st.wordChars(':', ':'); // colon

        // parse the acl file lines
        Stack<String> stack = new Stack<String>();
        int current;
        do {
            current = _st.nextToken();
            switch (current) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                if (stack.isEmpty()) {
                    break; // blank line
                }

                // pull out the first token from the bottom of the stack and check arguments exist
                String first = stack.firstElement();
                stack.removeElementAt(0);
                if (stack.isEmpty()) {
                    throw new IllegalConfigurationException(String.format(NOT_ENOUGH_TOKENS_MSG, getLine()));
                }

                // check for and parse optional initial number for ACL lines
                Integer number = null;
                if (StringUtils.isNumeric(first)) {
                    // set the acl number and get the next element
                    number = Integer.valueOf(first);
                    first = stack.firstElement();
                    stack.removeElementAt(0);
                }

                if (StringUtils.equalsIgnoreCase(ACL, first)) {
                    parseAcl(number, stack);
                } else if (number == null) {
                    if (StringUtils.equalsIgnoreCase("GROUP", first)) {
                        throw new IllegalConfigurationException(String.format(
                                "GROUP keyword not supported. Groups should defined via a Group Provider, not in the ACL file.",
                                getLine()));
                    } else if (StringUtils.equalsIgnoreCase(CONFIG, first)) {
                        parseConfig(stack);
                    } else {
                        throw new IllegalConfigurationException(
                                String.format(UNRECOGNISED_INITIAL_MSG, first, getLine()));
                    }
                } else {
                    throw new IllegalConfigurationException(
                            String.format(NUMBER_NOT_ALLOWED_MSG, first, getLine()));
                }

                // reset stack, start next line
                stack.clear();
                break;
            case StreamTokenizer.TT_NUMBER:
                stack.push(Integer.toString(Double.valueOf(_st.nval).intValue()));
                break;
            case StreamTokenizer.TT_WORD:
                stack.push(_st.sval); // token
                break;
            default:
                if (_st.ttype == CONTINUATION) {
                    int next = _st.nextToken();
                    if (next == StreamTokenizer.TT_EOL) {
                        break; // continue reading next line
                    }

                    // invalid location for continuation character (add one to line beacuse we ate the EOL)
                    throw new IllegalConfigurationException(
                            String.format(PREMATURE_CONTINUATION_MSG, getLine() + 1));
                } else if (_st.ttype == '\'' || _st.ttype == '"') {
                    stack.push(_st.sval); // quoted token
                } else {
                    stack.push(Character.toString((char) _st.ttype)); // single character
                }
            }
        } while (current != StreamTokenizer.TT_EOF);

        if (!stack.isEmpty()) {
            throw new IllegalConfigurationException(String.format(PREMATURE_EOF_MSG, getLine()));
        }
    } catch (IllegalArgumentException iae) {
        throw new IllegalConfigurationException(String.format(PARSE_TOKEN_FAILED_MSG, getLine()), iae);
    } catch (FileNotFoundException fnfe) {
        throw new IllegalConfigurationException(String.format(CONFIG_NOT_FOUND_MSG, file.getName()), fnfe);
    } catch (IOException ioe) {
        throw new IllegalConfigurationException(String.format(CANNOT_LOAD_MSG, file.getName()), ioe);
    } finally {
        if (fileReader != null) {
            try {
                fileReader.close();
            } catch (IOException e) {
                throw new IllegalConfigurationException(String.format(CANNOT_CLOSE_MSG, file.getName()), e);
            }
        }
    }

    return ruleSet;
}

From source file:org.apache.wiki.plugin.DefaultPluginManager.java

/**
 *  Parses plugin arguments.  Handles quotes and all other kewl stuff.
 *
 *  <h3>Special parameters</h3>
 *  The plugin body is put into a special parameter defined by {@link #PARAM_BODY};
 *  the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE};
 *  and the bounds of the plugin within the wiki page text by a parameter defined
 *  by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array,
 *  i.e., <tt>[start,end]</tt>.
 *
 * @param argstring The argument string to the plugin.  This is
 *  typically a list of key-value pairs, using "'" to escape
 *  spaces in strings, followed by an empty line and then the
 *  plugin body.  In case the parameter is null, will return an
 *  empty parameter list./*from   www .  j a  v a2 s  .c o  m*/
 *
 * @return A parsed list of parameters.
 *
 * @throws IOException If the parsing fails.
 */
public Map<String, String> parseArgs(String argstring) throws IOException {
    Map<String, String> arglist = new HashMap<String, String>();

    //
    //  Protection against funny users.
    //
    if (argstring == null)
        return arglist;

    arglist.put(PARAM_CMDLINE, argstring);

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString((int) tok.nval);
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;

                arglist.put(param, value);

                // log.debug("ARG: "+param+"="+value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

From source file:org.eclipse.mylyn.internal.phabricator.core.client.TracWebClient.java

/**
 * Parses the JavaScript code from the query page to extract repository configuration.
 *//*from w w  w  . ja  v  a2s.co  m*/
private void parseAttributesTokenizer(String text) throws IOException {
    StreamTokenizer t = new StreamTokenizer(new StringReader(text));
    t.quoteChar('"');

    TracConfiguration configuration = new TracConfiguration(data);
    AttributeFactory attributeFactory = null;
    String attributeType = null;

    AttributeState state = AttributeState.INIT;
    int tokenType;
    while ((tokenType = t.nextToken()) != StreamTokenizer.TT_EOF) {
        switch (tokenType) {
        case StreamTokenizer.TT_WORD:
        case '"':
            if (state == AttributeState.IN_LIST) {
                attributeFactory = configuration.getFactoryByField(t.sval);
                if (attributeFactory != null) {
                    attributeFactory.initialize();
                }
            } else if (state == AttributeState.IN_ATTRIBUTE_KEY) {
                attributeType = t.sval;
            } else if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST && "options".equals(attributeType)) { //$NON-NLS-1$
                if (attributeFactory != null) {
                    attributeFactory.addAttribute(t.sval);
                }
            }
            break;
        case ':':
            if (state == AttributeState.IN_ATTRIBUTE_KEY) {
                state = AttributeState.IN_ATTRIBUTE_VALUE;
            }
            break;
        case ',':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_ATTRIBUTE_KEY;
            }
            break;
        case '[':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_ATTRIBUTE_VALUE_LIST;
            }
            break;
        case ']':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST) {
                state = AttributeState.IN_ATTRIBUTE_VALUE;
            }
            break;
        case '{':
            if (state == AttributeState.INIT) {
                state = AttributeState.IN_LIST;
            } else if (state == AttributeState.IN_LIST) {
                state = AttributeState.IN_ATTRIBUTE_KEY;
            } else {
                throw new IOException("Error parsing attributes: unexpected token '{'"); //$NON-NLS-1$
            }
            break;
        case '}':
            if (state == AttributeState.IN_ATTRIBUTE_KEY || state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_LIST;
            } else if (state == AttributeState.IN_LIST) {
                state = AttributeState.INIT;
            } else {
                throw new IOException("Error parsing attributes: unexpected token '}'"); //$NON-NLS-1$
            }
            break;
        }
    }
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Gets a description of the current token
 * /*from   w  w w. j  a va2  s.co m*/
 * @return a description of the current token
 */
private String tokenString() {
    switch (tokenizer.ttype) {
    case StreamTokenizer.TT_NUMBER:
        return "<NUMBER>";
    case StreamTokenizer.TT_EOL:
        return "End-of-Line";
    case StreamTokenizer.TT_EOF:
        return "End-of-Stream";
    case StreamTokenizer.TT_WORD:
        return "'" + tokenizer.sval + "'";
    }
    return "'" + (char) tokenizer.ttype + "'";
}

From source file:org.jdesigner.platform.web.converter.AbstractArrayConverter.java

/**
 * <p>/*from w w w . j  a va2 s.  c  om*/
 * Parse an incoming String of the form similar to an array initializer in
 * the Java language into a <code>List</code> individual Strings for each
 * element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around
 * the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes.
 * Within a quoted elements, the normal Java escape sequences are valid.</li>
 * </ul>
 * 
 * @param svalue
 *            String value to be parsed
 * @return The parsed list of String values
 * 
 * @exception ConversionException
 *                if the syntax of <code>svalue</code> is not syntactically
 *                valid
 * @exception NullPointerException
 *                if <code>svalue</code> is <code>null</code>
 */
protected List parseElements(String svalue) {

    // Validate the passed argument
    if (svalue == null) {
        throw new NullPointerException();
    }

    // Trim any matching '{' and '}' delimiters
    svalue = svalue.trim();
    if (svalue.startsWith("{") && svalue.endsWith("}")) {
        svalue = svalue.substring(1, svalue.length() - 1);
    }

    try {

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(svalue));
        st.whitespaceChars(',', ','); // Commas are delimiters
        st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
        st.ordinaryChars('.', '.');
        st.ordinaryChars('-', '-');
        st.wordChars('0', '9'); // Needed to make part of tokens
        st.wordChars('.', '.');
        st.wordChars('-', '-');

        // Split comma-delimited tokens into a List
        ArrayList list = new ArrayList();
        while (true) {
            int ttype = st.nextToken();
            if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
                list.add(st.sval);
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype);
            }
        }

        // Return the completed list
        return (list);

    } catch (IOException e) {

        throw new ConversionException(e);

    }

}

From source file:org.jdesigner.platform.web.converter.ArrayConverter.java

/**
 * <p>//from w  ww . jav  a2  s.co  m
 * Parse an incoming String of the form similar to an array initializer in
 * the Java language into a <code>List</code> individual Strings for each
 * element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around
 * the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes.
 * Within a quoted elements, the normal Java escape sequences are valid.</li>
 * </ul>
 * 
 * @param type
 *            The type to convert the value to
 * @param value
 *            String value to be parsed
 * @return List of parsed elements.
 * 
 * @throws ConversionException
 *             if the syntax of <code>svalue</code> is not syntactically
 *             valid
 * @throws NullPointerException
 *             if <code>svalue</code> is <code>null</code>
 */
private List parseElements(Class type, String value) {

    if (log().isDebugEnabled()) {
        log().debug("Parsing elements, delimiter=[" + delimiter + "], value=[" + value + "]");
    }

    // Trim any matching '{' and '}' delimiters
    value = value.trim();
    if (value.startsWith("{") && value.endsWith("}")) {
        value = value.substring(1, value.length() - 1);
    }

    try {

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(value));
        st.whitespaceChars(delimiter, delimiter); // Set the delimiters
        st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
        st.wordChars('0', '9'); // Needed to make part of tokens
        for (int i = 0; i < allowedChars.length; i++) {
            st.ordinaryChars(allowedChars[i], allowedChars[i]);
            st.wordChars(allowedChars[i], allowedChars[i]);
        }

        // Split comma-delimited tokens into a List
        List list = null;
        while (true) {
            int ttype = st.nextToken();
            if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
                if (st.sval != null) {
                    if (list == null) {
                        list = new ArrayList();
                    }
                    list.add(st.sval);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException(
                        "Encountered token of type " + ttype + " parsing elements to '" + toString(type) + ".");
            }
        }

        if (list == null) {
            list = Collections.EMPTY_LIST;
        }
        if (log().isDebugEnabled()) {
            log().debug(list.size() + " elements parsed");
        }

        // Return the completed list
        return (list);

    } catch (IOException e) {

        throw new ConversionException(
                "Error converting from String to '" + toString(type) + "': " + e.getMessage(), e);

    }

}

From source file:org.jrman.parser.Parser.java

public void parse(String filename) throws Exception {
    if (currentDirectory != null) {
        String fullFileName = (String) fullFileNames.get(filename);
        if (fullFileName == null) {
            fullFileName = currentDirectory + File.separator + filename;
            fullFileNames.put(filename, fullFileName);
        }/*  w w  w . java 2 s  . co  m*/
        filename = fullFileName;
    }
    FileReader fr = new FileReader(filename);
    Tokenizer st = new Tokenizer(new BufferedReader(fr));
    // st.commentChar('#');
    int tk;
    while ((tk = st.nextToken()) != StreamTokenizer.TT_EOF) {
        try {
            if (tk != StreamTokenizer.TT_WORD)
                throw new Exception("Expected keyword at line " + st.lineno());
            String keyword = st.sval;
            KeywordParser kp = getKeyWordParser(keyword);
            if (!kp.getValidStates().contains(state))
                throw new IllegalStateException(
                        "Keyword" + kp + " is not valid in state " + state + ", at line " + st.lineno());
            kp.parse(st);
        } catch (Exception pe) {
            System.err.println("Error: " + pe);
            pe.printStackTrace();
        }
    }
    fr.close();
}

Example usage for java.io StreamTokenizer TT_EOF

Introduction

Prototype

Document

Usage