List of usage examples for java.io StreamTokenizer wordChars
public void wordChars(int low, int hi)
low <= c <= high are word constituents. From source file:com.enonic.cms.business.portal.datasource.expressionfunctions.ExpressionFunctions.java
/** * This method will take a freetext search string and create a valid query that can be used in the getContent* methods. The search * string is spilt into tokens. Using the operator, it may be specified whether the field must contain all or any of the words in the * search string./*w w w . jav a 2 s . c o m*/ * * @param fieldName The name of the field to search for the words in the search string. * @param searchString The words to search for. * @param operator Must be either AND or OR. Case doesn't matter. * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care, * it may also be merged with other queries using AND or OR. * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR. */ public String buildFreetextQuery(String fieldName, String searchString, String operator) { if (searchString == null || searchString.trim().equals("")) { return ""; } if (fieldName == null || fieldName.trim().equals("")) { throw new IllegalArgumentException("fieldName can not be empty."); } String op = ""; if (operator != null) { op = operator.trim().toUpperCase(); } if (!(op.equals("AND") || op.equals("OR"))) { throw new IllegalArgumentException("Illegal operator: " + operator); } boolean first = true; StringBuffer queryTokens = new StringBuffer(); Reader searchStringReader = new StringReader(searchString); StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader); searchStringTokens.slashSlashComments(false); searchStringTokens.slashStarComments(false); searchStringTokens.eolIsSignificant(false); searchStringTokens.ordinaryChar('!'); searchStringTokens.ordinaryChars('#', '}'); searchStringTokens.wordChars('!', '!'); searchStringTokens.wordChars('#', '}'); try { while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) { String token = searchStringTokens.sval; addQueryPart(queryTokens, first, fieldName, token, op); if (first) { first = false; } } } catch (IOException e) { throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!"); } return queryTokens.toString(); }
From source file:com.enonic.cms.core.portal.datasource.el.ExpressionFunctions.java
/** * This method will take a freetext search string and create a valid query that can be used in the getContent* methods. The search * string is spilt into tokens. Using the operator, it may be specified whether the field must contain all or any of the words in the * search string./* ww w. ja v a 2 s. c o m*/ * * @param fieldName The name of the field to search for the words in the search string. * @param searchString The words to search for. * @param operator Must be either AND or OR. Case doesn't matter. * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care, * it may also be merged with other queries using AND or OR. * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR. */ public String buildFreetextQuery(String fieldName, String searchString, String operator) { if (searchString == null || searchString.trim().equals("")) { return ""; } if (fieldName == null || fieldName.trim().equals("")) { throw new IllegalArgumentException("fieldName can not be empty."); } String op = ""; if (operator != null) { op = operator.trim().toUpperCase(); } if (!(op.equals("AND") || op.equals("OR"))) { throw new IllegalArgumentException("Illegal operator: " + operator); } boolean first = true; StringBuilder queryTokens = new StringBuilder(); Reader searchStringReader = new StringReader(searchString); StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader); searchStringTokens.slashSlashComments(false); searchStringTokens.slashStarComments(false); searchStringTokens.eolIsSignificant(false); searchStringTokens.ordinaryChar('!'); searchStringTokens.ordinaryChars('#', '}'); searchStringTokens.wordChars('!', '!'); searchStringTokens.wordChars('#', '}'); try { while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) { String token = searchStringTokens.sval; addQueryPart(queryTokens, first, fieldName, token, op); if (first) { first = false; } } } catch (IOException e) { throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!"); } return queryTokens.toString(); }
From source file:Matrix.java
/** * Read a matrix from a stream. The format is the same the print method, so * printed matrices can be read back in (provided they were printed using US * Locale). Elements are separated by whitespace, all the elements for each * row appear on a single line, the last row is followed by a blank line. * /*from www.j a va 2 s. c o m*/ * @param input * the input stream. */ public static Matrix read(BufferedReader input) throws java.io.IOException { StreamTokenizer tokenizer = new StreamTokenizer(input); // Although StreamTokenizer will parse numbers, it doesn't recognize // scientific notation (E or D); however, Double.valueOf does. // The strategy here is to disable StreamTokenizer's number parsing. // We'll only get whitespace delimited words, EOL's and EOF's. // These words should all be numbers, for Double.valueOf to parse. tokenizer.resetSyntax(); tokenizer.wordChars(0, 255); tokenizer.whitespaceChars(0, ' '); tokenizer.eolIsSignificant(true); java.util.Vector v = new java.util.Vector(); // Ignore initial empty lines while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) ; if (tokenizer.ttype == StreamTokenizer.TT_EOF) throw new java.io.IOException("Unexpected EOF on matrix read."); do { v.addElement(Double.valueOf(tokenizer.sval)); // Read & store 1st // row. } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD); int n = v.size(); // Now we've got the number of columns! double row[] = new double[n]; for (int j = 0; j < n; j++) // extract the elements of the 1st row. row[j] = ((Double) v.elementAt(j)).doubleValue(); v.removeAllElements(); v.addElement(row); // Start storing rows instead of columns. while (tokenizer.nextToken() == StreamTokenizer.TT_WORD) { // While non-empty lines v.addElement(row = new double[n]); int j = 0; do { if (j >= n) throw new java.io.IOException("Row " + v.size() + " is too long."); row[j++] = Double.valueOf(tokenizer.sval).doubleValue(); } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD); if (j < n) throw new java.io.IOException("Row " + v.size() + " is too short."); } int m = v.size(); // Now we've got the number of rows. double[][] A = new double[m][]; v.copyInto(A); // copy the rows out of the vector return new Matrix(A); }
From source file:com.redskyit.scriptDriver.RunTests.java
private void initTokenizer(StreamTokenizer tokenizer) { tokenizer.quoteChar('"'); tokenizer.slashStarComments(true);/* w w w . ja v a 2 s . c o m*/ tokenizer.slashSlashComments(true); tokenizer.whitespaceChars(' ', ' '); tokenizer.whitespaceChars(0x09, 0x09); tokenizer.wordChars('$', '$'); // treat $ as part of word tokenizer.wordChars('#', '#'); // treat # as part of word tokenizer.wordChars('_', '_'); // treat $# as part of word }
From source file:org.apache.openjpa.jdbc.kernel.SQLStoreQuery.java
/** * Utility method to substitute '?num' for parameters in the given SQL * statement, and fill-in the order of the parameter tokens *///from w w w .j av a2 s . co m public static String substituteParams(String sql, List<Integer> paramOrder) throws IOException { // if there's no "?" parameter marker, then we don't need to // perform the parsing process if (sql.indexOf("?") == -1) return sql; paramOrder.clear(); StreamTokenizer tok = new StreamTokenizer(new StringReader(sql)); tok.resetSyntax(); tok.quoteChar('\''); tok.wordChars('0', '9'); tok.wordChars('?', '?'); StringBuilder buf = new StringBuilder(sql.length()); for (int ttype; (ttype = tok.nextToken()) != StreamTokenizer.TT_EOF;) { switch (ttype) { case StreamTokenizer.TT_WORD: // a token is a positional parameter if it starts with // a "?" and the rest of the token are all numbers if (tok.sval.startsWith("?")) { buf.append("?"); String pIndex = tok.sval.substring(1); if (pIndex.length() > 0) { paramOrder.add(Integer.valueOf(pIndex)); } else { // or nothing paramOrder.add(paramOrder.size() + 1); } } else buf.append(tok.sval); break; case '\'': buf.append('\''); if (tok.sval != null) { buf.append(tok.sval); buf.append('\''); } break; default: buf.append((char) ttype); } } return buf.toString(); }
From source file:org.jdesigner.platform.web.converter.AbstractArrayConverter.java
/** * <p>//from w w w. j a v a 2s.c o m * Parse an incoming String of the form similar to an array initializer in * the Java language into a <code>List</code> individual Strings for each * element, according to the following rules. * </p> * <ul> * <li>The string is expected to be a comma-separated list of values.</li> * <li>The string may optionally have matching '{' and '}' delimiters around * the list.</li> * <li>Whitespace before and after each element is stripped.</li> * <li>Elements in the list may be delimited by single or double quotes. * Within a quoted elements, the normal Java escape sequences are valid.</li> * </ul> * * @param svalue * String value to be parsed * @return The parsed list of String values * * @exception ConversionException * if the syntax of <code>svalue</code> is not syntactically * valid * @exception NullPointerException * if <code>svalue</code> is <code>null</code> */ protected List parseElements(String svalue) { // Validate the passed argument if (svalue == null) { throw new NullPointerException(); } // Trim any matching '{' and '}' delimiters svalue = svalue.trim(); if (svalue.startsWith("{") && svalue.endsWith("}")) { svalue = svalue.substring(1, svalue.length() - 1); } try { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(svalue)); st.whitespaceChars(',', ','); // Commas are delimiters st.ordinaryChars('0', '9'); // Needed to turn off numeric flag st.ordinaryChars('.', '.'); st.ordinaryChars('-', '-'); st.wordChars('0', '9'); // Needed to make part of tokens st.wordChars('.', '.'); st.wordChars('-', '-'); // Split comma-delimited tokens into a List ArrayList list = new ArrayList(); while (true) { int ttype = st.nextToken(); if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) { list.add(st.sval); } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype); } } // Return the completed list return (list); } catch (IOException e) { throw new ConversionException(e); } }
From source file:org.jdesigner.platform.web.converter.ArrayConverter.java
/** * <p>//from w w w . ja va 2 s .c om * Parse an incoming String of the form similar to an array initializer in * the Java language into a <code>List</code> individual Strings for each * element, according to the following rules. * </p> * <ul> * <li>The string is expected to be a comma-separated list of values.</li> * <li>The string may optionally have matching '{' and '}' delimiters around * the list.</li> * <li>Whitespace before and after each element is stripped.</li> * <li>Elements in the list may be delimited by single or double quotes. * Within a quoted elements, the normal Java escape sequences are valid.</li> * </ul> * * @param type * The type to convert the value to * @param value * String value to be parsed * @return List of parsed elements. * * @throws ConversionException * if the syntax of <code>svalue</code> is not syntactically * valid * @throws NullPointerException * if <code>svalue</code> is <code>null</code> */ private List parseElements(Class type, String value) { if (log().isDebugEnabled()) { log().debug("Parsing elements, delimiter=[" + delimiter + "], value=[" + value + "]"); } // Trim any matching '{' and '}' delimiters value = value.trim(); if (value.startsWith("{") && value.endsWith("}")) { value = value.substring(1, value.length() - 1); } try { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); st.whitespaceChars(delimiter, delimiter); // Set the delimiters st.ordinaryChars('0', '9'); // Needed to turn off numeric flag st.wordChars('0', '9'); // Needed to make part of tokens for (int i = 0; i < allowedChars.length; i++) { st.ordinaryChars(allowedChars[i], allowedChars[i]); st.wordChars(allowedChars[i], allowedChars[i]); } // Split comma-delimited tokens into a List List list = null; while (true) { int ttype = st.nextToken(); if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) { if (st.sval != null) { if (list == null) { list = new ArrayList(); } list.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException( "Encountered token of type " + ttype + " parsing elements to '" + toString(type) + "."); } } if (list == null) { list = Collections.EMPTY_LIST; } if (log().isDebugEnabled()) { log().debug(list.size() + " elements parsed"); } // Return the completed list return (list); } catch (IOException e) { throw new ConversionException( "Error converting from String to '" + toString(type) + "': " + e.getMessage(), e); } }
From source file:org.openhab.io.caldav.internal.util.ExecuteCommandJob.java
/** * Parses a <code>command</code>. Utilizes the {@link StreamTokenizer} which * takes care of quoted Strings as well. * // ww w . j av a 2s.co m * @param command the command to parse * @return the tokenized command which can be processed by the * <code>ConsoleInterpreter</code> * * @see org.openhab.io.console.ConsoleInterpreter */ protected String[] parseCommand(String command) { logger.trace("going to parse command '{}'", command); // if the command starts with '>' it contains a script which needs no // further handling here ... if (command.startsWith(">")) { return new String[] { ">", command.substring(1).trim() }; } StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(command)); tokenizer.wordChars('_', '_'); tokenizer.wordChars('-', '-'); tokenizer.wordChars('.', '.'); List<String> tokens = new ArrayList<String>(); try { int tokenType = 0; while (tokenType != StreamTokenizer.TT_EOF && tokenType != StreamTokenizer.TT_EOL) { tokenType = tokenizer.nextToken(); String token = ""; switch (tokenType) { case StreamTokenizer.TT_WORD: case 34 /* quoted String */: token = tokenizer.sval; break; case StreamTokenizer.TT_NUMBER: token = String.valueOf(tokenizer.nval); break; } tokens.add(token); logger.trace("read value {} from the given command", token); } } catch (IOException ioe) { } return tokens.toArray(new String[0]); }
From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString/*www . j a v a 2 s. co m*/ * A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus" * @param variableSpace * Context for resolving variable names. If {@code null}, no variable resolution we happen. * @param ignoreSqoopCommand * If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export". * @return List of parsed arguments * @throws IOException * when the command line could not be parsed */ public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace, boolean ignoreSqoopCommand) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments, // e.g. "sqoop <tool>") boolean skipToken = false; // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; if (variableSpace != null) { s = variableSpace.environmentSubstitute(s); } if (ignoreSqoopCommand && args.isEmpty()) { // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of // arguments directly // from a working command line if ("sqoop".equals(s)) { skipToken = true; continue; // skip this one and the next } else if (skipToken) { ignoreSqoopCommand = false; // Don't attempt to ignore any more commands // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing skipToken = false; continue; } } if (s.startsWith(ARG_D)) { handleCustomOption(args, s, tokenizer, variableSpace); continue; } args.add(escapeEscapeSequences(s)); } } } finally { reader.close(); } return args; }
From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString A command line string. * @return List of parsed arguments/*from w ww . j ava 2 s . c om*/ * @throws IOException when the command line could not be parsed */ public List<String> parseCommandLine(String commandLineString) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; s = environmentSubstitute(s); args.add(s); } } } finally { reader.close(); } return args; }