List of usage examples for java.io StreamTokenizer whitespaceChars
public void whitespaceChars(int low, int hi)
low <= c <= high are white space characters. From source file:org.jdesigner.platform.web.converter.ArrayConverter.java
/** * <p>// w w w. jav a 2s .co m * Parse an incoming String of the form similar to an array initializer in * the Java language into a <code>List</code> individual Strings for each * element, according to the following rules. * </p> * <ul> * <li>The string is expected to be a comma-separated list of values.</li> * <li>The string may optionally have matching '{' and '}' delimiters around * the list.</li> * <li>Whitespace before and after each element is stripped.</li> * <li>Elements in the list may be delimited by single or double quotes. * Within a quoted elements, the normal Java escape sequences are valid.</li> * </ul> * * @param type * The type to convert the value to * @param value * String value to be parsed * @return List of parsed elements. * * @throws ConversionException * if the syntax of <code>svalue</code> is not syntactically * valid * @throws NullPointerException * if <code>svalue</code> is <code>null</code> */ private List parseElements(Class type, String value) { if (log().isDebugEnabled()) { log().debug("Parsing elements, delimiter=[" + delimiter + "], value=[" + value + "]"); } // Trim any matching '{' and '}' delimiters value = value.trim(); if (value.startsWith("{") && value.endsWith("}")) { value = value.substring(1, value.length() - 1); } try { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); st.whitespaceChars(delimiter, delimiter); // Set the delimiters st.ordinaryChars('0', '9'); // Needed to turn off numeric flag st.wordChars('0', '9'); // Needed to make part of tokens for (int i = 0; i < allowedChars.length; i++) { st.ordinaryChars(allowedChars[i], allowedChars[i]); st.wordChars(allowedChars[i], allowedChars[i]); } // Split comma-delimited tokens into a List List list = null; while (true) { int ttype = st.nextToken(); if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) { if (st.sval != null) { if (list == null) { list = new ArrayList(); } list.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException( "Encountered token of type " + ttype + " parsing elements to '" + toString(type) + "."); } } if (list == null) { list = Collections.EMPTY_LIST; } if (log().isDebugEnabled()) { log().debug(list.size() + " elements parsed"); } // Return the completed list return (list); } catch (IOException e) { throw new ConversionException( "Error converting from String to '" + toString(type) + "': " + e.getMessage(), e); } }
From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString/*w w w.ja v a2 s . c om*/ * A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus" * @param variableSpace * Context for resolving variable names. If {@code null}, no variable resolution we happen. * @param ignoreSqoopCommand * If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export". * @return List of parsed arguments * @throws IOException * when the command line could not be parsed */ public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace, boolean ignoreSqoopCommand) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments, // e.g. "sqoop <tool>") boolean skipToken = false; // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; if (variableSpace != null) { s = variableSpace.environmentSubstitute(s); } if (ignoreSqoopCommand && args.isEmpty()) { // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of // arguments directly // from a working command line if ("sqoop".equals(s)) { skipToken = true; continue; // skip this one and the next } else if (skipToken) { ignoreSqoopCommand = false; // Don't attempt to ignore any more commands // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing skipToken = false; continue; } } if (s.startsWith(ARG_D)) { handleCustomOption(args, s, tokenizer, variableSpace); continue; } args.add(escapeEscapeSequences(s)); } } } finally { reader.close(); } return args; }
From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString A command line string. * @return List of parsed arguments//from w w w . j a v a 2s .c o m * @throws IOException when the command line could not be parsed */ public List<String> parseCommandLine(String commandLineString) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; s = environmentSubstitute(s); args.add(s); } } } finally { reader.close(); } return args; }
From source file:org.xwiki.extension.internal.ExtensionUtils.java
/** * @param str the String to parse/*from w ww . ja v a 2 s . c o m*/ * @param trim true if the passed String should be trimmed * @return the collection of Strings extracted from the passed String * @since 8.3M1 */ public static List<String> importPropertyStringList(String str, boolean trim) { try { String cleanedString = str; // Trim if (trim) { cleanedString = cleanedString.trim(); } // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters st.whitespaceChars(',', ','); st.whitespaceChars(' ', ' '); st.whitespaceChars('\t', '\t'); st.whitespaceChars('\n', '\n'); st.whitespaceChars('\r', '\r'); // Split comma-delimited tokens into a List List<String> collection = new ArrayList<>(); while (true) { int ttype = st.nextToken(); if (ttype == StreamTokenizer.TT_WORD || ttype > 0) { if (st.sval != null) { collection.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype + " parsing elements."); } } // Return the completed list return collection; } catch (Exception e) { // Log ? } return Collections.emptyList(); }
From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java
/** * Create and initializer a {@link StreamTokenizer} to parse the value. * //from ww w .ja v a 2s .c o m * @param value the string to parse * @return the {@link StreamTokenizer} used to parse the string */ protected StreamTokenizer createStreamTokenizer(String value) { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters for (char c : getDelimiters().toCharArray()) { st.whitespaceChars(c, c); } return st; }