List of usage examples for java.io StreamTokenizer quoteChar
public void quoteChar(int ch)
From source file:com.indeed.imhotep.index.builder.util.SmartArgs.java
/** * parse a String into an argument list, in the same way java parses * arguments passed to main()//from w ww . j a v a 2 s . c om */ public static String[] parseArgParams(String line) { StreamTokenizer tok = new StreamTokenizer(new StringReader(line)); tok.resetSyntax(); tok.wordChars('\u0000', '\uFFFF'); tok.whitespaceChars(' ', ' '); tok.quoteChar('\"'); ArrayList<String> output = new ArrayList<String>(); try { while (tok.nextToken() != StreamTokenizer.TT_EOF) { output.add(tok.sval); } } catch (IOException e) { } return output.toArray(new String[output.size()]); }
From source file:com.xpn.xwiki.util.Util.java
/** * Create a Map from a string holding a space separated list of key=value pairs. If keys or values must contain * spaces, they can be placed inside quotes, like <code>"this key"="a larger value"</code>. To use a quote as part * of a key/value, use <code>%_Q_%</code>. * /*from w w w. jav a 2 s . c o m*/ * @param mapString The string that must be parsed. * @return A Map containing the keys and values. If a key is defined more than once, the last value is used. */ public static Hashtable<String, String> keyValueToHashtable(String mapString) throws IOException { Hashtable<String, String> result = new Hashtable<String, String>(); StreamTokenizer st = new StreamTokenizer(new BufferedReader(new StringReader(mapString))); st.resetSyntax(); st.quoteChar('"'); st.wordChars('a', 'z'); st.wordChars('A', 'Z'); st.whitespaceChars(' ', ' '); st.whitespaceChars('=', '='); while (st.nextToken() != StreamTokenizer.TT_EOF) { String key = st.sval; st.nextToken(); String value = (st.sval != null) ? st.sval : ""; result.put(key, restoreValue(value)); } return result; }
From source file:com.zimbra.common.calendar.ZoneInfo2iCalendar.java
private static void readExtraData(Reader reader) throws IOException, ParseException { char dquote = '"'; StreamTokenizer tokenizer = new StreamTokenizer(reader); tokenizer.resetSyntax();//www.j a v a 2s . c o m tokenizer.wordChars(32, 126); tokenizer.whitespaceChars(' ', ' '); tokenizer.whitespaceChars('\t', '\t'); tokenizer.whitespaceChars(0, 20); tokenizer.commentChar('#'); tokenizer.quoteChar(dquote); tokenizer.eolIsSignificant(true); List<String> tokenList = new ArrayList<String>(); LineType lineType = LineType.UNKNOWN; boolean atLineStart = true; int ttype; int prevTtype = StreamTokenizer.TT_EOL; // used for empty line detection while ((ttype = tokenizer.nextToken()) != StreamTokenizer.TT_EOF) { int lineNum = tokenizer.lineno(); if (ttype == StreamTokenizer.TT_WORD || ttype == dquote) { String token = tokenizer.sval; if (atLineStart) { lineType = LineType.lookUp(token); if (LineType.UNKNOWN.equals(lineType)) throw new ParseException("Invalid line type", lineNum); } else { tokenList.add(token); } atLineStart = false; } else if (ttype == StreamTokenizer.TT_EOL) { if (prevTtype == StreamTokenizer.TT_EOL) { prevTtype = ttype; continue; } atLineStart = true; switch (lineType) { case PRIMARYZONE: if (tokenList.size() < 1) throw new ParseException("Not enough fields in a PrimaryZone line", lineNum); String primaryTZID = tokenList.get(0); sPrimaryTZIDs.add(primaryTZID); break; case ZONEMATCHSCORE: if (tokenList.size() < 2) throw new ParseException("Not enough fields in a ZoneMatchScore line", lineNum); String zoneName = tokenList.get(0); String zoneMatchScoreStr = tokenList.get(1); int zoneMatchScore = 0; try { zoneMatchScore = Integer.parseInt(zoneMatchScoreStr); } catch (NumberFormatException e) { throw new ParseException("Zone match score must be an integer: " + zoneMatchScoreStr, lineNum); } sMatchScores.put(zoneName, zoneMatchScore); break; } if (atLineStart) { tokenList.clear(); lineType = LineType.UNKNOWN; } } else if (ttype == StreamTokenizer.TT_NUMBER) { // shouldn't happen throw new ParseException("Invalid parser state: TT_NUMBER found", lineNum); } prevTtype = ttype; } }
From source file:com.redskyit.scriptDriver.RunTests.java
private void initTokenizer(StreamTokenizer tokenizer) { tokenizer.quoteChar('"'); tokenizer.slashStarComments(true);/*from w ww . j a v a 2 s. c o m*/ tokenizer.slashSlashComments(true); tokenizer.whitespaceChars(' ', ' '); tokenizer.whitespaceChars(0x09, 0x09); tokenizer.wordChars('$', '$'); // treat $ as part of word tokenizer.wordChars('#', '#'); // treat # as part of word tokenizer.wordChars('_', '_'); // treat $# as part of word }
From source file:org.apache.openjpa.jdbc.kernel.SQLStoreQuery.java
/** * Utility method to substitute '?num' for parameters in the given SQL * statement, and fill-in the order of the parameter tokens *//*from ww w . j a va2s. c o m*/ public static String substituteParams(String sql, List<Integer> paramOrder) throws IOException { // if there's no "?" parameter marker, then we don't need to // perform the parsing process if (sql.indexOf("?") == -1) return sql; paramOrder.clear(); StreamTokenizer tok = new StreamTokenizer(new StringReader(sql)); tok.resetSyntax(); tok.quoteChar('\''); tok.wordChars('0', '9'); tok.wordChars('?', '?'); StringBuilder buf = new StringBuilder(sql.length()); for (int ttype; (ttype = tok.nextToken()) != StreamTokenizer.TT_EOF;) { switch (ttype) { case StreamTokenizer.TT_WORD: // a token is a positional parameter if it starts with // a "?" and the rest of the token are all numbers if (tok.sval.startsWith("?")) { buf.append("?"); String pIndex = tok.sval.substring(1); if (pIndex.length() > 0) { paramOrder.add(Integer.valueOf(pIndex)); } else { // or nothing paramOrder.add(paramOrder.size() + 1); } } else buf.append(tok.sval); break; case '\'': buf.append('\''); if (tok.sval != null) { buf.append(tok.sval); buf.append('\''); } break; default: buf.append((char) ttype); } } return buf.toString(); }
From source file:org.eclipse.mylyn.internal.phabricator.core.client.TracWebClient.java
/** * Parses the JavaScript code from the query page to extract repository configuration. */// w ww . j a v a2 s .c o m private void parseAttributesTokenizer(String text) throws IOException { StreamTokenizer t = new StreamTokenizer(new StringReader(text)); t.quoteChar('"'); TracConfiguration configuration = new TracConfiguration(data); AttributeFactory attributeFactory = null; String attributeType = null; AttributeState state = AttributeState.INIT; int tokenType; while ((tokenType = t.nextToken()) != StreamTokenizer.TT_EOF) { switch (tokenType) { case StreamTokenizer.TT_WORD: case '"': if (state == AttributeState.IN_LIST) { attributeFactory = configuration.getFactoryByField(t.sval); if (attributeFactory != null) { attributeFactory.initialize(); } } else if (state == AttributeState.IN_ATTRIBUTE_KEY) { attributeType = t.sval; } else if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST && "options".equals(attributeType)) { //$NON-NLS-1$ if (attributeFactory != null) { attributeFactory.addAttribute(t.sval); } } break; case ':': if (state == AttributeState.IN_ATTRIBUTE_KEY) { state = AttributeState.IN_ATTRIBUTE_VALUE; } break; case ',': if (state == AttributeState.IN_ATTRIBUTE_VALUE) { state = AttributeState.IN_ATTRIBUTE_KEY; } break; case '[': if (state == AttributeState.IN_ATTRIBUTE_VALUE) { state = AttributeState.IN_ATTRIBUTE_VALUE_LIST; } break; case ']': if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST) { state = AttributeState.IN_ATTRIBUTE_VALUE; } break; case '{': if (state == AttributeState.INIT) { state = AttributeState.IN_LIST; } else if (state == AttributeState.IN_LIST) { state = AttributeState.IN_ATTRIBUTE_KEY; } else { throw new IOException("Error parsing attributes: unexpected token '{'"); //$NON-NLS-1$ } break; case '}': if (state == AttributeState.IN_ATTRIBUTE_KEY || state == AttributeState.IN_ATTRIBUTE_VALUE) { state = AttributeState.IN_LIST; } else if (state == AttributeState.IN_LIST) { state = AttributeState.INIT; } else { throw new IOException("Error parsing attributes: unexpected token '}'"); //$NON-NLS-1$ } break; } } }
From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString// www . j a v a 2s . c o m * A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus" * @param variableSpace * Context for resolving variable names. If {@code null}, no variable resolution we happen. * @param ignoreSqoopCommand * If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export". * @return List of parsed arguments * @throws IOException * when the command line could not be parsed */ public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace, boolean ignoreSqoopCommand) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments, // e.g. "sqoop <tool>") boolean skipToken = false; // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; if (variableSpace != null) { s = variableSpace.environmentSubstitute(s); } if (ignoreSqoopCommand && args.isEmpty()) { // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of // arguments directly // from a working command line if ("sqoop".equals(s)) { skipToken = true; continue; // skip this one and the next } else if (skipToken) { ignoreSqoopCommand = false; // Don't attempt to ignore any more commands // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing skipToken = false; continue; } } if (s.startsWith(ARG_D)) { handleCustomOption(args, s, tokenizer, variableSpace); continue; } args.add(escapeEscapeSequences(s)); } } } finally { reader.close(); } return args; }
From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString A command line string. * @return List of parsed arguments/*w w w .j a v a 2 s.c o m*/ * @throws IOException when the command line could not be parsed */ public List<String> parseCommandLine(String commandLineString) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; s = environmentSubstitute(s); args.add(s); } } } finally { reader.close(); } return args; }
From source file:org.xwiki.extension.internal.ExtensionUtils.java
/** * @param str the String to parse/*from w w w . ja va2 s. c o m*/ * @param trim true if the passed String should be trimmed * @return the collection of Strings extracted from the passed String * @since 8.3M1 */ public static List<String> importPropertyStringList(String str, boolean trim) { try { String cleanedString = str; // Trim if (trim) { cleanedString = cleanedString.trim(); } // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters st.whitespaceChars(',', ','); st.whitespaceChars(' ', ' '); st.whitespaceChars('\t', '\t'); st.whitespaceChars('\n', '\n'); st.whitespaceChars('\r', '\r'); // Split comma-delimited tokens into a List List<String> collection = new ArrayList<>(); while (true) { int ttype = st.nextToken(); if (ttype == StreamTokenizer.TT_WORD || ttype > 0) { if (st.sval != null) { collection.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype + " parsing elements."); } } // Return the completed list return collection; } catch (Exception e) { // Log ? } return Collections.emptyList(); }
From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java
/** * Create and initializer a {@link StreamTokenizer} to parse the value. * /*from w w w . j a va2 s . c om*/ * @param value the string to parse * @return the {@link StreamTokenizer} used to parse the string */ protected StreamTokenizer createStreamTokenizer(String value) { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters for (char c : getDelimiters().toCharArray()) { st.whitespaceChars(c, c); } return st; }