List of usage examples for java.io StreamTokenizer StreamTokenizer
public StreamTokenizer(Reader r)
From source file:org.openmrs.module.reporting.query.evaluator.CompositionQueryEvaluator.java
/** * Elements in this list can be: an Integer, indicating a 1-based index into a search history a * BooleanOperator (AND, OR, NOT) a Query, another List of the same form, which indicates a parenthetical expression *///from w w w . ja v a 2s . c o m public List<Object> parseIntoTokens(String expression) throws EvaluationException { List<Object> tokens = new ArrayList<Object>(); try { StreamTokenizer st = new StreamTokenizer(new StringReader(expression)); for (Character c : CHARACTER_WORDS) { st.ordinaryChar(c); } while (st.nextToken() != StreamTokenizer.TT_EOF) { if (st.ttype == StreamTokenizer.TT_NUMBER) { Integer thisInt = (int) st.nval; if (thisInt < 1) { throw new IllegalArgumentException("Invalid number < 1 found"); } tokens.add(thisInt); } else if (OPEN_PARENTHESES_WORDS.contains(Character.valueOf((char) st.ttype))) { tokens.add("("); } else if (CLOSE_PARENTHESES_WORDS.contains(Character.valueOf((char) st.ttype))) { tokens.add(")"); } else if (st.ttype == StreamTokenizer.TT_WORD) { tokens.add(st.sval); } } return parseIntoTokens(tokens); } catch (Exception e) { throw new EvaluationException("Unable to parse expression <" + expression + "> into tokens", e); } }
From source file:org.openmrs.reporting.PatientSearch.java
public static PatientSearch createCompositionSearch(String description) { // TODO This is a rewrite of the code in CohortSearchHistory.createCompositionFilter(String). That method should probably delegate to this one in some way. // TODO use open/closeParenthesesWords declared above List<Object> tokens = new ArrayList<Object>(); try {// w w w . j ava 2s .co m StreamTokenizer st = new StreamTokenizer(new StringReader(description)); st.ordinaryChar('('); st.ordinaryChar(')'); while (st.nextToken() != StreamTokenizer.TT_EOF) { if (st.ttype == StreamTokenizer.TT_NUMBER) { Integer thisInt = new Integer((int) st.nval); if (thisInt < 1) { log.error("number < 1"); return null; } tokens.add(thisInt); } else if (st.ttype == '(') { tokens.add("("); } else if (st.ttype == ')') { tokens.add(")"); } else if (st.ttype == StreamTokenizer.TT_WORD) { String str = st.sval.toLowerCase(); tokens.add(str); } } return createCompositionSearch(tokens); } catch (Exception ex) { log.error("Error in description string: " + description, ex); return null; } }
From source file:org.osaf.cosmo.calendar.ICalValueParser.java
/** * Constructs a parser instance for the given EIM text value. *///from w ww. ja v a2s .c o m public ICalValueParser(String text) { tokenizer = new StreamTokenizer(new StringReader(text)); value = text; params = new HashMap<String, String>(); tokenizer.resetSyntax(); tokenizer.wordChars(32, 126); tokenizer.whitespaceChars(0, 20); tokenizer.ordinaryChar(':'); tokenizer.ordinaryChar(';'); tokenizer.ordinaryChar('='); tokenizer.eolIsSignificant(false); tokenizer.whitespaceChars(0, 0); tokenizer.quoteChar('"'); }
From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString//from ww w. j a va2s.c o m * A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus" * @param variableSpace * Context for resolving variable names. If {@code null}, no variable resolution we happen. * @param ignoreSqoopCommand * If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export". * @return List of parsed arguments * @throws IOException * when the command line could not be parsed */ public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace, boolean ignoreSqoopCommand) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments, // e.g. "sqoop <tool>") boolean skipToken = false; // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; if (variableSpace != null) { s = variableSpace.environmentSubstitute(s); } if (ignoreSqoopCommand && args.isEmpty()) { // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of // arguments directly // from a working command line if ("sqoop".equals(s)) { skipToken = true; continue; // skip this one and the next } else if (skipToken) { ignoreSqoopCommand = false; // Don't attempt to ignore any more commands // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing skipToken = false; continue; } } if (s.startsWith(ARG_D)) { handleCustomOption(args, s, tokenizer, variableSpace); continue; } args.add(escapeEscapeSequences(s)); } } } finally { reader.close(); } return args; }
From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString A command line string. * @return List of parsed arguments/*from www .jav a 2 s. co m*/ * @throws IOException when the command line could not be parsed */ public List<String> parseCommandLine(String commandLineString) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; s = environmentSubstitute(s); args.add(s); } } } finally { reader.close(); } return args; }
From source file:org.structr.core.function.Functions.java
public static Object evaluate(final ActionContext actionContext, final GraphObject entity, final String expression) throws FrameworkException { final String expressionWithoutNewlines = expression.replace('\n', ' '); final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(expressionWithoutNewlines)); tokenizer.eolIsSignificant(true);/* w ww .ja va 2 s . c om*/ tokenizer.ordinaryChar('.'); tokenizer.wordChars('_', '_'); tokenizer.wordChars('.', '.'); tokenizer.wordChars('!', '!'); Expression root = new RootExpression(); Expression current = root; Expression next = null; String lastToken = null; int token = 0; int level = 0; while (token != StreamTokenizer.TT_EOF) { token = nextToken(tokenizer); switch (token) { case StreamTokenizer.TT_EOF: break; case StreamTokenizer.TT_EOL: break; case StreamTokenizer.TT_NUMBER: if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before NUMBER"); } next = new ConstantExpression(tokenizer.nval); current.add(next); lastToken += "NUMBER"; break; case StreamTokenizer.TT_WORD: if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before " + tokenizer.sval); } next = checkReservedWords(tokenizer.sval); Expression previousExpression = current.getPrevious(); if (tokenizer.sval.startsWith(".") && previousExpression != null && previousExpression instanceof FunctionExpression && next instanceof ValueExpression) { final FunctionExpression previousFunctionExpression = (FunctionExpression) previousExpression; final ValueExpression valueExpression = (ValueExpression) next; current.replacePrevious( new FunctionValueExpression(previousFunctionExpression, valueExpression)); } else { current.add(next); } lastToken = tokenizer.sval; break; case '(': if (((current == null || current instanceof RootExpression) && next == null) || current == next) { // an additional bracket without a new function, // this can only be an execution group. next = new GroupExpression(); current.add(next); } current = next; lastToken += "("; level++; break; case ')': if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before " + lastToken); } current = current.getParent(); if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken); } lastToken += ")"; level--; break; case '[': // bind directly to the previous expression next = new ArrayExpression(); current.add(next); current = next; lastToken += "["; level++; break; case ']': if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket before " + lastToken); } current = current.getParent(); if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken); } lastToken += "]"; level--; break; case ';': next = null; lastToken += ";"; break; case ',': next = current; lastToken += ","; break; default: if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before " + tokenizer.sval); } current.add(new ConstantExpression(tokenizer.sval)); lastToken = tokenizer.sval; } } if (level > 0) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken); } return root.evaluate(actionContext, entity); }
From source file:org.wso2.andes.server.security.access.config.PlainConfiguration.java
@Override public RuleSet load() throws ConfigurationException { RuleSet ruleSet = super.load(); try {/*w w w.jav a 2 s . c o m*/ _st = new StreamTokenizer(new BufferedReader(new FileReader(_file))); _st.resetSyntax(); // setup the tokenizer _st.commentChar(COMMENT); // single line comments _st.eolIsSignificant(true); // return EOL as a token _st.lowerCaseMode(true); // case insensitive tokens _st.ordinaryChar('='); // equals is a token _st.ordinaryChar(CONTINUATION); // continuation character (when followed by EOL) _st.quoteChar('"'); // double quote _st.quoteChar('\''); // single quote _st.whitespaceChars('\u0000', '\u0020'); // whitespace (to be ignored) TODO properly _st.wordChars('a', 'z'); // unquoted token characters [a-z] _st.wordChars('A', 'Z'); // [A-Z] _st.wordChars('0', '9'); // [0-9] _st.wordChars('_', '_'); // underscore _st.wordChars('-', '-'); // dash _st.wordChars('.', '.'); // dot _st.wordChars('*', '*'); // star _st.wordChars('@', '@'); // at _st.wordChars(':', ':'); // colon // parse the acl file lines Stack<String> stack = new Stack<String>(); int current; do { current = _st.nextToken(); switch (current) { case StreamTokenizer.TT_EOF: case StreamTokenizer.TT_EOL: if (stack.isEmpty()) { break; // blank line } // pull out the first token from the bottom of the stack and check arguments exist String first = stack.firstElement(); stack.removeElementAt(0); if (stack.isEmpty()) { throw new ConfigurationException(String.format(NOT_ENOUGH_TOKENS_MSG, getLine())); } // check for and parse optional initial number for ACL lines Integer number = null; if (StringUtils.isNumeric(first)) { // set the acl number and get the next element number = Integer.valueOf(first); first = stack.firstElement(); stack.removeElementAt(0); } if (StringUtils.equalsIgnoreCase(ACL, first)) { parseAcl(number, stack); } else if (number == null) { if (StringUtils.equalsIgnoreCase(GROUP, first)) { parseGroup(stack); } else if (StringUtils.equalsIgnoreCase(CONFIG, first)) { parseConfig(stack); } else { throw new ConfigurationException( String.format(UNRECOGNISED_INITIAL_MSG, first, getLine())); } } else { throw new ConfigurationException(String.format(NUMBER_NOT_ALLOWED_MSG, first, getLine())); } // reset stack, start next line stack.clear(); break; case StreamTokenizer.TT_NUMBER: stack.push(Integer.toString(Double.valueOf(_st.nval).intValue())); break; case StreamTokenizer.TT_WORD: stack.push(_st.sval); // token break; default: if (_st.ttype == CONTINUATION) { int next = _st.nextToken(); if (next == StreamTokenizer.TT_EOL) { break; // continue reading next line } // invalid location for continuation character (add one to line beacuse we ate the EOL) throw new ConfigurationException(String.format(PREMATURE_CONTINUATION_MSG, getLine() + 1)); } else if (_st.ttype == '\'' || _st.ttype == '"') { stack.push(_st.sval); // quoted token } else { stack.push(Character.toString((char) _st.ttype)); // single character } } } while (current != StreamTokenizer.TT_EOF); if (!stack.isEmpty()) { throw new ConfigurationException(String.format(PREMATURE_EOF_MSG, getLine())); } } catch (IllegalArgumentException iae) { throw new ConfigurationException(String.format(PARSE_TOKEN_FAILED_MSG, getLine()), iae); } catch (FileNotFoundException fnfe) { throw new ConfigurationException(String.format(CONFIG_NOT_FOUND_MSG, getFile().getName()), fnfe); } catch (IOException ioe) { throw new ConfigurationException(String.format(CANNOT_LOAD_MSG, getFile().getName()), ioe); } return ruleSet; }
From source file:org.xwiki.extension.internal.ExtensionUtils.java
/** * @param str the String to parse/* w ww . j av a 2 s . co m*/ * @param trim true if the passed String should be trimmed * @return the collection of Strings extracted from the passed String * @since 8.3M1 */ public static List<String> importPropertyStringList(String str, boolean trim) { try { String cleanedString = str; // Trim if (trim) { cleanedString = cleanedString.trim(); } // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters st.whitespaceChars(',', ','); st.whitespaceChars(' ', ' '); st.whitespaceChars('\t', '\t'); st.whitespaceChars('\n', '\n'); st.whitespaceChars('\r', '\r'); // Split comma-delimited tokens into a List List<String> collection = new ArrayList<>(); while (true) { int ttype = st.nextToken(); if (ttype == StreamTokenizer.TT_WORD || ttype > 0) { if (st.sval != null) { collection.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype + " parsing elements."); } } // Return the completed list return collection; } catch (Exception e) { // Log ? } return Collections.emptyList(); }
From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java
/** * Create and initializer a {@link StreamTokenizer} to parse the value. * /*w ww .j a v a 2s.c o m*/ * @param value the string to parse * @return the {@link StreamTokenizer} used to parse the string */ protected StreamTokenizer createStreamTokenizer(String value) { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters for (char c : getDelimiters().toCharArray()) { st.whitespaceChars(c, c); } return st; }
From source file:org.yestech.jmlnitrate.transformation.inbound.BaseInboundTransformation.java
/** * Parses a Parameter Request String and return a array of the tokens in the * string. The default delimeters are ^ and ; * * @param rawRequest the Parameter Request * @return Array of the tokens/*from w w w. j av a 2 s. c o m*/ * @throws Exception if an error happens */ private Object[] parseParameterRequest(String rawRequest) throws Exception { StreamTokenizer tokenStream = new StreamTokenizer( new BufferedReader(new StringReader(rawRequest), BUFFER_SIZE)); //reset the stream tokenStream.resetSyntax(); //configure tokens tokenStream.lowerCaseMode(false); tokenStream.eolIsSignificant(false); //add word chars tokenStream.wordChars(32, 58); tokenStream.wordChars(60, 93); tokenStream.wordChars(95, 126); //add <CR>\r <LF>\n tokenStream.wordChars(10, 10); tokenStream.wordChars(13, 13); //set ^ AND ; as delimeters to string tokens tokenStream.quoteChar(94); //removed ; //tokenStream.quoteChar(59); //set up the temp arraylist ArrayList tokens = new ArrayList(); int result = tokenStream.ttype; while (result != StreamTokenizer.TT_EOF) { result = tokenStream.nextToken(); switch (result) { case StreamTokenizer.TT_EOF: break; default: tokens.add(tokenStream.sval); break; } } return tokens.toArray(); }