Example usage for org.apache.commons.lang3.text StrTokenizer StrTokenizer

Introduction

In this page you can find the example usage for org.apache.commons.lang3.text StrTokenizer StrTokenizer.

Prototype

public StrTokenizer(final char[] input, final StrMatcher delim, final StrMatcher quote) 

Source Link

Document

Constructs a tokenizer splitting using the specified delimiter matcher and handling quotes using the specified quote matcher.

Usage

From source file:io.cloudslang.content.utilities.util.ProcessExecutor.java

private List<String> processCommand(String commandLine) {
    List<String> command = new ArrayList<>(arguments);
    if (!isEmpty(commandLine)) {
        command.addAll(new StrTokenizer(commandLine, ',', '"').getTokenList());
    }//from   w  w  w. j av a 2 s  .  c om

    return command;
}

---

From source file:edu.sabanciuniv.sentilab.sare.controllers.opinion.OpinionCorpusFactory.java

@Override
protected OpinionCorpusFactory addTextPacket(OpinionCorpus corpus, InputStream input, String delimiter)
        throws IOException {

    Validate.notNull(corpus, CannedMessages.NULL_ARGUMENT, "corpus");
    Validate.notNull(input, CannedMessages.NULL_ARGUMENT, "input");

    OpinionDocumentFactory opinionFactory = null;
    BufferedReader reader = new BufferedReader(new InputStreamReader(input));
    String line;/* ww w . j a v a  2  s.  c o  m*/

    while ((line = reader.readLine()) != null) {
        StrTokenizer tokenizer = new StrTokenizer(line, StrMatcher.stringMatcher(delimiter),
                StrMatcher.quoteMatcher());
        List<String> columns = tokenizer.getTokenList();
        if (columns.size() < 1) {
            continue;
        }

        opinionFactory = new OpinionDocumentFactory().setCorpus(corpus).setContent(columns.get(0));

        if (columns.size() > 1) {
            try {
                opinionFactory.setPolarity(Double.parseDouble(columns.get(1)));
            } catch (NumberFormatException e) {
                opinionFactory.setPolarity(null);
            }
        }

        corpus.addDocument(opinionFactory.create());
    }

    return this;
}

---

From source file:edu.sabanciuniv.sentilab.sare.controllers.aspect.AspectLexiconFactory.java

@Override
protected AspectLexiconFactory addTextPacket(AspectLexicon lexicon, InputStream input, String delimiter)
        throws IOException {
    Validate.notNull(lexicon, CannedMessages.NULL_ARGUMENT, "lexicon");
    Validate.notNull(input, CannedMessages.NULL_ARGUMENT, "input");

    delimiter = StringUtils.defaultString(delimiter, "\t");

    BufferedReader reader = new BufferedReader(new InputStreamReader(input));
    String line;//from  ww w . j  a v a2s .  c o  m

    while ((line = reader.readLine()) != null) {
        StrTokenizer tokenizer = new StrTokenizer(line, StrMatcher.stringMatcher(delimiter),
                StrMatcher.quoteMatcher());
        List<String> columns = tokenizer.getTokenList();
        if (columns.size() < 1) {
            continue;
        }

        String aspectStr = columns.get(0);
        Matcher matcher = Pattern.compile("^<(.*)>$").matcher(aspectStr);
        if (matcher.matches()) {
            aspectStr = matcher.group(1);
        } else {
            continue;
        }

        AspectLexicon aspect = lexicon.addAspect(aspectStr);
        for (int i = 1; i < columns.size(); i++) {
            aspect.addExpression(columns.get(i));
        }
    }

    return this;
}

---

From source file:com.jkoolcloud.tnt4j.streams.parsers.ActivityTokenParser.java

@Override
protected ActivityContext prepareItem(TNTInputStream<?, ?> stream, Object data) throws ParseException {
    // Get next string to parse
    String dataStr = getNextActivityString(data);
    if (StringUtils.isEmpty(dataStr)) {
        return null;
    }/*from   www.jav a2 s. c om*/
    logger().log(OpLevel.DEBUG, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
            "ActivityParser.splitting.string"), dataStr);
    if (pattern != null) {
        Matcher matcher = pattern.matcher(dataStr);
        if (matcher == null || !matcher.matches()) {
            logger().log(OpLevel.DEBUG, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
                    "ActivityParser.input.not.match"), getName(), pattern.pattern());
            return null;
        }
    }
    StrTokenizer tk = stripQuotes ? new StrTokenizer(dataStr, fieldDelim, StrMatcher.doubleQuoteMatcher())
            : new StrTokenizer(dataStr, fieldDelim);
    tk.setIgnoreEmptyTokens(false);
    String[] fields = tk.getTokenArray();
    if (ArrayUtils.isEmpty(fields)) {
        logger().log(OpLevel.DEBUG,
                StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "ActivityParser.no.fields"));
        return null;
    }
    logger().log(OpLevel.DEBUG,
            StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "ActivityParser.split"),
            fields.length);

    ActivityContext cData = new ActivityContext(stream, data, fields);
    cData.setMessage(getRawDataAsMessage(fields));

    return cData;
}

---

From source file:com.jkoolcloud.tnt4j.streams.parsers.ActivityNameValueParser.java

@Override
protected ActivityContext prepareItem(TNTInputStream<?, ?> stream, Object data) throws ParseException {
    String dataStr = getNextActivityString(data);
    if (StringUtils.isEmpty(dataStr)) {
        return null;
    }/*from   ww w .  ja v a2s  .  c om*/
    logger().log(OpLevel.DEBUG, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
            "ActivityParser.splitting.string"), dataStr);
    if (pattern != null) {
        Matcher matcher = pattern.matcher(dataStr);
        if (matcher == null || !matcher.matches()) {
            logger().log(OpLevel.DEBUG, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
                    "ActivityParser.input.not.match"), getName(), pattern.pattern());
            return null;
        }
    }
    StrTokenizer tk = stripQuotes ? new StrTokenizer(dataStr, fieldDelim, StrMatcher.doubleQuoteMatcher())
            : new StrTokenizer(dataStr, fieldDelim);
    tk.setIgnoreEmptyTokens(false);
    String[] fields = tk.getTokenArray();
    if (ArrayUtils.isEmpty(fields)) {
        logger().log(OpLevel.DEBUG,
                StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "ActivityParser.no.fields"));
        return null;
    }
    logger().log(OpLevel.DEBUG,
            StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "ActivityParser.split"),
            fields.length);
    Map<String, String> nameValues = new HashMap<>(fields.length);
    for (String field : fields) {
        if (field != null) {
            String[] nv = field.split(Pattern.quote(valueDelim));
            if (ArrayUtils.isNotEmpty(nv)) {
                nameValues.put(nv[0], nv.length > 1 ? nv[1].trim() : "");
            }
            logger().log(OpLevel.TRACE, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
                    "ActivityNameValueParser.found"), field);
        }
    }

    ActivityContext cData = new ActivityContext(stream, data, nameValues);
    cData.setMessage(getRawDataAsMessage(nameValues));

    return cData;
}

---

From source file:org.apache.hadoop.hive.metastore.tools.TestSchemaToolCatalogOps.java

private static void execute(SchemaToolTask task, String taskArgs) throws HiveMetaException {
    try {/*from  www . j av a 2 s  .  c o m*/
        StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"');
        SchemaToolCommandLine cl = new SchemaToolCommandLine(tokenizer.getTokenArray(), null);
        task.setCommandLineArguments(cl);
    } catch (Exception e) {
        throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e);
    }

    task.setHiveSchemaTool(schemaTool);
    task.execute();
}

---

From source file:org.kalypso.model.wspm.core.profil.sobek.parser.SobekLineParser.java

public SobekLineParser(final LineNumberReader reader) throws IOException, CoreException {
    final String line = reader.readLine();
    if (line == null)
        throw SobekParsing.throwError(format(Messages.getString("SobekLineParser_0"))); //$NON-NLS-1$

    m_tokenizer = new StrTokenizer(line, StrMatcher.spaceMatcher(), StrMatcher.singleQuoteMatcher());
    m_lineNumber = reader.getLineNumber();
}

---

From source file:org.omnaest.utils.strings.StringUtils.java

/**
 * Simple form of the {@link StrTokenizer}
 * /*from   w  ww  . j  ava2  s .  co  m*/
 * @param text
 * @param delimiter
 * @param quote
 * @return
 */
public static String[] split(String text, char delimiter, char quote) {
    return new StrTokenizer(text, delimiter, quote).getTokenArray();
}

---

From source file:org.squashtest.tm.service.internal.advancedsearch.AdvancedSearchServiceImpl.java

private List<String> parseInput(String textInput) {
    return new StrTokenizer(textInput, StrMatcher.trimMatcher(), StrMatcher.doubleQuoteMatcher())
            .getTokenList();/*  w  w  w.  j  a  v a 2s.c o m*/
}

---