Example usage for org.apache.commons.lang3.text StrTokenizer getTokenList

Introduction

In this page you can find the example usage for org.apache.commons.lang3.text StrTokenizer getTokenList.

Prototype

public List<String> getTokenList() 

Source Link

Document

Gets a copy of the full token list as an independent modifiable list.

Usage

From source file:com.stratelia.silverpeas.silverstatistics.control.SilverStatisticsService.java

/**
 * @param type//w  w w.j av a 2 s.  c o m
 * @param data
 */
@Override
public void putStats(StatType type, String data) {
    StrTokenizer stData = new StrTokenizer(data, SEPARATOR);
    List<String> dataArray = stData.getTokenList();
    if (myStatsConfig.isGoodDatas(type, dataArray)) {
        Connection myCon = DBUtil.makeConnection(SILVERSTATISTICS_DATASOURCE);
        try {
            SilverStatisticsDAO.putDataStats(myCon, type, dataArray, myStatsConfig);
            if (!myCon.getAutoCommit()) {
                myCon.commit();
            }
        } catch (SQLException e) {
            SilverTrace.error("silverstatistics", "SilverStatisticsEJB.putStats",
                    "silverstatistics.MSG_ALIMENTATION_BD",
                    "typeOfStats = " + type + ", dataArray = " + dataArray, e);
        } catch (StatisticsRuntimeException e) {
            SilverTrace.error("silverstatistics", "SilverStatisticsEJB.putStats", "MSG_CONNECTION_BD");
        } finally {
            DBUtil.close(myCon);
        }

    } else {
        SilverTrace.error("silverstatistics", "SilverStatisticsEJB.putStats", "MSG_CONFIG_DATAS",
                "data en entree=" + data + " pour " + type);
    }
}

---

From source file:com.digitalgeneralists.assurance.model.entities.ApplicationConfiguration.java

private List<String> transformStringPropertyToListProperty(String property, boolean removeExtensionPatterns) {
    StrTokenizer tokenizer = new StrTokenizer(property, ',');
    List<String> tokenizedList = tokenizer.getTokenList();
    // NOTE: May want to reconsider the toLowercase conversion.
    ListIterator<String> iterator = tokenizedList.listIterator();
    while (iterator.hasNext()) {
        String extension = iterator.next();
        // NOTE: This is probably a bit overly-aggressive and less-sophisticated than it could be,
        // but it should handle 99% of the input that will be entered.
        if (removeExtensionPatterns) {
            extension = StringUtils.replace(extension, "*.", "");
            extension = StringUtils.replace(extension, "*", "");
            extension = StringUtils.replace(extension, ".", "");
        }//from  w ww.  j a  v  a2 s  .  c o  m
        iterator.set(extension.toLowerCase().trim());
    }
    return tokenizedList;
}

---

From source file:edu.sabanciuniv.sentilab.sare.controllers.opinion.OpinionCorpusFactory.java

@Override
protected OpinionCorpusFactory addTextPacket(OpinionCorpus corpus, InputStream input, String delimiter)
        throws IOException {

    Validate.notNull(corpus, CannedMessages.NULL_ARGUMENT, "corpus");
    Validate.notNull(input, CannedMessages.NULL_ARGUMENT, "input");

    OpinionDocumentFactory opinionFactory = null;
    BufferedReader reader = new BufferedReader(new InputStreamReader(input));
    String line;/* ww w  .  j  a  v a2 s  .co  m*/

    while ((line = reader.readLine()) != null) {
        StrTokenizer tokenizer = new StrTokenizer(line, StrMatcher.stringMatcher(delimiter),
                StrMatcher.quoteMatcher());
        List<String> columns = tokenizer.getTokenList();
        if (columns.size() < 1) {
            continue;
        }

        opinionFactory = new OpinionDocumentFactory().setCorpus(corpus).setContent(columns.get(0));

        if (columns.size() > 1) {
            try {
                opinionFactory.setPolarity(Double.parseDouble(columns.get(1)));
            } catch (NumberFormatException e) {
                opinionFactory.setPolarity(null);
            }
        }

        corpus.addDocument(opinionFactory.create());
    }

    return this;
}

---

From source file:com.haulmont.cuba.web.testsupport.TestContainer.java

protected void initAppContext() {
    EclipseLinkCustomizer.initTransientCompatibleAnnotations();

    String configProperty = AppContext.getProperty(AbstractAppContextLoader.SPRING_CONTEXT_CONFIG);

    StrTokenizer tokenizer = new StrTokenizer(configProperty);
    List<String> locations = tokenizer.getTokenList();
    locations.add(getSpringConfig());// ww w  . j  av a 2s  .com

    springAppContext = new CubaClassPathXmlApplicationContext(locations.toArray(new String[locations.size()]));
    AppContext.Internals.setApplicationContext(springAppContext);

    Events events = AppBeans.get(Events.NAME);
    events.publish(new AppContextInitializedEvent(springAppContext));
}

---

From source file:edu.sabanciuniv.sentilab.sare.controllers.aspect.AspectLexiconFactory.java

@Override
protected AspectLexiconFactory addTextPacket(AspectLexicon lexicon, InputStream input, String delimiter)
        throws IOException {
    Validate.notNull(lexicon, CannedMessages.NULL_ARGUMENT, "lexicon");
    Validate.notNull(input, CannedMessages.NULL_ARGUMENT, "input");

    delimiter = StringUtils.defaultString(delimiter, "\t");

    BufferedReader reader = new BufferedReader(new InputStreamReader(input));
    String line;/*from  w  ww  .  ja  v  a 2  s.  co m*/

    while ((line = reader.readLine()) != null) {
        StrTokenizer tokenizer = new StrTokenizer(line, StrMatcher.stringMatcher(delimiter),
                StrMatcher.quoteMatcher());
        List<String> columns = tokenizer.getTokenList();
        if (columns.size() < 1) {
            continue;
        }

        String aspectStr = columns.get(0);
        Matcher matcher = Pattern.compile("^<(.*)>$").matcher(aspectStr);
        if (matcher.matches()) {
            aspectStr = matcher.group(1);
        } else {
            continue;
        }

        AspectLexicon aspect = lexicon.addAspect(aspectStr);
        for (int i = 1; i < columns.size(); i++) {
            aspect.addExpression(columns.get(i));
        }
    }

    return this;
}

---

From source file:com.mgmtp.perfload.perfalyzer.reporting.email.EmailReporter.java

private List<? extends List<String>> loadData(final File file) throws IOException {
    try (BufferedReader br = newReader(file, Charsets.UTF_8)) {
        List<List<String>> rows = newArrayList();
        StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
        tokenizer.setDelimiterChar(DELIMITER);

        for (String line; (line = br.readLine()) != null;) {
            tokenizer.reset(line);//  w w w. ja v  a 2  s.c  o  m
            List<String> tokenList = tokenizer.getTokenList();
            rows.add(tokenList);
        }

        return rows;
    }
}

---

From source file:com.mgmtp.perfload.perfalyzer.util.MarkersReader.java

public List<Marker> readMarkers() throws IOException {
    Map<String, Marker> markers = new LinkedHashMap<>();

    StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
    tokenizer.setDelimiterChar(DELIMITER);

    try (FileInputStream fis = new FileInputStream(inputFile)) {

        for (Scanner scanner = new Scanner(fis.getChannel()); scanner.hasNext();) {
            String line = scanner.nextLine();
            if (line.startsWith("#")) {
                continue;
            }/*  w ww  . j  av a 2 s  .co  m*/

            tokenizer.reset(line);

            List<String> tokenList = tokenizer.getTokenList();

            if (MARKER.equals(tokenList.get(COL_MARKER))) {
                // no whitespace allowed in marker in order to avoid issues in HTML
                String markerName = tokenList.get(COL_MARKER_NAME).replaceAll("\\s+", "_");
                String markerType = tokenList.get(COL_MARKER_TYPE);
                long timeMillis = Long.parseLong(tokenList.get(COL_TIMESTAMP));

                switch (markerType) {
                case MARKER_LEFT: {
                    Marker marker = new Marker(markerName);
                    markers.put(markerName, marker);
                    marker.setLeftMillis(timeMillis);
                    break;
                }
                case MARKER_RIGHT: {
                    Marker marker = markers.get(markerName);
                    marker.setRightMillis(timeMillis);
                    break;
                }
                default:
                    throw new IllegalStateException("Invalid marker type: " + markerType);
                }
            }
        }

        return markers.values().stream().map(marker -> {
            marker.calculateDateTimeFields(testStart);
            return marker;
        }).collect(toList());
    }
}

---

From source file:com.mgmtp.perfload.perfalyzer.util.BinnedFilesMerger.java

public void mergeFiles() throws IOException {
    if (!inputDir.isDirectory()) {
        throw new IllegalArgumentException("The input File must be a directory");
    }/*from  w  ww  .j  ava 2  s . co m*/

    StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
    tokenizer.setDelimiterChar(DELIMITER);
    Map<String, FileChannel> destChannels = newHashMap();
    List<OutputStream> outputStreams = newArrayList();
    File[] filesInInputDirectory = inputDir.listFiles();

    try {
        for (File file : filesInInputDirectory) {
            FileInputStream fis = null;
            try {
                fis = new FileInputStream(file);
                for (Scanner scanner = new Scanner(fis.getChannel(), Charsets.UTF_8.name()); scanner
                        .hasNext();) {
                    String line = scanner.nextLine();
                    tokenizer.reset(line);

                    List<String> tokenList = tokenizer.getTokenList();
                    String key = tokenList.get(sortCriteriaColumn);
                    FileChannel destChannel = destChannels.get(key);
                    if (destChannel == null) {
                        FileOutputStream fos = new FileOutputStream(
                                new File(outputDir, FILE_TYPE + "_" + key + ".out"));
                        outputStreams.add(fos);
                        destChannel = fos.getChannel();
                        destChannels.put(key, destChannel);

                        //Write the Header...... Has to be improved
                        IoUtilities.writeLineToChannel(destChannel, getHeader(), Charsets.UTF_8);
                    }

                    StrBuilder outputLine = new StrBuilder();
                    for (String s : tokenList) {
                        StrBuilderUtils.appendEscapedAndQuoted(outputLine, DELIMITER, s);
                    }
                    IoUtilities.writeLineToChannel(destChannel, outputLine.toString(), Charsets.UTF_8);
                }
            } finally {
                closeQuietly(fis);
            }
        }
    } finally {
        outputStreams.forEach(IOUtils::closeQuietly);
    }

}

---

From source file:net.longfalcon.newsj.Releases.java

public List<String> getReleaseNameSearchTokens(String releaseName) {
    StrTokenizer strTokenizer = new StrTokenizer(releaseName, StrMatcher.charSetMatcher('.', '_'));
    List<String> tokenList = strTokenizer.getTokenList();

    return tokenList.stream().filter(s -> s.length() > 2).limit(2).collect(Collectors.toList());
}

---

From source file:org.apache.stratos.adc.mgt.cli.completer.CommandCompleter.java

@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
    if (logger.isTraceEnabled()) {
        logger.trace("Buffer: {}, cursor: {}", buffer, cursor);
        logger.trace("Candidates {}", candidates);
    }//from  w  w w  .j  av a2  s .  com
    if (StringUtils.isNotBlank(buffer)) {
        // User is typing a command
        StrTokenizer strTokenizer = new StrTokenizer(buffer);
        String action = strTokenizer.next();
        Collection<String> arguments = argumentMap.get(action);
        if (arguments != null) {
            if (logger.isTraceEnabled()) {
                logger.trace("Arguments found for {}, Tokens: {}", action, strTokenizer.getTokenList());
                logger.trace("Arguments for {}: {}", action, arguments);
            }
            List<String> args = new ArrayList<String>(arguments);
            List<Completer> completers = new ArrayList<Completer>();
            for (String token : strTokenizer.getTokenList()) {
                boolean argContains = arguments.contains(token);
                if (token.startsWith("-") && !argContains) {
                    continue;
                }
                if (argContains) {
                    if (logger.isTraceEnabled()) {
                        logger.trace("Removing argument {}", token);
                    }
                    args.remove(token);
                }
                completers.add(new StringsCompleter(token));
            }
            completers.add(new StringsCompleter(args));
            Completer completer = new ArgumentCompleter(completers);
            return completer.complete(buffer, cursor, candidates);
        } else if (CliConstants.HELP_ACTION.equals(action)) {
            // For help action, we need to display available commands as arguments
            return helpCommandCompleter.complete(buffer, cursor, candidates);
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Using Default Completer...");
    }
    return defaultCommandCompleter.complete(buffer, cursor, candidates);
}

---