Example usage for org.apache.commons.lang3.text StrTokenizer getTokenArray

List of usage examples for org.apache.commons.lang3.text StrTokenizer getTokenArray

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.lang3.text.*
  5. StrTokenizer
  6. getTokenArray

Introduction

In this page you can find the example usage for org.apache.commons.lang3.text StrTokenizer getTokenArray.

Prototype

public String[] getTokenArray()

Source Link

Document

Gets a copy of the full token list as an independent modifiable array.

Usage

From source file:de.vandermeer.asciitable.commons.ArrayTransformations.java

/**
 * Takes an object (used as a string) and returns a string array with all processed lines.
 * The process is as follows://from  www.  j a v  a2 s.  c o  m
 * (1) replace all line breaks (CR LF, CR, LF) into HTML4 line break entity (<br>).
 * (2) replace all HTML4 line break entities to HTML5 entities (as in self-closing <br/> entity).
 * (3) use a tokenizer to process the resulting string (not ignoring empty tokens, since they mark required line breaks).
 * (4) return the array of the tokenizer
 * 
 * As a result, a string containing 1 line break will be converted into 2 paragraphs (array length 2):
 * <pre>{@code
String: "paragraph 1\nparagraph 2"
Array:  {paragraph 1,paragraph 2}
 * }</pre>
 * A string containing 2 line breaks will be converted into 3 strings (first paragraph, additional line break, second paragraph):
 * <pre>{@code
String: "paragraph 1\n\nparagraph 2"
Array: {paragraph 1,,paragraph 2}
 * }</pre>
 * 
 * @param content the content to process
 * @return null if content was null, empty array if content was an empty string, string array with lines otherwise
 */
public static final String[] PROCESS_CONTENT(Object content) {
    if (content == null || content.toString() == null) {
        return null;
    }
    if ("".equals(content)) {
        return new String[] { "" };
    }

    String lfRep = StringUtils.replacePattern(content.toString(), "\\r\\n|\\r|\\n", "<br>");
    lfRep = StringUtils.replace(lfRep, "<br>", "<br/>");
    StrTokenizer tok = new StrTokenizer(lfRep, "<br/>").setIgnoreEmptyTokens(false);
    return tok.getTokenArray();
}

From source file:com.mgmtp.perfload.perfalyzer.util.PerfAlyzerUtils.java

public static Map<String, String> readAggregatedMap(final File executionsFile, final Charset charset)
        throws IOException {
    final StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
    tokenizer.setDelimiterChar(';');

    Map<String, String> result = newHashMapWithExpectedSize(11);

    List<String> lines = Files.readLines(executionsFile, charset);
    String[] headers = null;// ww  w.  ja va  2  s . co  m

    for (String line : lines) {
        tokenizer.reset(line);
        String[] tokens = tokenizer.getTokenArray();

        if (headers == null) {
            headers = tokens;
        } else {

            String[] data = tokenizer.getTokenArray();

            String operation = data[0];
            for (int i = 1; i < headers.length; ++i) {
                result.put(operation + "." + headers[i], data[i]);
            }
        }
    }

    return result;
}

From source file:com.mgmtp.perfload.perfalyzer.util.PerfAlyzerUtils.java

/**
 * Reads a semicolon-delimited CSV file into a list. Each line in the result list will be
 * another list of {@link Number} objects. The file is expected to have two numberic columns
 * which are parsed using the specified number format.
 * /*from  w  w  w.j  av  a  2s  .c om*/
 * @param file
 *            the file
 * @param charset
 *            the character set to read the file
 * @param numberFormat
 *            the number format for parsing the column values
 * @return the immutable result list
 */
public static List<SeriesPoint> readDataFile(final File file, final Charset charset,
        final NumberFormat numberFormat) throws IOException {
    final StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
    tokenizer.setDelimiterChar(';');

    try (BufferedReader br = newReader(file, charset)) {
        boolean headerLine = true;
        List<SeriesPoint> result = newArrayListWithExpectedSize(200);

        for (String line; (line = br.readLine()) != null;) {
            try {
                if (headerLine) {
                    headerLine = false;
                } else {
                    tokenizer.reset(line);
                    String[] tokens = tokenizer.getTokenArray();
                    double x = numberFormat.parse(tokens[0]).doubleValue();
                    double y = numberFormat.parse(tokens[1]).doubleValue();

                    if (!result.isEmpty()) {
                        // additional point for histogram
                        SeriesPoint previousPoint = getLast(result);
                        result.add(new SeriesPoint(x, previousPoint.getY()));
                    }
                    tokenizer.reset(line);
                    result.add(new SeriesPoint(x, y));
                }
            } catch (ParseException ex) {
                throw new IOException("Error parsing number in file: " + file, ex);
            }
        }

        int size = result.size();
        if (size > 2) {
            // additional point at end for histogram
            SeriesPoint nextToLast = result.get(size - 3);
            SeriesPoint last = result.get(size - 1);
            double dX = last.getX().doubleValue() - nextToLast.getX().doubleValue();
            result.add(new SeriesPoint(last.getX().doubleValue() + dX, last.getY()));
        }
        return ImmutableList.copyOf(result);
    }
}

From source file:com.mgmtp.perfload.perfalyzer.util.PerfAlyzerUtils.java

/**
 * Reads a semicolon-delimited CSV file into a map of lists series values. Values for each
 * column are return as list of lists in the map, the key being the column header.
 * /*from  www.j av  a  2  s  .c om*/
 * @param file
 *            the file
 * @param charset
 *            the character set to read the file
 * @param numberFormat
 *            the number format for formatting the column values
 * @param columnNames
 *            the columns to consider
 * 
 * @return an immutable map of lists of series values
 */
public static Map<String, List<SeriesPoint>> readDataFile(final File file, final Charset charset,
        final NumberFormat numberFormat, final Set<String> columnNames) throws IOException {
    final StrTokenizer tokenizer = StrTokenizer.getCSVInstance();
    tokenizer.setDelimiterChar(';');

    return readLines(file, charset, new LineProcessor<Map<String, List<SeriesPoint>>>() {
        private String[] headers;
        private final Map<String, List<SeriesPoint>> result = newHashMapWithExpectedSize(4);
        private int colCount;

        @Override
        public boolean processLine(final String line) throws IOException {
            try {
                tokenizer.reset(line);
                String[] tokens = tokenizer.getTokenArray();

                if (headers == null) {
                    headers = tokens;
                    colCount = tokens.length;
                } else {
                    Integer counter = Integer.valueOf(tokens[0]);
                    for (int i = 1; i < colCount; ++i) {
                        String header = headers[i];
                        if (columnNames.contains(header)) {
                            List<SeriesPoint> colValues = result.get(header);
                            if (colValues == null) {
                                colValues = newArrayListWithExpectedSize(50);
                                result.put(header, colValues);
                            }
                            colValues.add(new SeriesPoint(counter, numberFormat.parse(tokens[i])));
                        }
                    }
                }
                return true;
            } catch (ParseException ex) {
                throw new IOException("Error parsing number in file: " + file, ex);
            }
        }

        @Override
        public Map<String, List<SeriesPoint>> getResult() {
            return ImmutableMap.copyOf(result);
        }
    });
}

From source file:com.mmone.gpdati.allotment.GpDatiDispoRecord.java

private void detokenizeValues(String value) {
    // '1|1|13|2016-10-27|2016-10-28'
    StrTokenizer tokenizer = new StrTokenizer(key, "|");
    String[] ta = tokenizer.getTokenArray();

    structureId = ta[0];//  w  w w . j a v a2 s .  c  o m
    roomId = ta[1];
    allotment = ta[2];
    dateFrom = ta[3];
    dateTo = ta[4];

}

From source file:de.vandermeer.skb.interfaces.transformers.String_To_ConditionalBreak.java

/**
 * Transforms a String to a String[] processing conditional line breaks.
 * Conditional line breaks are CR LF, CR, LF, &lt;br&gt;, and &lt;br/&gt;.
 * //www  . j a va2  s. com
 * The method proceeds as follows:
 * 
 *     . replace all line breaks (CR LF, CR, LF) into HTML4 line break entity &lt;br&gt;
 *     . replace all HTML4 line break entities to HTML5 entities (as in self-closing &lt;br/&gt; entity).
 *     . use a `tokenizer` to process the resulting string (not ignoring empty tokens, since they mark required line breaks).
 *     . return the array of the `tokenizer`
 * 
 * As a result, a string containing 1 line break will be converted into an array length 2:
 * ----
 * String: "paragraph 1\nparagraph 2"
 * Array:  {paragraph 1,paragraph 2}
 * ----
 * 
 * A string containing 2 line breaks will be converted into a string array with 3 entries (first paragraph, additional line break, second paragraph):
 * ----
 * String: "paragraph 1\n\nparagraph 2"
 * Array: {paragraph 1,,paragraph 2}
 * ----
 * 
 * @param s input string
 * @return array with conditional line breaks converted to empty entries, `null` if `s` was `null`
 */
@Override
default String[] transform(String s) {
    IsTransformer.super.transform(s);
    if ("".equals(s)) {
        return new String[] { "" };
    }

    String lfRep = StringUtils.replacePattern(s.toString(), "\\r\\n|\\r|\\n", "<br>");
    lfRep = StringUtils.replace(lfRep, "<br>", "<br/>");
    lfRep = StringUtils.replace(lfRep, "<br/>", "<br />");
    StrTokenizer tok = new StrTokenizer(lfRep, "<br />").setIgnoreEmptyTokens(false);
    return tok.getTokenArray();
}

From source file:com.mgmtp.jfunk.core.util.CsvDataProcessor.java

private List<Column> initColumns(final StrTokenizer st, final String headerLine) {
    st.reset(headerLine);/* ww w . j a v a2 s.com*/

    String[] headers = st.getTokenArray();
    List<Column> columns = newArrayListWithCapacity(headers.length);
    for (String header : headers) {
        columns.add(new Column(header));
    }
    return columns;
}

From source file:com.bekwam.resignator.commands.KeytoolCommand.java

public List<KeystoreEntry> parseKeystoreEntries(BufferedReader br) throws IOException {

    List<KeystoreEntry> entries = new ArrayList<>();

    String line = "";
    KeystoreListParseStateType parseState = KeystoreListParseStateType.HEADER;
    KeystoreEntry currEntry = null;// w w  w.j a  v a2  s  .  c o  m

    while (parseState != KeystoreListParseStateType.END && parseState != KeystoreListParseStateType.ERROR_END) {

        switch (parseState) {
        case START:
            parseState = KeystoreListParseStateType.HEADER;
            break;
        case HEADER:

            line = br.readLine();

            if (StringUtils.startsWith(line, "Your keystore")) {
                parseState = KeystoreListParseStateType.ENTRY;
            } else if (line == null) {
                parseState = KeystoreListParseStateType.ERROR_END;
            }

            break;
        case ENTRY:

            line = br.readLine();

            if (StringUtils.startsWith(line, "Certificate fingerprint")) {
                parseState = KeystoreListParseStateType.FP;
            } else {

                if (line == null) {

                    parseState = KeystoreListParseStateType.ERROR_END;

                } else if (StringUtils.isNotEmpty(line)) {
                    String[] toks = StringUtils.split(line, ",");
                    currEntry = new KeystoreEntry(StringUtils.trim(toks[0]),
                            StringUtils.trim(toks[1] + toks[2]), StringUtils.trim(toks[3]));
                }
            }

            break;
        case FP:

            if (StringUtils.isNotEmpty(line)) {
                StrTokenizer st = new StrTokenizer(line, ": ");
                if (st != null) {
                    String[] toks = st.getTokenArray();
                    currEntry = new KeystoreEntry(currEntry, toks[1]);
                    entries.add(currEntry);
                }
            }

            parseState = KeystoreListParseStateType.ENTRY;
            break;

        case ERROR_END:
        case END:
            break;
        }
    }

    return entries;
}

From source file:de.vandermeer.skb.interfaces.transformers.textformat.Text_To_WrappedFormat.java

@Override
default Pair<ArrayList<String>, ArrayList<String>> transform(String input) {
    Validate.notBlank(input);//  www  . ja v  a 2  s . c  om
    Validate.isTrue(this.getWidth() > 0);

    ArrayList<String> topList = new ArrayList<>();
    ArrayList<String> bottomList = new ArrayList<>();

    //an emergency break, counting loops to avoid endless loops
    int count;

    String text = StringUtils.replacePattern(input, "\\r\\n|\\r|\\n", LINEBREAK);
    text = StringUtils.replace(text, "<br>", LINEBREAK);
    text = StringUtils.replace(text, "<br/>", LINEBREAK);

    StrBuilder sb = new StrBuilder(text);
    if (this.getTopSettings() != null) {
        //we have a top request, do that one first
        Validate.notNull(this.getTopSettings().getLeft());
        Validate.notNull(this.getTopSettings().getRight());
        Validate.isTrue(this.getTopSettings().getLeft() > 0);
        Validate.isTrue(this.getTopSettings().getRight() > 0);

        int topLines = this.getTopSettings().getLeft();
        int topWidth = this.getTopSettings().getRight();
        count = 0;

        while (sb.size() > 0 && topLines > 0 && count++ < 200) {
            if (sb.startsWith(LINEBREAK)) {
                sb.replaceFirst(LINEBREAK, "");
            }
            String s = null;
            boolean wln = false;
            if (sb.indexOf(LINEBREAK) > 0) {
                s = sb.substring(0, sb.indexOf(LINEBREAK));
                wln = true;
                //sb.replace(0, sb.indexOf(LINEBREAK) + LINEBREAK.length(), "");
            } else {
                s = sb.toString();
                //sb.clear();
            }
            String wrap = WordUtils.wrap(s, topWidth, LINEBREAK, true);
            StrTokenizer tok = new StrTokenizer(wrap, LINEBREAK).setIgnoreEmptyTokens(false);
            String[] ar = tok.getTokenArray();
            if (ar.length <= topLines) {
                //all lines done, cleanup
                for (String str : ar) {
                    topList.add(str.trim());
                }
                if (wln == true) {
                    //if we had a conditional linebreak there might be more text, remove the line we processed
                    sb.replace(0, sb.indexOf(LINEBREAK) + LINEBREAK.length(), "");
                } else {
                    //no conditional line break, clean builder
                    sb.clear();
                }
                topLines = 0;
            } else {
                //we have more lines than we need, so remove the text we have from the builder and copy processed lines
                StrBuilder replace = new StrBuilder();
                for (int i = 0; i < topLines; i++) {
                    topList.add(ar[i].trim());
                    replace.appendSeparator(' ').append(ar[i]);
                }
                if (wln == true) {
                    replace.append(LINEBREAK);
                }
                sb.replaceFirst(replace.toString(), "");
                topLines = 0;
            }
        }
    }

    //no top, simple wrapping with recognition of conditional line breaks
    count = 0;
    while (sb.size() > 0 && count++ < 200) {
        if (sb.startsWith(LINEBREAK)) {
            sb.replaceFirst(LINEBREAK, "");
        }
        String s = null;
        if (sb.indexOf(LINEBREAK) > 0) {
            s = sb.substring(0, sb.indexOf(LINEBREAK));
            sb.replace(0, sb.indexOf(LINEBREAK) + LINEBREAK.length(), "");
        } else {
            s = sb.toString();
            sb.clear();
        }
        s = WordUtils.wrap(s, this.getWidth(), LINEBREAK, true);
        StrTokenizer tok = new StrTokenizer(s, LINEBREAK).setIgnoreEmptyTokens(false);
        for (String str : tok.getTokenArray()) {
            bottomList.add(str.trim());
        }
    }

    return Pair.of(topList, bottomList);
}

From source file:com.jkoolcloud.tnt4j.streams.parsers.ActivityTokenParser.java

@Override
protected ActivityContext prepareItem(TNTInputStream<?, ?> stream, Object data) throws ParseException {
    // Get next string to parse
    String dataStr = getNextActivityString(data);
    if (StringUtils.isEmpty(dataStr)) {
        return null;
    }/* w  w  w . j  a  v  a  2  s  . c  o  m*/
    logger().log(OpLevel.DEBUG, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
            "ActivityParser.splitting.string"), dataStr);
    if (pattern != null) {
        Matcher matcher = pattern.matcher(dataStr);
        if (matcher == null || !matcher.matches()) {
            logger().log(OpLevel.DEBUG, StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME,
                    "ActivityParser.input.not.match"), getName(), pattern.pattern());
            return null;
        }
    }
    StrTokenizer tk = stripQuotes ? new StrTokenizer(dataStr, fieldDelim, StrMatcher.doubleQuoteMatcher())
            : new StrTokenizer(dataStr, fieldDelim);
    tk.setIgnoreEmptyTokens(false);
    String[] fields = tk.getTokenArray();
    if (ArrayUtils.isEmpty(fields)) {
        logger().log(OpLevel.DEBUG,
                StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "ActivityParser.no.fields"));
        return null;
    }
    logger().log(OpLevel.DEBUG,
            StreamsResources.getString(StreamsResources.RESOURCE_BUNDLE_NAME, "ActivityParser.split"),
            fields.length);

    ActivityContext cData = new ActivityContext(stream, data, fields);
    cData.setMessage(getRawDataAsMessage(fields));

    return cData;
}