List of usage examples for org.apache.commons.lang3.text StrTokenizer getCSVInstance
public static StrTokenizer getCSVInstance()
From source file:com.mgmtp.perfload.perfalyzer.util.MarkersReader.java
public List<Marker> readMarkers() throws IOException { Map<String, Marker> markers = new LinkedHashMap<>(); StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(DELIMITER); try (FileInputStream fis = new FileInputStream(inputFile)) { for (Scanner scanner = new Scanner(fis.getChannel()); scanner.hasNext();) { String line = scanner.nextLine(); if (line.startsWith("#")) { continue; }//from w w w. j a v a 2 s .c o m tokenizer.reset(line); List<String> tokenList = tokenizer.getTokenList(); if (MARKER.equals(tokenList.get(COL_MARKER))) { // no whitespace allowed in marker in order to avoid issues in HTML String markerName = tokenList.get(COL_MARKER_NAME).replaceAll("\\s+", "_"); String markerType = tokenList.get(COL_MARKER_TYPE); long timeMillis = Long.parseLong(tokenList.get(COL_TIMESTAMP)); switch (markerType) { case MARKER_LEFT: { Marker marker = new Marker(markerName); markers.put(markerName, marker); marker.setLeftMillis(timeMillis); break; } case MARKER_RIGHT: { Marker marker = markers.get(markerName); marker.setRightMillis(timeMillis); break; } default: throw new IllegalStateException("Invalid marker type: " + markerType); } } } return markers.values().stream().map(marker -> { marker.calculateDateTimeFields(testStart); return marker; }).collect(toList()); } }
From source file:com.mgmtp.perfload.perfalyzer.util.BinnedFilesMerger.java
public void mergeFiles() throws IOException { if (!inputDir.isDirectory()) { throw new IllegalArgumentException("The input File must be a directory"); }/* w w w . j ava2 s .co m*/ StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(DELIMITER); Map<String, FileChannel> destChannels = newHashMap(); List<OutputStream> outputStreams = newArrayList(); File[] filesInInputDirectory = inputDir.listFiles(); try { for (File file : filesInInputDirectory) { FileInputStream fis = null; try { fis = new FileInputStream(file); for (Scanner scanner = new Scanner(fis.getChannel(), Charsets.UTF_8.name()); scanner .hasNext();) { String line = scanner.nextLine(); tokenizer.reset(line); List<String> tokenList = tokenizer.getTokenList(); String key = tokenList.get(sortCriteriaColumn); FileChannel destChannel = destChannels.get(key); if (destChannel == null) { FileOutputStream fos = new FileOutputStream( new File(outputDir, FILE_TYPE + "_" + key + ".out")); outputStreams.add(fos); destChannel = fos.getChannel(); destChannels.put(key, destChannel); //Write the Header...... Has to be improved IoUtilities.writeLineToChannel(destChannel, getHeader(), Charsets.UTF_8); } StrBuilder outputLine = new StrBuilder(); for (String s : tokenList) { StrBuilderUtils.appendEscapedAndQuoted(outputLine, DELIMITER, s); } IoUtilities.writeLineToChannel(destChannel, outputLine.toString(), Charsets.UTF_8); } } finally { closeQuietly(fis); } } } finally { outputStreams.forEach(IOUtils::closeQuietly); } }
From source file:com.mgmtp.perfload.perfalyzer.binning.RequestFilesMerger.java
public void mergeFiles(final List<PerfAlyzerFile> inputFiles) throws IOException { Predicate<PerfAlyzerFile> predicate1 = perfAlyzerFilePartsMatchWildcards("measuring", "*", "requestsPerInterval"); Predicate<PerfAlyzerFile> predicate2 = perfAlyzerFilePartsMatchWildcards("measuring", "*", "aggregatedResponseTimes"); Predicate<PerfAlyzerFile> predicateOr = predicate1.or(predicate2); Set<PerfAlyzerFile> paFiles = inputFiles.stream().filter(predicateOr).collect(toSet()); ListMultimap<String, PerfAlyzerFile> byOperationMultimap = ArrayListMultimap.create(); for (PerfAlyzerFile paf : paFiles) { byOperationMultimap.put(paf.getFileNameParts().get(1), paf); }// ww w .j a va2s.c o m StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(DELIMITER); for (String operation : byOperationMultimap.keySet()) { List<PerfAlyzerFile> list = byOperationMultimap.get(operation); checkState(list.size() == 2, "Two files are required by operation but found %d for '%s'", list.size(), operation); List<String> resultLines = newArrayListWithCapacity(2); PerfAlyzerFile paf1 = list.stream().filter(predicate1).findFirst().get(); File file1 = new File(binnedDir, paf1.getFile().getPath()); List<String> lines1 = Files.readLines(file1, Charsets.UTF_8); PerfAlyzerFile paf2 = list.stream().filter(predicate2).findFirst().get(); File file2 = new File(binnedDir, paf2.getFile().getPath()); List<String> lines2 = Files.readLines(file2, Charsets.UTF_8); if (lines1.size() == lines2.size()) { File resultFile = new File(binnedDir, paf1.copy().removeFileNamePart(2).addFileNamePart("aggregated").getFile().getPath()); for (int i = 0; i < lines1.size(); ++i) { String line1 = get(lines1, i); String line2 = get(lines2, i); resultLines.add(line1 + DELIMITER + line2); } writeLines(resultFile, Charsets.UTF_8.name(), resultLines); deleteQuietly(file1); deleteQuietly(file2); } else { log.warn("Files to merge must have the same number of lines. Merging not possible: {}", list); } } }
From source file:com.mgmtp.perfload.perfalyzer.util.PerfAlyzerUtils.java
/** * Reads a semicolon-delimited CSV file into a list. Each line in the result list will be * another list of {@link Number} objects. The file is expected to have two numberic columns * which are parsed using the specified number format. * // ww w. j a va2s .c o m * @param file * the file * @param charset * the character set to read the file * @param numberFormat * the number format for parsing the column values * @return the immutable result list */ public static List<SeriesPoint> readDataFile(final File file, final Charset charset, final NumberFormat numberFormat) throws IOException { final StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(';'); try (BufferedReader br = newReader(file, charset)) { boolean headerLine = true; List<SeriesPoint> result = newArrayListWithExpectedSize(200); for (String line; (line = br.readLine()) != null;) { try { if (headerLine) { headerLine = false; } else { tokenizer.reset(line); String[] tokens = tokenizer.getTokenArray(); double x = numberFormat.parse(tokens[0]).doubleValue(); double y = numberFormat.parse(tokens[1]).doubleValue(); if (!result.isEmpty()) { // additional point for histogram SeriesPoint previousPoint = getLast(result); result.add(new SeriesPoint(x, previousPoint.getY())); } tokenizer.reset(line); result.add(new SeriesPoint(x, y)); } } catch (ParseException ex) { throw new IOException("Error parsing number in file: " + file, ex); } } int size = result.size(); if (size > 2) { // additional point at end for histogram SeriesPoint nextToLast = result.get(size - 3); SeriesPoint last = result.get(size - 1); double dX = last.getX().doubleValue() - nextToLast.getX().doubleValue(); result.add(new SeriesPoint(last.getX().doubleValue() + dX, last.getY())); } return ImmutableList.copyOf(result); } }
From source file:com.mgmtp.jfunk.core.util.CsvDataProcessor.java
/** * Processes the specified CSV file. For every line but the header line (which is required), the * specified command is executed.// w w w .ja v a 2s . c o m * * @param reader * the reader for loading the CSV data * @param delimiter * the column separator * @param quoteChar * the quote character ('\0' for no quoting) * @param command * the command (i. e. a Groovy closure if used in a Groovy script) to be executed for * every processed line */ public void processFile(final Reader reader, final String delimiter, final char quoteChar, final Runnable command) { try { List<String> inputLines = CharStreams.readLines(reader); StrTokenizer st = StrTokenizer.getCSVInstance(); st.setDelimiterString(delimiter); if (quoteChar != '\0') { st.setQuoteChar(quoteChar); } else { st.setQuoteMatcher(StrMatcher.noneMatcher()); } // extract header String headerLine = inputLines.remove(0); List<Column> columns = initColumns(st, headerLine); for (String line : inputLines) { st.reset(line); String[] colArray = st.getTokenArray(); int len = colArray.length; checkState(len == columns.size(), "Mismatch between number of header columns and number of line columns."); DataSource dataSource = dataSourceProvider.get(); Configuration config = configProvider.get(); for (int i = 0; i < len; ++i) { String value = StringUtils.trimToEmpty(colArray[i]); String dataSetKey = columns.get(i).dataSetKey; String key = columns.get(i).key; if (dataSetKey != null) { if ("<auto>".equals(value)) { dataSource.resetFixedValue(dataSetKey, key); } else { log.debug("Setting data set entry for " + this + " to value=" + value); dataSource.setFixedValue(dataSetKey, key, value); } } else { log.debug("Setting property for " + this + " to value=" + value); config.put(key, value); } } command.run(); } } catch (IOException ex) { throw new JFunkException("Error processing CSV data", ex); } }
From source file:com.mgmtp.perfload.perfalyzer.reportpreparation.PerfMonReportPreparationStrategy.java
private void createCsvFiles(final File sourceDir, final File destDir, final ListMultimap<String, PerfAlyzerFile> byTypeAndHostMultimap) throws IOException { ListMultimap<String, String> globalContentListMultimap = LinkedListMultimap.create(); StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(DELIMITER); for (String key : byTypeAndHostMultimap.keySet()) { List<PerfAlyzerFile> filesByType = byTypeAndHostMultimap.get(key); File destFile = new File(destDir, key); String[] split = split(key, SystemUtils.FILE_SEPARATOR, 2); String host = split[0];/* w ww . j a va2s . c o m*/ String keyWithoutHost = split[1]; List<String> contentList = newLinkedList(); for (PerfAlyzerFile f : filesByType) { String type = f.getFileNameParts().get(1); // aggregated files always have two lines, a header and a data line List<String> lines = readLines(new File(sourceDir, f.getFile().getPath()), Charsets.UTF_8); if (lines.size() < 2) { // needs at least header and one content line continue; } if (contentList.isEmpty()) { // write header contentList.add(0, "\"type\"" + DELIMITER + lines.get(0)); } String line = lines.get(1); tokenizer.reset(line); String[] columns = tokenizer.getTokenArray(); StrBuilder sb = new StrBuilder(10 + line.length()); appendEscapedAndQuoted(sb, DELIMITER, type); for (String column : columns) { appendEscapedAndQuoted(sb, DELIMITER, column); } line = sb.toString(); contentList.add(line); List<String> globalContentList = globalContentListMultimap.get(keyWithoutHost); if (globalContentList.isEmpty()) { globalContentList.add("\"host\"" + DELIMITER + "\"type\"" + DELIMITER + lines.get(0)); } globalContentList.add("\"" + host + "\"" + DELIMITER + line); } // exclude header line from sorting Collections.sort(contentList.subList(1, contentList.size())); writeLines(destFile, Charsets.UTF_8.name(), contentList); } for (String key : globalContentListMultimap.keySet()) { List<String> globalContentList = globalContentListMultimap.get(key); // exclude header line from sorting Collections.sort(globalContentList.subList(1, globalContentList.size())); writeLines(new File(destDir, "global" + SystemUtils.FILE_SEPARATOR + key), Charsets.UTF_8.name(), globalContentList); } }
From source file:com.mgmtp.perfload.core.common.config.XmlConfigReader.java
private ListMultimap<ProcessKey, LoadProfileEvent> readLoadProfileEvents(final Element testplan) throws IOException { ListMultimap<ProcessKey, LoadProfileEvent> eventsByProcess = ArrayListMultimap.create(); String loadProfile = testplan.elementTextTrim("loadProfile"); // relative to testplan File loadProfileConfigFile = new File(new File(testplanFile.getParentFile(), "loadprofiles"), loadProfile); try (BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(loadProfileConfigFile), "UTF-8"))) { StrTokenizer st = StrTokenizer.getCSVInstance(); st.setDelimiterChar(';'); for (String line = null; (line = br.readLine()) != null;) { // ignore line that are blank, commented out, or represent markers if (isBlank(line) || startsWith(line, "#") || MARKER_PATTERN.matcher(line).matches()) { continue; }/*from w w w.j ava 2 s. c o m*/ st.reset(line); String[] tokens = st.getTokenArray(); long startTime = Long.parseLong(tokens[0]); String operation = tokens[1]; String target = tokens[2]; int daemonId = Integer.parseInt(tokens[3]); int processId = Integer.parseInt(tokens[4]); eventsByProcess.put(new ProcessKey(daemonId, processId), new LoadProfileEvent(startTime, operation, target, daemonId, processId)); } } return eventsByProcess; }
From source file:com.mgmtp.perfload.perfalyzer.util.PerfAlyzerUtils.java
/** * Reads a semicolon-delimited CSV file into a map of lists series values. Values for each * column are return as list of lists in the map, the key being the column header. * /*from www . j a v a 2s.co m*/ * @param file * the file * @param charset * the character set to read the file * @param numberFormat * the number format for formatting the column values * @param columnNames * the columns to consider * * @return an immutable map of lists of series values */ public static Map<String, List<SeriesPoint>> readDataFile(final File file, final Charset charset, final NumberFormat numberFormat, final Set<String> columnNames) throws IOException { final StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(';'); return readLines(file, charset, new LineProcessor<Map<String, List<SeriesPoint>>>() { private String[] headers; private final Map<String, List<SeriesPoint>> result = newHashMapWithExpectedSize(4); private int colCount; @Override public boolean processLine(final String line) throws IOException { try { tokenizer.reset(line); String[] tokens = tokenizer.getTokenArray(); if (headers == null) { headers = tokens; colCount = tokens.length; } else { Integer counter = Integer.valueOf(tokens[0]); for (int i = 1; i < colCount; ++i) { String header = headers[i]; if (columnNames.contains(header)) { List<SeriesPoint> colValues = result.get(header); if (colValues == null) { colValues = newArrayListWithExpectedSize(50); result.put(header, colValues); } colValues.add(new SeriesPoint(counter, numberFormat.parse(tokens[i]))); } } } return true; } catch (ParseException ex) { throw new IOException("Error parsing number in file: " + file, ex); } } @Override public Map<String, List<SeriesPoint>> getResult() { return ImmutableMap.copyOf(result); } }); }
From source file:com.mgmtp.perfload.perfalyzer.util.PerfAlyzerUtils.java
public static Map<String, String> readAggregatedMap(final File executionsFile, final Charset charset) throws IOException { final StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(';'); Map<String, String> result = newHashMapWithExpectedSize(11); List<String> lines = Files.readLines(executionsFile, charset); String[] headers = null;//from w w w. j a va 2 s . c om for (String line : lines) { tokenizer.reset(line); String[] tokens = tokenizer.getTokenArray(); if (headers == null) { headers = tokens; } else { String[] data = tokenizer.getTokenArray(); String operation = data[0]; for (int i = 1; i < headers.length; ++i) { result.put(operation + "." + headers[i], data[i]); } } } return result; }
From source file:com.mgmtp.perfload.perfalyzer.reporting.email.EmailReporter.java
private List<? extends List<String>> loadData(final File file) throws IOException { try (BufferedReader br = newReader(file, Charsets.UTF_8)) { List<List<String>> rows = newArrayList(); StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); tokenizer.setDelimiterChar(DELIMITER); for (String line; (line = br.readLine()) != null;) { tokenizer.reset(line);/*ww w . j a v a2 s . c om*/ List<String> tokenList = tokenizer.getTokenList(); rows.add(tokenList); } return rows; } }