Example usage for org.apache.commons.io IOUtils readLines

List of usage examples for org.apache.commons.io IOUtils readLines

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.io.*
  5. IOUtils
  6. readLines

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils readLines.

Prototype

public static List readLines(Reader input) throws IOException

Source Link

Document

Get the contents of a Reader as a list of Strings, one entry per line.

Usage

From source file:ita.parthenope.twitternlp.semantic.CloudOfWords.java

private Set<String> loadStopWords() {
    try {/*  w w w .  j a va 2s .  c om*/
        final List<String> lines = IOUtils.readLines(getInputStream("resources/stopwords_en.txt"));
        return new HashSet<>(lines);

    } catch (IOException e) {
        Logger.getLogger(this.getClass().getName()).log(Level.SEVERE, null, e);
    }
    return Collections.emptySet();
}

From source file:it.greenvulcano.util.txt.TextUtils.java

/**
 * Opens and loads the content of a text file into a list of String
 * //  www .  j  ava  2  s  .c  o m
 * @param filename
 *        the name of the file
 * @return The content of the text file as a list of String
 * @throws FileNotFoundException
 * @throws IOException
 */
public static List<String> readFileAsLines(String filename) throws FileNotFoundException, IOException {
    BufferedReader reader = null;
    try {
        filename = adjustPath(filename);
        reader = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
        return IOUtils.readLines(reader);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}

From source file:com.linkedin.pinot.tools.perf.QueryRunner.java

/**
 * Use multiple threads to run queries as fast as possible.
 * <p>Use a concurrent linked queue to buffer the queries to be sent. Use the main thread to insert queries into the
 * queue whenever the queue length is low, and start <code>numThreads</code> worker threads to fetch queries from the
 * queue and send them./*w w w .  j a va  2 s  . com*/
 * <p>The main thread is responsible for collecting and logging the statistic information periodically.
 * <p>Queries are picked sequentially from the query file.
 * <p>Query runner will stop when all queries in the query file has been executed number of times configured.
 *
 * @param conf perf benchmark driver config.
 * @param queryFile query file.
 * @param numTimesToRunQueries number of times to run all queries in the query file, 0 means infinite times.
 * @param numThreads number of threads sending queries.
 * @param reportIntervalMs report interval in milliseconds.
 * @param numIntervalsToReportAndClearStatistics number of report intervals to report detailed statistics and clear
 *                                               them, 0 means never.
 * @throws Exception
 */
public static void multiThreadedQueryRunner(PerfBenchmarkDriverConf conf, String queryFile,
        int numTimesToRunQueries, int numThreads, int reportIntervalMs,
        int numIntervalsToReportAndClearStatistics) throws Exception {
    List<String> queries;
    try (FileInputStream input = new FileInputStream(new File(queryFile))) {
        queries = IOUtils.readLines(input);
    }

    PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf);
    ConcurrentLinkedQueue<String> queryQueue = new ConcurrentLinkedQueue<>();
    AtomicInteger numQueriesExecuted = new AtomicInteger(0);
    AtomicLong totalBrokerTime = new AtomicLong(0L);
    AtomicLong totalClientTime = new AtomicLong(0L);
    List<Statistics> statisticsList = Collections.singletonList(new Statistics(CLIENT_TIME_STATISTICS));

    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    for (int i = 0; i < numThreads; i++) {
        executorService.submit(new Worker(driver, queryQueue, numQueriesExecuted, totalBrokerTime,
                totalClientTime, statisticsList));
    }
    executorService.shutdown();

    long startTime = System.currentTimeMillis();
    long reportStartTime = startTime;
    int numReportIntervals = 0;
    int numTimesExecuted = 0;
    while (numTimesToRunQueries == 0 || numTimesExecuted < numTimesToRunQueries) {
        if (executorService.isTerminated()) {
            LOGGER.error("All threads got exception and already dead.");
            return;
        }

        for (String query : queries) {
            queryQueue.add(query);

            // Keep 20 queries inside the query queue.
            while (queryQueue.size() == 20) {
                Thread.sleep(1);

                long currentTime = System.currentTimeMillis();
                if (currentTime - reportStartTime >= reportIntervalMs) {
                    long timePassed = currentTime - startTime;
                    int numQueriesExecutedInt = numQueriesExecuted.get();
                    LOGGER.info(
                            "Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, Average Broker Time: {}ms, "
                                    + "Average Client Time: {}ms.",
                            timePassed, numQueriesExecutedInt,
                            numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND),
                            totalBrokerTime.get() / (double) numQueriesExecutedInt,
                            totalClientTime.get() / (double) numQueriesExecutedInt);
                    reportStartTime = currentTime;
                    numReportIntervals++;

                    if ((numIntervalsToReportAndClearStatistics != 0)
                            && (numReportIntervals == numIntervalsToReportAndClearStatistics)) {
                        numReportIntervals = 0;
                        startTime = currentTime;
                        reportAndClearStatistics(numQueriesExecuted, totalBrokerTime, totalClientTime,
                                statisticsList);
                    }
                }
            }
        }
        numTimesExecuted++;
    }

    // Wait for all queries getting executed.
    while (queryQueue.size() != 0) {
        Thread.sleep(1);
    }
    executorService.shutdownNow();
    while (!executorService.isTerminated()) {
        Thread.sleep(1);
    }

    long timePassed = System.currentTimeMillis() - startTime;
    int numQueriesExecutedInt = numQueriesExecuted.get();
    LOGGER.info("--------------------------------------------------------------------------------");
    LOGGER.info("FINAL REPORT:");
    LOGGER.info(
            "Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, Average Broker Time: {}ms, "
                    + "Average Client Time: {}ms.",
            timePassed, numQueriesExecutedInt,
            numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND),
            totalBrokerTime.get() / (double) numQueriesExecutedInt,
            totalClientTime.get() / (double) numQueriesExecutedInt);
    for (Statistics statistics : statisticsList) {
        statistics.report();
    }
}

From source file:com.github.tteofili.looseen.Test20NewsgroupsClassification.java

private NewsPost parse(File postFile, String groupName, String number) throws IOException {
    StringBuilder body = new StringBuilder();
    String subject = "";
    FileInputStream stream = new FileInputStream(postFile);
    boolean inBody = false;
    for (String line : IOUtils.readLines(stream)) {
        if (line.startsWith("Subject:")) {
            subject = line.substring(8);
        } else {//  ww  w  . ja  v a  2  s .  co m
            if (inBody) {
                if (body.length() > 0) {
                    body.append("\n");
                }
                body.append(line);
            } else if (line.isEmpty() || line.trim().length() == 0) {
                inBody = true;
            }
        }
    }
    return new NewsPost(body.toString(), subject, groupName, number);
}

From source file:ml.shifu.shifu.core.processor.VarSelectModelProcessor.java

/**
 * Load variable selection history file into VarSelDesc
 * /*from ww w.ja  v  a 2 s.c  om*/
 * @param varselHistory
 *            - variable selection history file file
 * @return
 * @throws IOException
 */
private List<VarSelDesc> loadVarSelDescList(String varselHistory) throws IOException {
    Reader reader = ShifuFileUtils.getReader(varselHistory, SourceType.LOCAL);
    List<String> autoFilterList = IOUtils.readLines(reader);
    IOUtils.closeQuietly(reader);

    List<VarSelDesc> varSelDescList = new ArrayList<VarSelDesc>();
    for (String filterDesc : autoFilterList) {
        VarSelDesc varSelDesc = VarSelDesc.fromString(filterDesc);
        if (varSelDesc != null) {
            varSelDescList.add(varSelDesc);
        }
    }

    return varSelDescList;
}

From source file:io.fabric8.maven.core.service.openshift.OpenshiftBuildServiceTest.java

@Test
public void checkTarPackageSecret() throws Exception {
    int nTries = 0;
    boolean bTestComplete = false;
    do {//w  ww.  j a v  a2  s  . com
        try {
            nTries++;
            BuildService.BuildServiceConfig config = defaultConfigSecret.build();
            WebServerEventCollector<OpenShiftMockServer> collector = createMockServer(config, true, 50, true,
                    true);
            OpenShiftMockServer mockServer = collector.getMockServer();

            OpenShiftClient client = mockServer.createOpenShiftClient();
            final OpenshiftBuildService service = new OpenshiftBuildService(client, logger, dockerServiceHub,
                    config);

            ImageConfiguration imageWithEnv = new ImageConfiguration.Builder(image)
                    .buildConfig(new BuildImageConfiguration.Builder(image.getBuildConfiguration())
                            .env(Collections.singletonMap("FOO", "BAR")).build())
                    .build();

            service.createBuildArchive(imageWithEnv);

            final List<ArchiverCustomizer> customizer = new LinkedList<>();
            new Verifications() {
                {
                    archiveService.createDockerBuildArchive(withInstanceOf(ImageConfiguration.class),
                            withInstanceOf(MojoParameters.class), withCapture(customizer));

                    assertTrue(customizer.size() == 1);
                }
            };

            customizer.get(0).customize(tarArchiver);

            final List<File> file = new LinkedList<>();
            new Verifications() {
                {
                    String path;
                    tarArchiver.addFile(withCapture(file), path = withCapture());

                    assertEquals(".s2i/environment", path);
                }
            };

            assertEquals(1, file.size());
            List<String> lines;
            try (FileReader reader = new FileReader(file.get(0))) {
                lines = IOUtils.readLines(reader);
            }
            assertTrue(lines.contains("FOO=BAR"));
            bTestComplete = true;
        } catch (Fabric8ServiceException exception) {
            Throwable rootCause = getRootCause(exception);
            logger.warn("A problem encountered while running test {}, retrying..", exception.getMessage());
            // Let's wait for a while, and then retry again
            if (rootCause != null && rootCause instanceof IOException) {
                continue;
            }
        }
    } while (nTries < MAX_TIMEOUT_RETRIES && !bTestComplete);
}

From source file:it.greenvulcano.util.txt.TextUtils.java

/**
 * Opens and loads the content of a text file (read from URL) into a
 * list of String//www  . j  a va2  s . c om
 * 
 * @param url
 *        the url of the file
 * @return The content of the text file as a list of String
 * @throws IOException
 */
public static List<String> readFileAsLinesFromURL(URL url) throws IOException {
    BufferedReader reader = null;
    try {
        reader = new BufferedReader(new InputStreamReader(url.openStream()));
        return IOUtils.readLines(reader);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}

From source file:com.opentable.db.postgres.embedded.EmbeddedPostgres.java

private static List<String> system(ProcessBuilder.Redirect errorRedirector,
        ProcessBuilder.Redirect outputRedirector, String... command) {
    try {/*w  w  w . j  ava2 s .co m*/
        final ProcessBuilder builder = new ProcessBuilder(command);
        builder.redirectError(errorRedirector);
        builder.redirectOutput(outputRedirector);
        final Process process = builder.start();
        Verify.verify(0 == process.waitFor(), "Process %s failed\n%s", Arrays.asList(command),
                IOUtils.toString(process.getErrorStream()));
        try (InputStream stream = process.getInputStream()) {
            return IOUtils.readLines(stream);
        }
    } catch (final Exception e) {
        throw Throwables.propagate(e);
    }
}

From source file:cn.z.Ocr5.java

public static String getSingleCharOcr(BufferedImage img, Map<BufferedImage, String> map) throws Exception {
    if (useSvm) {
        final String input = new File("img/" + clazz + "/input.txt").getAbsolutePath();
        final String output = new File("result/" + clazz + "/output.txt").getAbsolutePath();
        CommonUtil.imgToSvmInput(img, input, whiteThreshold);
        svm_predict.main(new String[] { input, new File("train/" + clazz + "/data.txt.model").getAbsolutePath(),
                output });//from  w w w .  j av  a 2 s.c om
        final List<String> predict = IOUtils.readLines(new FileInputStream(output));
        if (predict.size() > 0 && predict.get(0).length() > 0) {
            return predict.get(0).substring(0, 1);
        }
        return "#";
    }

    String result = "#";
    img = scaleImage(img);
    final int width = img.getWidth();
    final int height = img.getHeight();
    int min = width * height;
    final boolean bNotEight = isNotEight(img);
    final boolean bNotThree = isNotThree(img);
    final boolean bNotFive = isNotFive(img);
    for (final BufferedImage bi : map.keySet()) {
        if (bNotThree && map.get(bi).startsWith("3")) {
            continue;
        }
        if (bNotEight && map.get(bi).startsWith("8")) {
            continue;
        }
        if (bNotFive && map.get(bi).startsWith("5")) {
            continue;
        }
        final double count1 = getBlackCount(img);
        final double count2 = getBlackCount(bi);
        if (Math.abs(count1 - count2) / Math.max(count1, count2) > 0.25) {
            continue;
        }
        int count = 0;
        if (width < bi.getWidth() && height < bi.getHeight()) {
            for (int m = 0; m <= bi.getWidth() - width; m++) {
                for (int n = 0; n <= bi.getHeight() - height; n++) {
                    Label1: for (int x = m; x < m + width; ++x) {
                        for (int y = n; y < n + height; ++y) {
                            if (CommonUtil.isWhite(img.getRGB(x - m, y - n), whiteThreshold) != CommonUtil
                                    .isWhite(bi.getRGB(x, y), whiteThreshold)) {
                                count++;
                                if (count >= min) {
                                    break Label1;
                                }
                            }
                        }
                    }
                }
            }
        } else {
            final int widthmin = width < bi.getWidth() ? width : bi.getWidth();
            final int heightmin = height < bi.getHeight() ? height : bi.getHeight();
            Label1: for (int x = 0; x < widthmin; ++x) {
                for (int y = 0; y < heightmin; ++y) {
                    if (CommonUtil.isWhite(img.getRGB(x, y), whiteThreshold) != CommonUtil
                            .isWhite(bi.getRGB(x, y), whiteThreshold)) {
                        count++;
                        if (count >= min) {
                            break Label1;
                        }
                    }
                }
            }
        }
        if (count < min) {
            min = count;
            result = map.get(bi);
        }
    }
    return result;
}

From source file:com.ikon.dao.HibernateUtil.java

/**
 * Replace "create" or "update" by "none" to prevent repository reset on restart
 *//*from  w  w w. j a va2  s  . co m*/
@SuppressWarnings("unchecked")
public static void hibernateCreateAutofix(String configFile) throws IOException {
    FileReader fr = null;
    FileWriter fw = null;

    try {
        // Open and close reader
        fr = new FileReader(configFile);
        List<String> lines = IOUtils.readLines(fr);
        IOUtils.closeQuietly(fr);

        // Modify configuration file
        fw = new FileWriter(configFile);

        for (String line : lines) {
            line = line.trim();
            int idx = line.indexOf("=");

            if (idx > -1) {
                String key = line.substring(0, idx).trim();
                String value = line.substring(idx + 1, line.length()).trim();

                if (Config.PROPERTY_HIBERNATE_HBM2DDL.equals(key)) {
                    value = HBM2DDL_NONE;
                }

                fw.write(key + "=" + value + "\n");
            } else {
                fw.write(line + "\n");
            }
        }

        fw.flush();
    } finally {
        IOUtils.closeQuietly(fr);
        IOUtils.closeQuietly(fw);
    }
}