List of usage examples for org.apache.commons.io IOUtils readLines
public static List readLines(Reader input) throws IOException
Reader as a list of Strings, one entry per line. From source file:de.tudarmstadt.ukp.dkpro.tc.svmhmm.util.SVMHMMUtils.java
/** * Reads the featureVectorsFile and splits comment on each line into a list of strings, i.e. * "TAG qid:4 1:1 2:1 4:2 # token TAG 4" produces "token", "TAG", "4" * * @param featureVectorsFileStream featureVectors file stream * @return list (for each line) of list of comment parts * @throws IOException//from w w w .jav a 2 s . c om */ protected static List<List<String>> extractComments(InputStream featureVectorsFileStream // int expectedFieldsCount ) throws IOException, IllegalArgumentException { List<List<String>> result = new ArrayList<>(); List<String> lines = IOUtils.readLines(featureVectorsFileStream); IOUtils.closeQuietly(featureVectorsFileStream); for (String line : lines) { String comment = line.split("#", 2)[1]; List<String> list = new ArrayList<>(); String[] tokens = comment.split("\\s+"); // filter empty tokens for (String token : tokens) { String trim = token.trim(); if (!trim.isEmpty()) { // decode from URL representation String s = URLDecoder.decode(trim, "utf-8"); list.add(s); } } result.add(list); } return result; }
From source file:edu.utexas.cs.tactex.servercustomers.factoredcustomer.TimeseriesGenerator.java
private void initArima101x101RefSeries() { InputStream refStream;/*from w w w .j a v a2 s .c o m*/ String seriesName = tsStructure.refSeriesName; switch (tsStructure.refSeriesSource) { case BUILTIN: throw new Error("Unknown builtin series name: " + seriesName); // break; case CLASSPATH: refStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(seriesName); break; case FILEPATH: try { refStream = new FileInputStream(seriesName); } catch (FileNotFoundException e) { throw new Error("Could not find file to initialize reference timeseries: " + seriesName); } break; default: throw new Error("Unexpected reference timeseries source type: " + tsStructure.refSeriesSource); } if (refStream == null) throw new Error("Reference timeseries input stream is uninitialized!"); try { @SuppressWarnings("unchecked") List<String> series = (List<String>) IOUtils.readLines(refStream); for (String line : series) { Double element = Double.parseDouble(line); refSeries.add(element); } } catch (java.io.EOFException e) { final int MIN_TIMESERIES_LENGTH = 26; if (refSeries.size() < MIN_TIMESERIES_LENGTH) { throw new Error("Insufficient data in reference series; expected " + MIN_TIMESERIES_LENGTH + " elements, found only " + genSeries.size()); } } catch (java.io.IOException e) { throw new Error("Error reading timeseries data from file: " + seriesName + "; caught IOException: " + e.toString()); } }
From source file:alluxio.cli.AlluxioFrameworkIntegrationTest.java
private static boolean processExists(String processName) throws Exception { Process ps = Runtime.getRuntime().exec(new String[] { "ps", "ax" }); InputStream psOutput = ps.getInputStream(); Process processGrep = Runtime.getRuntime().exec(new String[] { "grep", processName }); OutputStream processGrepInput = processGrep.getOutputStream(); IOUtils.copy(psOutput, processGrepInput); InputStream processGrepOutput = processGrep.getInputStream(); processGrepInput.close();// w ww . ja va 2 s . c o m // Filter out the grep process itself. Process filterGrep = Runtime.getRuntime().exec(new String[] { "grep", "-v", "grep" }); OutputStream filterGrepInput = filterGrep.getOutputStream(); IOUtils.copy(processGrepOutput, filterGrepInput); filterGrepInput.close(); return IOUtils.readLines(filterGrep.getInputStream()).size() >= 1; }
From source file:functionaltests.RestSchedulerPushPullFileTest.java
public void testIt(String spaceName, String spacePath, String destPath, boolean encode) throws Exception { File testPushFile = RestFuncTHelper.getDefaultJobXmlfile(); // you can test pushing pulling a big file : // testPushFile = new File("path_to_a_big_file"); File destFile = new File(new File(spacePath, destPath), testPushFile.getName()); if (destFile.exists()) { destFile.delete();/*from w w w . j a v a2 s .c o m*/ } // PUSHING THE FILE String pushfileUrl = getResourceUrl("dataspace/" + spaceName + "/" + (encode ? URLEncoder.encode(destPath, "UTF-8") : destPath.replace("\\", "/"))); // either we encode or we test human readable path (with no special character inside) HttpPost reqPush = new HttpPost(pushfileUrl); setSessionHeader(reqPush); // we push a xml job as a simple test MultipartEntity multipartEntity = new MultipartEntity(); multipartEntity.addPart("fileName", new StringBody(testPushFile.getName())); multipartEntity.addPart("fileContent", new InputStreamBody(FileUtils.openInputStream(testPushFile), MediaType.APPLICATION_OCTET_STREAM, null)); reqPush.setEntity(multipartEntity); HttpResponse response = executeUriRequest(reqPush); System.out.println(response.getStatusLine()); assertHttpStatusOK(response); Assert.assertTrue(destFile + " exists for " + spaceName, destFile.exists()); Assert.assertTrue("Original file and result are equals for " + spaceName, FileUtils.contentEquals(testPushFile, destFile)); // LISTING THE TARGET DIRECTORY String pullListUrl = getResourceUrl("dataspace/" + spaceName + "/" + (encode ? URLEncoder.encode(destPath, "UTF-8") : destPath.replace("\\", "/"))); HttpGet reqPullList = new HttpGet(pullListUrl); setSessionHeader(reqPullList); HttpResponse response2 = executeUriRequest(reqPullList); System.out.println(response2.getStatusLine()); assertHttpStatusOK(response2); InputStream is = response2.getEntity().getContent(); List<String> lines = IOUtils.readLines(is); HashSet<String> content = new HashSet<>(lines); System.out.println(lines); Assert.assertTrue("Pushed file correctly listed", content.contains(testPushFile.getName())); // PULLING THE FILE String pullfileUrl = getResourceUrl("dataspace/" + spaceName + "/" + (encode ? URLEncoder.encode(destPath + "/" + testPushFile.getName(), "UTF-8") : destPath.replace("\\", "/") + "/" + testPushFile.getName())); HttpGet reqPull = new HttpGet(pullfileUrl); setSessionHeader(reqPull); HttpResponse response3 = executeUriRequest(reqPull); System.out.println(response3.getStatusLine()); assertHttpStatusOK(response3); InputStream is2 = response3.getEntity().getContent(); File answerFile = tmpFolder.newFile(); FileUtils.copyInputStreamToFile(is2, answerFile); Assert.assertTrue("Original file and result are equals for " + spaceName, FileUtils.contentEquals(answerFile, testPushFile)); // DELETING THE HIERARCHY String rootPath = destPath.substring(0, destPath.contains("/") ? destPath.indexOf("/") : destPath.length()); String deleteUrl = getResourceUrl("dataspace/" + spaceName + "/" + (encode ? URLEncoder.encode(rootPath, "UTF-8") : rootPath.replace("\\", "/"))); HttpDelete reqDelete = new HttpDelete(deleteUrl); setSessionHeader(reqDelete); HttpResponse response4 = executeUriRequest(reqDelete); System.out.println(response4.getStatusLine()); assertHttpStatusOK(response4); Assert.assertTrue(destFile + " still exist", !destFile.exists()); }
From source file:gov.nih.nci.nbia.StandaloneDMV1.java
private static List<String> connectAndReadFromURL(URL url, String fileName) { List<String> data = null; DefaultHttpClient httpClient = null; TrustStrategy easyStrategy = new TrustStrategy() { @Override/*w w w . j a v a 2 s .c o m*/ public boolean isTrusted(X509Certificate[] certificate, String authType) throws CertificateException { return true; } }; try { // SSLContext sslContext = SSLContext.getInstance("SSL"); // set up a TrustManager that trusts everything // sslContext.init(null, new TrustManager[] { new // EasyX509TrustManager(null)}, null); SSLSocketFactory sslsf = new SSLSocketFactory(easyStrategy, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); Scheme httpsScheme = new Scheme("https", 443, sslsf); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(httpsScheme); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); ClientConnectionManager ccm = new ThreadSafeClientConnManager(schemeRegistry); HttpParams httpParams = new BasicHttpParams(); HttpConnectionParams.setConnectionTimeout(httpParams, 50000); HttpConnectionParams.setSoTimeout(httpParams, new Integer(12000)); httpClient = new DefaultHttpClient(ccm, httpParams); httpClient.setRoutePlanner(new ProxySelectorRoutePlanner(schemeRegistry, ProxySelector.getDefault())); // // Additions by lrt for tcia - // // attempt to reduce errors going through a Coyote Point // Equalizer load balance switch httpClient.getParams().setParameter("http.socket.timeout", new Integer(12000)); httpClient.getParams().setParameter("http.socket.receivebuffer", new Integer(16384)); httpClient.getParams().setParameter("http.tcp.nodelay", true); httpClient.getParams().setParameter("http.connection.stalecheck", false); // // end lrt additions HttpPost httpPostMethod = new HttpPost(url.toString()); List<BasicNameValuePair> postParams = new ArrayList<BasicNameValuePair>(); postParams.add(new BasicNameValuePair("serverjnlpfileloc", fileName)); UrlEncodedFormEntity query = new UrlEncodedFormEntity(postParams); httpPostMethod.setEntity(query); HttpResponse response = httpClient.execute(httpPostMethod); // int responseCode = response.getStatusLine().getStatusCode(); // System.out.println("Response code for requesting datda file: " + // responseCode); InputStream inputStream = response.getEntity().getContent(); data = IOUtils.readLines(inputStream); } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (KeyManagementException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (NoSuchAlgorithmException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (KeyStoreException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (UnrecoverableKeyException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (httpClient != null) { httpClient.getConnectionManager().shutdown(); } } return data; }
From source file:io.milton.common.FileUtils.java
public static void readLines(File f, List<String> lines) { InputStream in = null;/* ww w.ja v a 2 s . c o m*/ try { in = new FileInputStream(f); for (Object oLine : IOUtils.readLines(in)) { lines.add(oLine.toString()); } } catch (Exception e) { throw new RuntimeException(f.getAbsolutePath(), e); } finally { IOUtils.closeQuietly(in); } }
From source file:com.kennycason.kumo.LayeredWordCloudITest.java
private static Set<String> loadStopWords() { try {/*from w w w .java 2s.com*/ final List<String> lines = IOUtils.readLines(getInputStream("text/stop_words.txt")); return new HashSet<>(lines); } catch (IOException e) { LOGGER.error(e.getMessage(), e); } return Collections.emptySet(); }
From source file:hoot.services.nativeInterfaces.ProcessStreamInterface.java
@Override public JSONObject exec(JSONObject cmd) throws NativeInterfaceException { log.debug("Executing command : " + cmd.toJSONString()); String[] commandArr = null;//from w w w . j a v a 2 s. c om JSONObject ret = new JSONObject(); CommandResult res = null; boolean addedToQ = false; try { String exec = cmd.get("exectype").toString(); if (exec.equalsIgnoreCase("hoot")) { commandArr = createCmdArray(cmd); } else if (exec.equalsIgnoreCase("make")) { commandArr = createScriptCmdArray(cmd); } else if (exec.equalsIgnoreCase("bash")) { commandArr = createBashScriptCmdArray(cmd); } else { log.error("Failed to parse params: " + cmd.toJSONString()); throw new NativeInterfaceException("Failed to parse params.", NativeInterfaceException.HttpCode.BAD_RQUEST); } String commandStr = ArrayUtils.toString(commandArr); log.debug("Native call: " + commandStr); if (commandArr == null || commandArr.length == 0) { throw new NativeInterfaceException("Failed to parse params.", NativeInterfaceException.HttpCode.BAD_RQUEST); } ICommandRunner cmdRunner = new CommandRunner(); addedToQ = addToProcessQ(cmd, cmdRunner); log.debug("Start: " + new Date().getTime()); res = cmdRunner.exec(commandArr); log.debug("End: " + new Date().getTime()); if (res != null) { if (res.getExitStatus() == 0) { String stdOut = res.getStdout(); log.debug("stdout: " + stdOut); ret.put("stdout", stdOut); String warnings = ""; List<String> stdLines = IOUtils.readLines(new StringReader(stdOut)); for (String ln : stdLines) { if (ln.indexOf(" WARN ") > -1) { warnings += ln; warnings += "\n"; } // we will cap the maximum length of warnings to 1k if (warnings.length() > 1028) { warnings += " more .."; break; } } if (warnings.length() > 0) { System.out.println(stdOut); ret.put("warnings", warnings); } } else { String err = res.getStderr(); if (res.getExitStatus() == -9999) { throw new Exception("User requested termination."); } else { boolean doThrowException = true; if (cmd.containsKey("throwerror")) { doThrowException = Boolean.parseBoolean(cmd.get("throwerror").toString()); } if (doThrowException) { throw new Exception(err); } else { String stdOut = res.getStdout(); ret.put("stdout", stdOut); ret.put("stderr", err); } } } } } catch (Exception e) { if (res.getExitStatus() == -9999) { throw new NativeInterfaceException("Failed to execute." + e.getMessage(), NativeInterfaceException.HttpCode.USER_CANCEL); } else { throw new NativeInterfaceException("Failed to execute." + e.getMessage(), NativeInterfaceException.HttpCode.SERVER_ERROR); } } finally { if (addedToQ) { removeFromProcessQ(cmd); } } return ret; }
From source file:com.streamsets.datacollector.http.TestLogServlet.java
@Test public void testLogs() throws Exception { String baseLogUrl = startServer() + "/rest/v1/system/logs"; try {/*from w w w . j a va2s. com*/ HttpURLConnection conn = (HttpURLConnection) new URL(baseLogUrl + "/files").openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); Assert.assertTrue(conn.getContentType().startsWith("application/json")); List list = ObjectMapperFactory.get().readValue(conn.getInputStream(), List.class); Assert.assertEquals(2, list.size()); for (int i = 0; i < 2; i++) { Map map = (Map) list.get(i); String log = (String) map.get("file"); if (log.equals("test.log")) { Assert.assertEquals(logFile.lastModified(), (long) map.get("lastModified")); } else { Assert.assertEquals(oldLogFile.lastModified(), (long) map.get("lastModified")); } } conn = (HttpURLConnection) new URL(baseLogUrl + "/files/" + oldLogFile.getName()).openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); Assert.assertTrue(conn.getContentType().startsWith("text/plain")); List<String> lines = IOUtils.readLines(conn.getInputStream()); Assert.assertEquals(1, lines.size()); Assert.assertEquals("bye", lines.get(0)); } finally { stopServer(); } }
From source file:net.lyonlancer5.mcmp.karasu.util.VersionIdentifier.java
public void run() { if (!ModFileUtils.isDevEnv) { //MARKER: Do check whether the thread is restarted, just in case... if (!hasChecked) { //MARKER: First off, download a local copy of the cache try { LOGGER.info("Querying master repository"); ModFileUtils.download(remoteUpdateLoc, localUpdateLoc); } catch (IOException e) { LOGGER.warn("Failed to query remote repository"); isLatest = true;/*w w w . java 2 s . c om*/ hasChecked = true; return; } //MARKER: Load the cache and get the latest version try { LOGGER.info("Loading update cache"); //mcversion:<stable>:version:date_in_long:url //mcversion:<pre>:version:date_in_long:url List<String> lines = IOUtils.readLines(new FileInputStream(localUpdateLoc)); for (String s : lines) { String[] params = s.split(":"); if (params[0].equals(Loader.MC_VERSION)) { if (isPre && params[1].equals("pre")) { latestVersion = params[2]; date = Long.parseLong(params[3]); updateLink = params[4]; } else if (!isPre && params[1].equals("stable")) { latestVersion = params[2]; date = Long.parseLong(params[3]); updateLink = params[4]; } break; } } } catch (IOException e) { LOGGER.warn("Failed to load local cache", e); isLatest = true; hasChecked = true; return; } //MARKER: Then compare EACH Map<String, Integer> local = identify(Constants.VERSION), remote = identify(latestVersion); if (!isPre) { if (remote.get("major") > local.get("major") || remote.get("minor") > local.get("minor") || remote.get("revision") > local.get("revision") || remote.get("build") > local.get("build")) { LOGGER.info("A new update for Project Karasu is available!"); LOGGER.info("v" + latestVersion + " - released on " + ((new SimpleDateFormat("MM/dd/yyyy")).format(new Date(date)))); hasChecked = true; isLatest = false; return; } } else { if (remote.get("pre-release-id") > local.get("pre-release-id")) { LOGGER.info("A pre-release update has been released"); LOGGER.info("v" + latestVersion + " - released on " + ((new SimpleDateFormat("MM/dd/yyyy")).format(new Date(date)))); hasChecked = true; isLatest = false; return; } } isLatest = true; hasChecked = true; LOGGER.info("No updates found"); } } }