List of usage examples for java.io BufferedReader lines
public Stream<String> lines()
From source file:Main.java
public static void main(String[] argv) throws Exception { BufferedReader reader = Files.newBufferedReader(Paths.get("a.bat"), StandardCharsets.UTF_8); //BufferedReader's lines methods returns a stream of all lines reader.lines().forEach(System.out::println); }
From source file:org.apache.pulsar.io.file.utils.ZipFiles.java
/** * Get a lazily loaded stream of lines from a gzipped file, similar to * {@link Files#lines(java.nio.file.Path)}. * * @param path//from w ww. j a v a 2 s . com * The path to the zipped file. * @return stream with lines. */ public static Stream<String> lines(Path path) { ZipInputStream zipStream = null; try { zipStream = new ZipInputStream(Files.newInputStream(path)); } catch (IOException e) { closeSafely(zipStream); throw new UncheckedIOException(e); } // Reader decoder = new InputStreamReader(gzipStream, Charset.defaultCharset()); BufferedReader reader = new BufferedReader(new InputStreamReader(zipStream)); return reader.lines().onClose(() -> closeSafely(reader)); }
From source file:org.apache.pulsar.io.file.utils.GZipFiles.java
/** * Get a lazily loaded stream of lines from a gzipped file, similar to * {@link Files#lines(java.nio.file.Path)}. * * @param path// ww w . jav a 2s . co m * The path to the gzipped file. * @return stream with lines. */ public static Stream<String> lines(Path path) { GZIPInputStream gzipStream = null; try { gzipStream = new GZIPInputStream(Files.newInputStream(path)); } catch (IOException e) { closeSafely(gzipStream); throw new UncheckedIOException(e); } BufferedReader reader = new BufferedReader(new InputStreamReader(gzipStream)); return reader.lines().onClose(() -> closeSafely(reader)); }
From source file:com.fizzed.blaze.util.Streamables.java
static public Stream<String> lines(StreamableInput input) { BufferedReader br = new BufferedReader(new InputStreamReader(input.stream)); try {/*from w ww . j a va2 s.co m*/ return br.lines().onClose(asUncheckedRunnable(br)); } catch (Error | RuntimeException e) { try { br.close(); } catch (IOException ex) { try { e.addSuppressed(ex); } catch (Throwable ignore) { } } throw e; } }
From source file:edu.umd.umiacs.clip.tools.io.GZIPFiles.java
protected static Stream<String> lines(Path path) throws IOException { BufferedReader br = null; try {// w w w.ja va 2 s .c om br = new BufferedReader(new InputStreamReader( new GZIPInputStream(new BufferedInputStream(newInputStream(path), BUFFER_SIZE)), UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); } catch (IOException e) { try { br.close(); } catch (Exception ex) { try { e.addSuppressed(ex); } catch (Throwable ignore) { } } throw e; } }
From source file:edu.umd.umiacs.clip.tools.io.BZIP2Files.java
protected static Stream<String> lines(Path path) throws IOException { BufferedReader br = null; try {/*from w w w. j ava 2 s. c o m*/ br = new BufferedReader(new InputStreamReader( new BZip2CompressorInputStream(new BufferedInputStream(newInputStream(path), BUFFER_SIZE)), UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); } catch (IOException e) { try { br.close(); } catch (Exception ex) { try { e.addSuppressed(ex); } catch (Throwable ignore) { } } throw e; } }
From source file:edu.umd.umiacs.clip.tools.io.AllFiles.java
public static Stream<String> lines(InputStream is) { BufferedReader br = new BufferedReader( new InputStreamReader(is, UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); }
From source file:edu.umd.umiacs.clip.tools.io.AllFiles.java
public static Stream<String> lines() { BufferedReader br = new BufferedReader(new InputStreamReader( new BufferedInputStream(System.in, BUFFER_SIZE), UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); }
From source file:org.apache.hadoop.hive.ql.MetaStoreDumpUtility.java
public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf conf, String tmpBaseDir) { Connection conn = null;// w w w . j a va2s.co m try { Properties props = new Properties(); // connection properties props.put("user", conf.get("javax.jdo.option.ConnectionUserName")); props.put("password", conf.get("javax.jdo.option.ConnectionPassword")); String url = conf.get("javax.jdo.option.ConnectionURL"); conn = DriverManager.getConnection(url, props); ResultSet rs = null; Statement s = conn.createStatement(); if (LOG.isDebugEnabled()) { LOG.debug("Connected to metastore database "); } String mdbPath = HiveTestEnvSetup.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/"; // Setup the table column stats BufferedReader br = new BufferedReader(new FileReader(new File( HiveTestEnvSetup.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql"))); String command; s.execute("DROP TABLE APP.TABLE_PARAMS"); s.execute("DROP TABLE APP.TAB_COL_STATS"); // Create the column stats table while ((command = br.readLine()) != null) { if (!command.endsWith(";")) { continue; } if (LOG.isDebugEnabled()) { LOG.debug("Going to run command : " + command); } PreparedStatement psCommand = conn.prepareStatement(command.substring(0, command.length() - 1)); psCommand.execute(); psCommand.close(); if (LOG.isDebugEnabled()) { LOG.debug("successfully completed " + command); } } br.close(); java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TAB_COL_STATS.txt.bz2"); java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2"); // Set up the foreign key constraints properly in the TAB_COL_STATS data java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt"); java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt"); class MyComp implements Comparator<String> { @Override public int compare(String str1, String str2) { if (str2.length() != str1.length()) { return str2.length() - str1.length(); } return str1.compareTo(str2); } } final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp()); rs = s.executeQuery("SELECT * FROM APP.TBLS"); while (rs.next()) { String tblName = rs.getString("TBL_NAME"); Integer tblId = rs.getInt("TBL_ID"); tableNameToID.put(tblName, tblId); if (LOG.isDebugEnabled()) { LOG.debug("Resultset : " + tblName + " | " + tblId); } } final Map<String, Map<String, String>> data = new HashMap<>(); rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from " + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a" + " join APP.TBLS on TBLS.SD_ID = a.SD_ID"); while (rs.next()) { String tblName = rs.getString(1); String colName = rs.getString(2); String typeName = rs.getString(3); Map<String, String> cols = data.get(tblName); if (null == cols) { cols = new HashMap<>(); } cols.put(colName, typeName); data.put(tblName, cols); } BufferedReader reader = new BufferedReader(new InputStreamReader( new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ)))); Stream<String> replaced = reader.lines().parallel().map(str -> { String[] splits = str.split(","); String tblName = splits[0]; String colName = splits[1]; Integer tblID = tableNameToID.get(tblName); StringBuilder sb = new StringBuilder( "default@" + tblName + "@" + colName + "@" + data.get(tblName).get(colName) + "@"); for (int i = 2; i < splits.length; i++) { sb.append(splits[i] + "@"); } // Add tbl_id and empty bitvector return sb.append(tblID).append("@").toString(); }); Files.write(tmpFileLoc1, (Iterable<String>) replaced::iterator); replaced.close(); reader.close(); BufferedReader reader2 = new BufferedReader(new InputStreamReader( new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ)))); final Map<String, String> colStats = new ConcurrentHashMap<>(); Stream<String> replacedStream = reader2.lines().parallel().map(str -> { String[] splits = str.split("_@"); String tblName = splits[0]; Integer tblId = tableNameToID.get(tblName); Map<String, String> cols = data.get(tblName); StringBuilder sb = new StringBuilder(); sb.append("{\"COLUMN_STATS\":{"); for (String colName : cols.keySet()) { sb.append("\"" + colName + "\":\"true\","); } sb.append("},\"BASIC_STATS\":\"true\"}"); colStats.put(tblId.toString(), sb.toString()); return tblId.toString() + "@" + splits[1]; }); Files.write(tmpFileLoc2, (Iterable<String>) replacedStream::iterator); Files.write(tmpFileLoc2, (Iterable<String>) colStats.entrySet().stream() .map(map -> map.getKey() + "@COLUMN_STATS_ACCURATE@" + map.getValue())::iterator, StandardOpenOption.APPEND); replacedStream.close(); reader2.close(); // Load the column stats and table params with 30 TB scale String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" + "', '" + tmpFileLoc1.toAbsolutePath().toString() + "', '@', null, 'UTF-8', 1)"; String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" + "', '" + tmpFileLoc2.toAbsolutePath().toString() + "', '@', null, 'UTF-8', 1)"; PreparedStatement psImport1 = conn.prepareStatement(importStatement1); if (LOG.isDebugEnabled()) { LOG.debug("Going to execute : " + importStatement1); } psImport1.execute(); psImport1.close(); if (LOG.isDebugEnabled()) { LOG.debug("successfully completed " + importStatement1); } PreparedStatement psImport2 = conn.prepareStatement(importStatement2); if (LOG.isDebugEnabled()) { LOG.debug("Going to execute : " + importStatement2); } psImport2.execute(); psImport2.close(); if (LOG.isDebugEnabled()) { LOG.debug("successfully completed " + importStatement2); } s.execute("ALTER TABLE APP.TAB_COL_STATS ADD COLUMN CAT_NAME VARCHAR(256)"); s.execute("update APP.TAB_COL_STATS set CAT_NAME = '" + Warehouse.DEFAULT_CATALOG_NAME + "'"); s.close(); conn.close(); } catch (Exception e) { throw new RuntimeException("error while loading tpcds metastore dump", e); } }
From source file:de.tudarmstadt.ukp.dkpro.core.mallet.internal.wordembeddings.MalletEmbeddingsUtils.java
/** * Read embeddings in text format from an InputStream. * Each line is expected to have a whitespace-separated list {@code <token> <value1> <value2> ...}. * * @param inputStream an {@link InputStream} * @param hasHeader if true, read size and dimensionality from the first line * @return a {@code Map<String, double[]>} mapping each token to a vector. * @throws IOException if the input file cannot be read *//* w w w. ja v a 2 s.com*/ public static Map<String, double[]> readEmbeddingFileTxt(InputStream inputStream, boolean hasHeader) throws IOException { BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); final int dimensions; final int size; if (hasHeader) { String[] header = reader.readLine().split(" "); assert header.length == 2; size = Integer.parseInt(header[0]); dimensions = Integer.parseInt(header[1]); } else { dimensions = -1; size = -1; } Map<String, double[]> embeddings = reader.lines().map(MalletEmbeddingsUtils::lineToEmbedding) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); reader.close(); /* assert size and dimension */ if (hasHeader) { /* check that size read matches header information */ LOG.debug("Checking number and vector sizes for all embeddings."); assert size == embeddings.size(); assert embeddings.values().stream().allMatch(vector -> dimensions == vector.length); } else { LOG.debug("Checking vector sizes for all embeddings."); int firstLength = embeddings.values().stream().findAny().get().length; assert embeddings.values().stream().allMatch(vector -> firstLength == vector.length); } return embeddings; }