List of usage examples for org.apache.hadoop.fs FSDataInputStream read
@Override
public int read(ByteBuffer buf) throws IOException
From source file:com.google.cloud.hadoop.fs.gcs.HadoopFileSystemIntegrationHelper.java
License:Open Source License
/** * Helper which reads the entire file as a String. *///w w w . ja v a 2 s . c o m protected String readTextFile(Path hadoopPath) throws IOException { FSDataInputStream readStream = null; byte[] readBuffer = new byte[1024]; StringBuffer returnBuffer = new StringBuffer(); try { readStream = ghfs.open(hadoopPath, GoogleHadoopFileSystemBase.BUFFERSIZE_DEFAULT); int numBytesRead = readStream.read(readBuffer); while (numBytesRead > 0) { returnBuffer.append(new String(readBuffer, 0, numBytesRead, StandardCharsets.UTF_8)); numBytesRead = readStream.read(readBuffer); } } finally { if (readStream != null) { readStream.close(); } } return returnBuffer.toString(); }
From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java
License:Apache License
public void readSequentialDirect() throws Exception { System.out.println("reading sequential file in direct mode " + path); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); FileStatus status = fs.getFileStatus(path); FSDataInputStream instream = fs.open(path); ByteBuffer buf = ByteBuffer.allocateDirect(size); buf.clear();/*from ww w . j ava2 s .c o m*/ double sumbytes = 0; double ops = 0; System.out.println("file capacity " + status.getLen()); System.out.println("read size " + size); System.out.println("operations " + loop); long start = System.currentTimeMillis(); while (ops < loop) { buf.clear(); double ret = (double) instream.read(buf); if (ret > 0) { sumbytes = sumbytes + ret; ops = ops + 1.0; } else { ops = ops + 1.0; if (instream.getPos() == 0) { break; } else { instream.seek(0); } } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1024.0 / 1024.0; latency = 1000000.0 * executionTime / ops; } System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); System.out.println("closing stream"); instream.close(); fs.close(); }
From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java
License:Apache License
public void readRandomDirect() throws Exception { System.out.println("reading random file in direct mode " + path); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); FileStatus status = fs.getFileStatus(path); FSDataInputStream instream = fs.open(path); ByteBuffer buf = ByteBuffer.allocateDirect(size); buf.clear();/*from w w w . j a v a2 s . co m*/ double sumbytes = 0; double ops = 0; long _range = status.getLen() - ((long) buf.capacity()); double range = (double) _range; Random random = new Random(); System.out.println("file capacity " + status.getLen()); System.out.println("read size " + size); System.out.println("operations " + loop); long start = System.currentTimeMillis(); while (ops < loop) { buf.clear(); double _offset = range * random.nextDouble(); long offset = (long) _offset; instream.seek(offset); double ret = (double) instream.read(buf); if (ret > 0) { sumbytes = sumbytes + ret; ops = ops + 1.0; } else { break; } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1024.0 / 1024.0; latency = 1000000.0 * executionTime / ops; } System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); System.out.println("closing stream"); instream.close(); fs.close(); }
From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java
License:Apache License
void keyGet() throws Exception { System.out.println("key get, path " + path + ", size " + size + ", loop " + loop); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path[] paths = new Path[loop]; for (int i = 0; i < loop; i++) { String child = "" + i; paths[i] = new Path(path, child); System.out.println("path " + paths[i]); }/*from w w w.j a v a2 s.c o m*/ byte[] outBuf = new byte[size]; for (Path p : paths) { FSDataOutputStream outputStream = fs.create(p); outputStream.write(outBuf); outputStream.close(); } long start = System.currentTimeMillis(); ByteBuffer inBuf = ByteBuffer.allocateDirect(size); for (int i = 0; i < loop; i++) { Path p = paths[i]; FSDataInputStream inputStream = fs.open(p); inBuf.clear(); while (inBuf.remaining() > 0) { inputStream.read(inBuf); } inputStream.close(); } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)); double latency = executionTime * 1000.0 / ((double) loop); System.out.println("execution time [ms] " + executionTime); System.out.println("latency [us] " + latency); fs.close(); }
From source file:com.ibm.stocator.fs.swift2d.systemtests.TestSwiftFileSystemExtendedContract.java
License:Apache License
@Test(timeout = SwiftTestConstants.SWIFT_TEST_TIMEOUT) public void testWriteReadFile() throws Exception { final Path f = new Path(getBaseURI() + "/test/test"); final FSDataOutputStream fsDataOutputStream = sFileSystem.create(f); final String message = "Test string"; fsDataOutputStream.write(message.getBytes()); fsDataOutputStream.close();/*w w w .j a va2s. c o m*/ assertExists("created file", f); FSDataInputStream open = null; try { open = sFileSystem.open(f); final byte[] bytes = new byte[512]; final int read = open.read(bytes); final byte[] buffer = new byte[read]; System.arraycopy(bytes, 0, buffer, 0, read); assertEquals(message, new String(buffer)); } finally { sFileSystem.delete(f, false); IOUtils.closeStream(open); } }
From source file:com.inmobi.conduit.local.LocalStreamService.java
License:Apache License
private boolean isEmptyFile(FileStatus fileStatus, FileSystem fs) { boolean retVal = false; FSDataInputStream in = null; try {//from www . j a v a 2 s.c o m in = fs.open(fileStatus.getPath()); byte[] data = new byte[1]; // try reading 1 byte int bytesRead = in.read(data); if (bytesRead == 1) { // not empty file retVal = false; } else { // not able to read 1 bytes also then empty file retVal = true; } } catch (IOException e) { LOG.error("Unable to find if file is empty or not [" + fileStatus.getPath() + "]", e); } finally { if (in != null) { try { in.close(); } catch (IOException e1) { LOG.error("Error in closing file [" + fileStatus.getPath() + "]", e1); } } } return retVal; }
From source file:com.inmobi.databus.local.LocalStreamService.java
License:Apache License
private boolean isEmptyFile(FileStatus fileStatus, FileSystem fs) { boolean retVal = false; FSDataInputStream in = null; try {/* w w w . ja va 2 s . c o m*/ in = fs.open(fileStatus.getPath()); byte[] data = new byte[1]; //try reading 1 byte int bytesRead = in.read(data); if (bytesRead == 1) { //not empty file retVal = false; } else { //not able to read 1 bytes also then empty file retVal = true; } } catch (IOException e) { LOG.error("Unable to find if file is empty or not [" + fileStatus.getPath() + "]", e); } finally { if (in != null) { try { in.close(); } catch (IOException e1) { LOG.error("Error in closing file [" + fileStatus.getPath() + "]", e1); } } } return retVal; }
From source file:com.liveramp.hank.hadoop.HadoopTestCase.java
License:Apache License
protected String getContents(FileSystem fs, String path) throws IOException { FSDataInputStream in = fs.open(new Path(path)); StringBuilder builder = new StringBuilder(); byte[] buffer = new byte[1024]; int bytesRead; while ((bytesRead = in.read(buffer)) > 0) { builder.append(new String(buffer, 0, bytesRead)); }//from w ww. j a v a2 s.c o m in.close(); return builder.toString(); }
From source file:com.mvdb.etl.actions.ActionUtils.java
License:Apache License
public static String getActionChainBrokenCause() throws IOException { String hdfsHome = getConfigurationValue(ConfigurationKeys.GLOBAL_CUSTOMER, ConfigurationKeys.GLOBAL_HADOOP_HOME); org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); conf.addResource(new Path(hdfsHome + "/conf/core-site.xml")); FileSystem hdfsFileSystem = FileSystem.get(conf); String actionChainStatusFile = getConfigurationValue(ConfigurationKeys.GLOBAL_CUSTOMER, ConfigurationKeys.GLOBAL_ACTION_CHAIN_STATUS_FILE); String actionChainStatusFileName = /* hdfsHome + */File.separator + actionChainStatusFile; Path actionChainStatusFilePath = new Path(actionChainStatusFileName); if (hdfsFileSystem.exists(actionChainStatusFilePath)) { FSDataInputStream in = hdfsFileSystem.open(actionChainStatusFilePath); int bytesRead = -1; byte[] buffer = new byte[1024]; ByteArrayOutputStream baos = new ByteArrayOutputStream(); while ((bytesRead = in.read(buffer)) > 0) { baos.write(buffer, 0, bytesRead); }//from w w w .java2 s .co m return new String(baos.toByteArray()); } return null; }
From source file:com.rockstor.core.io.ChunkReader.java
License:Apache License
public static Chunk readHeader(FSDataInputStream input, long offset) throws IllegalArgumentException, IOException { if (offset == 0) { throw new IllegalArgumentException("position Error, param offset=" + offset); }//from ww w. j av a 2 s.co m align_read(input, offset); Chunk chunk = new Chunk(); chunk.setOffset(input.getPos()); byte[] rockID = new byte[Rock.ROCK_ID_LEN]; if (Rock.ROCK_ID_LEN != input.read(rockID)) { throw new IOException("read rock magic failed!"); } chunk.setRockID(rockID); byte[] chunkPrefix = new byte[Chunk.CHUNK_ID_LEN]; if (Chunk.CHUNK_ID_LEN != input.read(chunkPrefix)) { throw new IOException("read chunk ID failed!"); } chunk.setChunkPrefix(chunkPrefix); chunk.setTimestamp(input.readLong()); chunk.setSize(input.readLong()); chunk.setPartID(input.readShort()); chunk.setSeqID(input.readShort()); return chunk; }