List of usage examples for org.apache.hadoop.fs FileSystem open
public FSDataInputStream open(PathHandle fd, int bufferSize) throws IOException
From source file:azkaban.viewer.hdfs.BsonFileViewer.java
License:Apache License
@Override public void displayFile(FileSystem fs, Path path, OutputStream outStream, int startLine, int endLine) throws IOException { FSDataInputStream in = null;/*from www . j a v a 2s . c om*/ try { in = fs.open(path, 16 * 1024 * 1024); long endTime = System.currentTimeMillis() + STOP_TIME; BasicBSONCallback callback = new BasicBSONCallback(); BasicBSONDecoder decoder = new BasicBSONDecoder(); /* * keep reading and rendering bsonObjects until one of these conditions is * met: * * a. we have rendered all bsonObjects desired. b. we have run out of * time. */ for (int lineno = 1; lineno <= endLine && System.currentTimeMillis() <= endTime; lineno++) { if (lineno < startLine) { continue; } callback.reset(); decoder.decode(in, callback); BSONObject value = (BSONObject) callback.get(); StringBuilder bldr = new StringBuilder(); bldr.append("\n\n Record "); bldr.append(lineno); bldr.append('\n'); JSON.serialize(value, bldr); outStream.write(bldr.toString().getBytes("UTF-8")); } } catch (IOException e) { outStream.write(("Error in display avro file: " + e.getLocalizedMessage()).getBytes("UTF-8")); } finally { if (in != null) { in.close(); } outStream.flush(); } }
From source file:azkaban.webapp.servlet.hdfsviewer.BsonFileViewer.java
License:Apache License
@Override public void displayFile(FileSystem fs, Path path, OutputStream outStream, int startLine, int endLine) throws IOException { FSDataInputStream in = null;/*from w w w. ja v a 2s. co m*/ try { in = fs.open(path, 16 * 1024 * 1024); long endTime = System.currentTimeMillis() + STOP_TIME; BasicBSONCallback callback = new BasicBSONCallback(); BasicBSONDecoder decoder = new BasicBSONDecoder(); /* * keep reading and rendering bsonObjects until one of these conditions is met: * * a. we have rendered all bsonObjects desired. * b. we have run out of time. */ for (int lineno = 1; lineno <= endLine && System.currentTimeMillis() <= endTime; lineno++) { if (lineno < startLine) { continue; } callback.reset(); decoder.decode(in, callback); BSONObject value = (BSONObject) callback.get(); StringBuilder bldr = new StringBuilder(); bldr.append("\n\n Record "); bldr.append(lineno); bldr.append('\n'); JSON.serialize(value, bldr); outStream.write(bldr.toString().getBytes("UTF-8")); } } catch (IOException e) { outStream.write(("Error in display avro file: " + e.getLocalizedMessage()).getBytes("UTF-8")); } finally { if (in != null) { in.close(); } outStream.flush(); } }
From source file:cc.solr.lucene.store.hdfs.HdfsFileReader.java
License:Apache License
public HdfsFileReader(FileSystem fileSystem, Path path, int bufferSize) throws IOException { if (!fileSystem.exists(path)) { throw new FileNotFoundException(path.toString()); }//w w w .j av a 2 s . co m FileStatus fileStatus = fileSystem.getFileStatus(path); _hdfsLength = fileStatus.getLen(); _inputStream = fileSystem.open(path, bufferSize); // read meta blocks _inputStream.seek(_hdfsLength - 16); int numberOfBlocks = _inputStream.readInt(); _length = _inputStream.readLong(); int version = _inputStream.readInt(); if (version != VERSION) { throw new RuntimeException("Version of file [" + version + "] does not match reader [" + VERSION + "]"); } _inputStream.seek(_hdfsLength - 16 - (numberOfBlocks * 24)); // 3 longs per // block _metaBlocks = new ArrayList<HdfsMetaBlock>(numberOfBlocks); for (int i = 0; i < numberOfBlocks; i++) { HdfsMetaBlock hdfsMetaBlock = new HdfsMetaBlock(); hdfsMetaBlock.readFields(_inputStream); _metaBlocks.add(hdfsMetaBlock); } seek(0); }
From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGet.java
License:Apache License
private InputStream getInput(FileSystem fs, FileStatus status) throws IOException { if (RuntimeContext.get().isSimulation()) { return new VoidInputStream(); } else {/*www .java 2 s . c o m*/ return fs.open(status.getPath(), BUFFER_SIZE); } }
From source file:com.cloudera.CacheTool.java
License:Apache License
public static void main(String[] args) throws Exception { conf = new Configuration(); conf.addResource(new Path("/home/james/hdfs-conf/hdfs-site.xml")); conf.addResource(new Path("/home/james/hdfs-conf/core-site.xml")); URI uri = FileSystem.getDefaultUri(conf); final FileSystem fs = FileSystem.get(uri, conf); for (int i = 0; i < 8000; i += 10) { final int i_copy = i; pool.submit(new Runnable() { public void run() { for (int j = 0; j < 10; j++) { try { createFile(fs, new Path("/home/james/large" + (i_copy + j)), 1024 * 1024); } catch (IOException ioe) { System.out.println(ioe); }/*from ww w . j a v a 2s . c o m*/ } } }); } pool.shutdown(); pool.awaitTermination(1, TimeUnit.DAYS); long start = Time.monotonicNow(); Random r = new Random(0); for (int i = 0; i < 100; i++) { FSDataInputStream fdis = fs.open(new Path("/home/james/large" + r.nextInt(8000)), 512); byte[] buffer = new byte[512]; for (int j = 0; j < 100; j++) { int offset = r.nextInt(1024 * 1024 - 511); fdis.read(offset, buffer, 0, 512); } } System.out.println("Time taken for 10000 random 512 byte reads: " + (Time.monotonicNow() - start) / 1000.0); }
From source file:com.cloudera.hoop.fs.FSOpen.java
License:Open Source License
/** * Executes the filesystem operation.//from w w w .ja v a2s. c o m * * @param fs filesystem instance to use. * @return The inputstream of the file. * @throws IOException thrown if an IO error occured. */ @Override public InputStream execute(FileSystem fs) throws IOException { int bufferSize = HoopServer.get().getConfig().getInt("hoop.buffer.size", 4096); return fs.open(path, bufferSize); }
From source file:com.cloudera.integration.oracle.goldengate.ldv.mapreduce.lib.input.LengthDelimitedRecordReader.java
@Override public void initialize(InputSplit is, TaskAttemptContext tac) throws IOException, InterruptedException { this.fileSplit = (FileSplit) is; sizeRecordLength = tac.getConfiguration().getInt(Constants.RECORD_PREFIX_LENGTH, -1); Preconditions.checkArgument(sizeRecordLength > 0, Constants.RECORD_PREFIX_LENGTH + " must be configured."); Preconditions.checkArgument(sizeRecordLength == 2 || sizeRecordLength == 4 || sizeRecordLength == 8, Constants.RECORD_PREFIX_LENGTH + " must be either 2, 4, or 8."); recordLengthBuffer = new byte[sizeRecordLength]; sizeFieldLength = tac.getConfiguration().getInt(Constants.FIELD_PREFIX_LENGTH, -1); Preconditions.checkArgument(sizeFieldLength > 0, Constants.FIELD_PREFIX_LENGTH + " must be configured."); Preconditions.checkArgument(sizeFieldLength == 2 || sizeFieldLength == 4 || sizeFieldLength == 8, Constants.FIELD_PREFIX_LENGTH + " must be either 2, 4, or 8."); fieldLengthBuffer = new byte[sizeFieldLength]; FileSystem fileSystem = this.fileSplit.getPath().getFileSystem(tac.getConfiguration()); int inputBufferSize = tac.getConfiguration().getInt(Constants.INPUT_BUFFER_SIZE, 5 * 1024 * 1024); this.inputStream = fileSystem.open(this.fileSplit.getPath(), inputBufferSize); }
From source file:com.github.joshelser.accumulo.DelimitedIngest.java
License:Apache License
private void processSinglePathWithByteBuffer(BatchWriter writer, FileMapping mapping, Path p, CsvParser parser) throws IOException, MutationsRejectedException { final FileSystem fs = p.getFileSystem(conf); FSDataInputStream dis = fs.open(p, INPUT_BUFFER_SIZE); InputStreamReader reader = new InputStreamReader(dis, UTF_8); try {/*from w w w.j a v a 2s .co m*/ parser.beginParsing(reader); String[] line = null; while ((line = parser.parseNext()) != null) { writer.addMutation(parseLine(mapping, line)); } } finally { if (null != reader) { reader.close(); } } }
From source file:com.lightboxtechnologies.spectrum.BlockHashMapper.java
License:Apache License
void openImgFile(Path p, FileSystem fs) throws IOException { if (ImgFile != null && p.equals(ImgPath)) { return;/*from ww w.ja v a 2s . c om*/ } IOUtils.closeQuietly(ImgFile); ImgPath = p; ImgFile = fs.open(p, 512 * 1024); }
From source file:com.lightboxtechnologies.spectrum.ExtractDataMapper.java
License:Apache License
void openImgFile(Path p, FileSystem fs) throws IOException { if (ImgFile != null && p.equals(ImgPath)) { return;/* w w w . java 2 s. c om*/ } IOUtils.closeQuietly(ImgFile); ImgPath = p; ImgFile = fs.open(p, 64 * 1024 * 1024); }