List of usage examples for org.apache.hadoop.fs FileSystem open
public FSDataInputStream open(PathHandle fd) throws IOException
From source file:com.linkedin.thirdeye.hadoop.push.SegmentPushPhase.java
License:Apache License
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(".tar.gz")) { return;// w ww. j a v a2 s . c o m } long length = fs.getFileStatus(path).getLen(); for (String host : hosts) { InputStream inputStream = null; try { inputStream = fs.open(path); fileName = fileName.split(".tar")[0]; if (fileName.lastIndexOf(ThirdEyeConstants.SEGMENT_JOINER) != -1) { segmentName = fileName.substring(0, fileName.lastIndexOf(ThirdEyeConstants.SEGMENT_JOINER)); } LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, port); try { int responseCode = FileUploadUtils.sendSegmentFile(host, port, fileName, inputStream, length); LOGGER.info("Response code: {}", responseCode); if (uploadSuccess == true && responseCode != 200) { uploadSuccess = false; } } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, port); LOGGER.error("Caught exception during upload", e); throw new RuntimeException("Got Error during send tar files to push hosts!"); } } finally { inputStream.close(); } } }
From source file:com.linkedin.thirdeye.hadoop.util.ThirdeyeAvroUtils.java
License:Apache License
private static DataFileStream<GenericRecord> getAvroReader(Path avroFile) throws IOException { FileSystem fs = FileSystem.get(new Configuration()); if (avroFile.getName().endsWith("gz")) { return new DataFileStream<GenericRecord>(new GZIPInputStream(fs.open(avroFile)), new GenericDatumReader<GenericRecord>()); } else {/*from w w w . j a va 2 s. co m*/ return new DataFileStream<GenericRecord>(fs.open(avroFile), new GenericDatumReader<GenericRecord>()); } }
From source file:com.linkedin.thirdeye.hadoop.util.ThirdeyePinotSchemaUtils.java
License:Apache License
public static Schema createSchema(String configPath) throws IOException { FileSystem fs = FileSystem.get(new Configuration()); ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.decode(fs.open(new Path(configPath))); LOGGER.info("{}", thirdeyeConfig); return createSchema(thirdeyeConfig); }
From source file:com.liveramp.cascading_ext.bloom.BloomFilter.java
License:Apache License
public static BloomFilter read(FileSystem fs, Path path) throws IOException { FSDataInputStream inputStream = fs.open(path); BloomFilter bf = new BloomFilter(); bf.readFields(inputStream);/*w ww.j a v a 2 s .c o m*/ inputStream.close(); return bf; }
From source file:com.liveramp.hank.hadoop.HadoopTestCase.java
License:Apache License
protected String getContents(FileSystem fs, String path) throws IOException { FSDataInputStream in = fs.open(new Path(path)); StringBuilder builder = new StringBuilder(); byte[] buffer = new byte[1024]; int bytesRead; while ((bytesRead = in.read(buffer)) > 0) { builder.append(new String(buffer, 0, bytesRead)); }// w w w. j av a2 s . com in.close(); return builder.toString(); }
From source file:com.lucidworks.hadoop.utils.ZipFileRecordReader.java
License:Apache License
public ZipFileRecordReader(FileSplit fileSplit, Configuration conf) throws IOException { this.fileSplit = fileSplit; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(conf); //Open the stream if (CompressionHelper.isCompressed(path)) { compressedInputStream = CompressionHelper.openCompressedFile(path, conf); zip = new ZipInputStream(compressedInputStream); } else {// w w w. j a v a 2 s . c om fsin = fs.open(path); zip = new ZipInputStream(fsin); } }
From source file:com.mapred.EmplRecordReader.java
@Override public void initialize(InputSplit is, TaskAttemptContext tac) throws IOException, InterruptedException { FileSplit split = (FileSplit) is;//from ww w . j ava2s.com position = 0L; end = split.getLength(); Configuration job = tac.getConfiguration(); final Path file = split.getPath(); FileSystem fs = file.getFileSystem(job); FSDataInputStream fileIn = fs.open(split.getPath()); in = new LineRecordReader(); in.initialize(is, tac); }
From source file:com.marcolotz.lung.io.inputFormat.MultipleFilesRecordReader.java
License:Creative Commons License
/** * <p>/* w ww . j ava 2 s .c o m*/ * If the file has not already been read, this reads it into memory, so that * a call to getCurrentValue() will return the entire contents of this file * as Text, and getCurrentKey() will return the qualified path to this file * as Text. Then, returns true. If it has already been read, then returns * false without updating any internal state. * </p> * * @return Whether the file was read or not. * @throws IOException * if there is an error reading the file. * @throws InterruptedException * if there is an error. */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!isProcessed) { if (mFileLength > (long) Integer.MAX_VALUE) { throw new IOException("File is longer than Integer.MAX_VALUE."); } byte[] contents = new byte[(int) mFileLength]; FileSystem fs = mFileToRead.getFileSystem(mConf); FSDataInputStream in = null; try { // Set the contents of this file. in = fs.open(mFileToRead); IOUtils.readFully(in, contents, 0, contents.length); fileContent.set(contents, 0, contents.length); } finally { IOUtils.closeStream(in); } isProcessed = true; return true; } return false; }
From source file:com.marcolotz.lung.io.inputFormat.WholeFileRecordReader.java
License:Creative Commons License
@Override public boolean nextKeyValue() throws IOException, InterruptedException { /* if that record reader for that input split was not called yet */ if (!processed) { byte[] contents = new byte[(int) fileSplit.getLength()]; Path file = fileSplit.getPath(); // Reads from the conf file what is the desired file system. FileSystem fs = file.getFileSystem(conf); FSDataInputStream in = null;//from w w w . ja v a 2 s . co m try { in = fs.open(file); IOUtils.readFully(in, contents, 0, contents.length); value.set(contents, 0, contents.length); } finally { IOUtils.closeStream(in); } processed = true; return true; } return false; }
From source file:com.marklogic.contentpump.CombineDocumentReader.java
License:Apache License
@SuppressWarnings({ "unchecked" })
@Override/*from w w w .j av a 2 s .c o m*/
public boolean nextKeyValue() throws IOException, InterruptedException {
while (iterator.hasNext()) {
FileSplit split = iterator.next();
setFile(split.getPath());
String uri = makeURIFromPath(file);
FileSystem fs = file.getFileSystem(context.getConfiguration());
FSDataInputStream fileIn = fs.open(file);
long splitLength = split.getLength();
if (splitLength > (long) Integer.MAX_VALUE) {
setSkipKey(0, 0, "file size too large: " + splitLength);
return true;
}
if (setKey(uri, 0, 0, true)) {
return true;
}
byte[] buf = new byte[(int) splitLength];
try {
fileIn.readFully(buf);
if (value instanceof Text) {
((Text) value).set(new String(buf, encoding));
} else {
if (batchSize > 1) {
// Copy data since XCC won't do it when Content is
// created.
value = (VALUEIN) new BytesWritable(buf);
} else {
((BytesWritable) value).set(buf, 0, buf.length);
}
}
bytesRead += buf.length;
return true;
} catch (IOException e) {
LOG.error(e);
throw e;
} finally {
fileIn.close();
}
}
return false;
}