List of usage examples for org.apache.hadoop.fs FileContext getFileContext
public static FileContext getFileContext(final Configuration aConf) throws UnsupportedFileSystemException
From source file:co.cask.cdap.common.guice.FileContextProvider.java
License:Apache License
@Override public FileContext get() { return createUGI().doAs(new PrivilegedAction<FileContext>() { @Override//from www. java 2 s. c o m public FileContext run() { try { return FileContext.getFileContext(hConf); } catch (Exception e) { throw Throwables.propagate(e); } } }); }
From source file:com.alibaba.jstorm.hdfs.blobstore.HdfsBlobStoreFile.java
License:Apache License
@Override public void commit() throws IOException { checkIsNotTmp();//from w w w .jav a2 s . c o m // FileContext supports atomic rename, whereas FileSystem doesn't FileContext fc = FileContext.getFileContext(_hadoopConf); Path dest = new Path(_path.getParent(), BLOBSTORE_DATA_FILE); if (_mustBeNew) { fc.rename(_path, dest); } else { fc.rename(_path, dest, Options.Rename.OVERWRITE); } // Note, we could add support for setting the replication factor }
From source file:com.btoddb.chronicle.apps.AvroTools.java
License:Open Source License
private void echoFile(Path inFile) throws IOException { FileContext context = FileContext.getFileContext(hdfsConfig); AvroFSInput input = new AvroFSInput(context, inFile); ReflectDatumReader<StorableAvroEvent> reader = new ReflectDatumReader<>(StorableAvroEvent.class); FileReader<StorableAvroEvent> fileReader = DataFileReader.openReader(input, reader); long count = 0; try {/*from w w w.ja v a 2 s .c o m*/ Schema schema = fileReader.getSchema(); for (StorableAvroEvent event : fileReader) { count++; System.out.println("event -> " + event.toString()); } } finally { fileReader.close(); } System.out.println("count = " + count); }
From source file:com.btoddb.chronicle.apps.AvroTools.java
License:Open Source License
private void testFileAndFix(Path inFile) throws IOException { FileContext context = FileContext.getFileContext(hdfsConfig); AvroFSInput input = new AvroFSInput(context, inFile); ReflectDatumReader<Object> reader = new ReflectDatumReader<>(); FileReader<Object> fileReader = DataFileReader.openReader(input, reader); Path outFile = inFile.suffix(".fixing"); FSDataOutputStream output = FileSystem.create(outFile.getFileSystem(hdfsConfig), outFile, FsPermission.getDefault());//w ww . j a va 2 s . c om DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>()); writer.setCodec(CodecFactory.snappyCodec()); boolean corrupted = false; long count = 0; try { Schema schema = fileReader.getSchema(); writer.create(schema, output); for (;;) { try { if (fileReader.hasNext()) { Object obj = fileReader.next(); count++; writer.append(obj); } else { break; } } catch (AvroRuntimeException e) { corrupted = true; System.out.println(" - file pointer = " + input.tell()); if (e.getCause() instanceof EOFException) { System.out.println(" - EOF occurred so we're done : " + e.getMessage()); break; } else if (e.getCause() instanceof IOException) { System.out.println(" - will try to 'next' past the error : " + e.getMessage()); try { fileReader.next(); System.out.println(" - 'next' worked - didn't really expect it to, but great!"); } catch (Exception e2) { System.out.println(" - 'next' did not work - will continue on and see what happens : " + e2.getMessage()); } continue; } break; } catch (Exception e) { corrupted = true; System.out.println(" - file pointer = " + input.tell()); e.printStackTrace(); break; } } } catch (Exception e) { e.printStackTrace(); } finally { System.out.println((" - processed " + count + " records")); if (null != fileReader) { try { fileReader.close(); } catch (Exception e) { e.printStackTrace(); } } if (null != writer) { try { writer.close(); } catch (Exception e) { e.printStackTrace(); } } } if (!corrupted) { outFile.getFileSystem(hdfsConfig).delete(outFile, false); } else { outFile.getFileSystem(hdfsConfig).rename(outFile, inFile.suffix(".fixed")); } }
From source file:com.cloudera.RenameTest.java
License:Apache License
public static void testFileContextRename(URI uri) throws Exception { FileContext fc = FileContext.getFileContext(uri); Path testDir = new Path(new Path(uri), "testdir"); System.out.println("mkdir " + testDir); fc.mkdir(testDir, new FsPermission((short) 0755), true); Path testFile = new Path(new Path(uri), "testfile"); System.out.println("create " + testFile); FSDataOutputStream fos = fc.create(testFile, EnumSet.of(CREATE)); fos.close();/*from w w w . j a va 2 s . co m*/ System.out.println("rename " + testFile + " -> " + testDir); fc.rename(testFile, testDir); }
From source file:com.conductor.hadoop.WritableValueInputFormat.java
License:Apache License
/** * Writes the provided {@code values} to an input file to be read by the {@link Job}, and sets up all additional * necessary configuration.// www .jav a 2s . co m * * @param values * the values to be read by the job. * @param clazz * the type of the values. * @param inputsPerSplit * how man inputs each split gets * @param job * the job to configure * @param <V> * the type of the {@code values} * @throws IOException */ public static <V extends Writable> void setupInput(final List<V> values, Class<V> clazz, final int inputsPerSplit, final Job job) throws IOException { final Path inputPath = new Path("job_input_" + System.currentTimeMillis() + UUID.randomUUID().toString()); final Writer writer = SequenceFile.createWriter(FileContext.getFileContext(job.getConfiguration()), job.getConfiguration(), inputPath, NullWritable.class, clazz, CompressionType.NONE, CODEC, new Metadata(), EnumSet.of(CreateFlag.CREATE), DUMMY_VAR_ARGS); doSetupInput(values, clazz, inputsPerSplit, job, inputPath, writer); }
From source file:com.datatorrent.contrib.hdht.HDHTFileAccessFSImpl.java
License:Open Source License
@Override public void rename(long bucketKey, String fromName, String toName) throws IOException { FileContext fc = FileContext.getFileContext(fs.getUri()); Path bucketPath = getBucketPath(bucketKey); // file context requires absolute path if (!bucketPath.isAbsolute()) { bucketPath = new Path(fs.getWorkingDirectory(), bucketPath); }/*from ww w . ja v a2 s .c o m*/ fc.rename(new Path(bucketPath, fromName), new Path(bucketPath, toName), Rename.OVERWRITE); }
From source file:com.datatorrent.lib.io.fs.AbstractFileOutputOperator.java
License:Open Source License
@Override public void setup(Context.OperatorContext context) { LOG.debug("setup initiated"); rollingFile = maxLength < Long.MAX_VALUE; //Getting required file system instance. try {//from w ww. j ava 2 s.co m fs = getFSInstance(); } catch (IOException ex) { throw new RuntimeException(ex); } if (replication <= 0) { replication = fs.getDefaultReplication(new Path(filePath)); } LOG.debug("FS class {}", fs.getClass()); //When an entry is removed from the cache, removal listener is notified and it closes the output stream. RemovalListener<String, FSDataOutputStream> removalListener = new RemovalListener<String, FSDataOutputStream>() { @Override public void onRemoval(RemovalNotification<String, FSDataOutputStream> notification) { FSDataOutputStream value = notification.getValue(); if (value != null) { try { LOG.debug("closing {}", notification.getKey()); value.close(); } catch (IOException e) { throw new RuntimeException(e); } } } }; //Define cache CacheLoader<String, FSDataOutputStream> loader = new CacheLoader<String, FSDataOutputStream>() { @Override public FSDataOutputStream load(String filename) { String partFileName = getPartFileNamePri(filename); Path lfilepath = new Path(filePath + Path.SEPARATOR + partFileName); FSDataOutputStream fsOutput; boolean sawThisFileBefore = endOffsets.containsKey(filename); try { if (fs.exists(lfilepath)) { if (sawThisFileBefore) { FileStatus fileStatus = fs.getFileStatus(lfilepath); MutableLong endOffset = endOffsets.get(filename); if (endOffset != null) { endOffset.setValue(fileStatus.getLen()); } else { endOffsets.put(filename, new MutableLong(fileStatus.getLen())); } fsOutput = fs.append(lfilepath); LOG.debug("appending to {}", lfilepath); } //We never saw this file before and we don't want to append else { //If the file is rolling we need to delete all its parts. if (rollingFile) { int part = 0; while (true) { Path seenPartFilePath = new Path( filePath + Path.SEPARATOR + getPartFileName(filename, part)); if (!fs.exists(seenPartFilePath)) { break; } fs.delete(seenPartFilePath, true); part = part + 1; } fsOutput = fs.create(lfilepath, (short) replication); } //Not rolling is easy, just delete the file and create it again. else { fs.delete(lfilepath, true); fsOutput = fs.create(lfilepath, (short) replication); } } } else { fsOutput = fs.create(lfilepath, (short) replication); } //Get the end offset of the file. LOG.info("opened: {}", fs.getFileStatus(lfilepath).getPath()); return fsOutput; } catch (IOException e) { throw new RuntimeException(e); } } }; streamsCache = CacheBuilder.newBuilder().maximumSize(maxOpenFiles).removalListener(removalListener) .build(loader); try { LOG.debug("File system class: {}", fs.getClass()); LOG.debug("end-offsets {}", endOffsets); //Restore the files in case they were corrupted and the operator Path writerPath = new Path(filePath); if (fs.exists(writerPath)) { for (String seenFileName : endOffsets.keySet()) { String seenFileNamePart = getPartFileNamePri(seenFileName); LOG.debug("seenFileNamePart: {}", seenFileNamePart); Path seenPartFilePath = new Path(filePath + Path.SEPARATOR + seenFileNamePart); if (fs.exists(seenPartFilePath)) { LOG.debug("file exists {}", seenFileNamePart); long offset = endOffsets.get(seenFileName).longValue(); FSDataInputStream inputStream = fs.open(seenPartFilePath); FileStatus status = fs.getFileStatus(seenPartFilePath); if (status.getLen() != offset) { LOG.info("file corrupted {} {} {}", seenFileNamePart, offset, status.getLen()); byte[] buffer = new byte[COPY_BUFFER_SIZE]; Path tmpFilePath = new Path( filePath + Path.SEPARATOR + seenFileNamePart + TMP_EXTENSION); FSDataOutputStream fsOutput = fs.create(tmpFilePath, (short) replication); while (inputStream.getPos() < offset) { long remainingBytes = offset - inputStream.getPos(); int bytesToWrite = remainingBytes < COPY_BUFFER_SIZE ? (int) remainingBytes : COPY_BUFFER_SIZE; inputStream.read(buffer); fsOutput.write(buffer, 0, bytesToWrite); } flush(fsOutput); fsOutput.close(); inputStream.close(); FileContext fileContext = FileContext.getFileContext(fs.getUri()); LOG.debug("temp file path {}, rolling file path {}", tmpFilePath.toString(), status.getPath().toString()); fileContext.rename(tmpFilePath, status.getPath(), Options.Rename.OVERWRITE); } else { inputStream.close(); } } } } //delete the left over future rolling files produced from the previous crashed instance //of this operator. if (rollingFile) { for (String seenFileName : endOffsets.keySet()) { try { Integer part = openPart.get(seenFileName).getValue() + 1; while (true) { Path seenPartFilePath = new Path( filePath + Path.SEPARATOR + getPartFileName(seenFileName, part)); if (!fs.exists(seenPartFilePath)) { break; } fs.delete(seenPartFilePath, true); part = part + 1; } Path seenPartFilePath = new Path(filePath + Path.SEPARATOR + getPartFileName(seenFileName, openPart.get(seenFileName).intValue())); //Handle the case when restoring to a checkpoint where the current rolling file //already has a length greater than max length. if (fs.getFileStatus(seenPartFilePath).getLen() > maxLength) { LOG.debug("rotating file at setup."); rotate(seenFileName); } } catch (IOException e) { throw new RuntimeException(e); } catch (ExecutionException e) { throw new RuntimeException(e); } } } LOG.debug("setup completed"); LOG.debug("end-offsets {}", endOffsets); } catch (IOException e) { throw new RuntimeException(e); } this.context = context; lastTimeStamp = System.currentTimeMillis(); fileCounters.setCounter(Counters.TOTAL_BYTES_WRITTEN, new MutableLong()); fileCounters.setCounter(Counters.TOTAL_TIME_ELAPSED, new MutableLong()); }
From source file:com.datatorrent.lib.io.fs.AbstractFSWriter.java
License:Open Source License
@Override public void setup(Context.OperatorContext context) { rollingFile = maxLength < Long.MAX_VALUE; //Getting required file system instance. try {// w w w. j ava 2 s . c o m fs = getFSInstance(); } catch (IOException ex) { throw new RuntimeException(ex); } LOG.debug("FS class {}", fs.getClass()); //Setting listener for debugging LOG.debug("setup initiated"); RemovalListener<String, FSDataOutputStream> removalListener = new RemovalListener<String, FSDataOutputStream>() { @Override public void onRemoval(RemovalNotification<String, FSDataOutputStream> notification) { FSDataOutputStream value = notification.getValue(); if (value != null) { try { LOG.debug("closing {}", notification.getKey()); value.close(); } catch (IOException e) { throw new RuntimeException(e); } } } }; //Define cache CacheLoader<String, FSDataOutputStream> loader = new CacheLoader<String, FSDataOutputStream>() { @Override public FSDataOutputStream load(String filename) { String partFileName = getPartFileNamePri(filename); Path lfilepath = new Path(filePath + File.separator + partFileName); FSDataOutputStream fsOutput; if (replication <= 0) { replication = fs.getDefaultReplication(lfilepath); } boolean sawThisFileBefore = endOffsets.containsKey(filename); try { if (fs.exists(lfilepath)) { if (sawThisFileBefore || append) { FileStatus fileStatus = fs.getFileStatus(lfilepath); MutableLong endOffset = endOffsets.get(filename); if (endOffset != null) { endOffset.setValue(fileStatus.getLen()); } else { endOffsets.put(filename, new MutableLong(fileStatus.getLen())); } fsOutput = fs.append(lfilepath); LOG.debug("appending to {}", lfilepath); } //We never saw this file before and we don't want to append else { //If the file is rolling we need to delete all its parts. if (rollingFile) { int part = 0; while (true) { Path seenPartFilePath = new Path( filePath + "/" + getPartFileName(filename, part)); if (!fs.exists(seenPartFilePath)) { break; } fs.delete(seenPartFilePath, true); part = part + 1; } fsOutput = fs.create(lfilepath, (short) replication); } //Not rolling is easy, just delete the file and create it again. else { fs.delete(lfilepath, true); fsOutput = fs.create(lfilepath, (short) replication); } } } else { fsOutput = fs.create(lfilepath, (short) replication); } //Get the end offset of the file. LOG.debug("full path: {}", fs.getFileStatus(lfilepath).getPath()); return fsOutput; } catch (IOException e) { throw new RuntimeException(e); } } }; streamsCache = CacheBuilder.newBuilder().maximumSize(maxOpenFiles).removalListener(removalListener) .build(loader); try { LOG.debug("File system class: {}", fs.getClass()); LOG.debug("end-offsets {}", endOffsets); //Restore the files in case they were corrupted and the operator Path writerPath = new Path(filePath); if (fs.exists(writerPath)) { for (String seenFileName : endOffsets.keySet()) { String seenFileNamePart = getPartFileNamePri(seenFileName); LOG.debug("seenFileNamePart: {}", seenFileNamePart); Path seenPartFilePath = new Path(filePath + "/" + seenFileNamePart); if (fs.exists(seenPartFilePath)) { LOG.debug("file exists {}", seenFileNamePart); long offset = endOffsets.get(seenFileName).longValue(); FSDataInputStream inputStream = fs.open(seenPartFilePath); FileStatus status = fs.getFileStatus(seenPartFilePath); if (status.getLen() != offset) { LOG.info("file corrupted {} {} {}", seenFileNamePart, offset, status.getLen()); byte[] buffer = new byte[COPY_BUFFER_SIZE]; String tmpFileName = seenFileNamePart + TMP_EXTENSION; FSDataOutputStream fsOutput = streamsCache.get(tmpFileName); while (inputStream.getPos() < offset) { long remainingBytes = offset - inputStream.getPos(); int bytesToWrite = remainingBytes < COPY_BUFFER_SIZE ? (int) remainingBytes : COPY_BUFFER_SIZE; inputStream.read(buffer); fsOutput.write(buffer, 0, bytesToWrite); } flush(fsOutput); FileContext fileContext = FileContext.getFileContext(fs.getUri()); String tempTmpFilePath = getPartFileNamePri(filePath + File.separator + tmpFileName); Path tmpFilePath = new Path(tempTmpFilePath); tmpFilePath = fs.getFileStatus(tmpFilePath).getPath(); LOG.debug("temp file path {}, rolling file path {}", tmpFilePath.toString(), status.getPath().toString()); fileContext.rename(tmpFilePath, status.getPath(), Options.Rename.OVERWRITE); } } } } //delete the left over future rolling files produced from the previous crashed instance //of this operator. if (rollingFile) { for (String seenFileName : endOffsets.keySet()) { try { Integer part = openPart.get(seenFileName).getValue() + 1; while (true) { Path seenPartFilePath = new Path(filePath + "/" + getPartFileName(seenFileName, part)); if (!fs.exists(seenPartFilePath)) { break; } fs.delete(seenPartFilePath, true); part = part + 1; } Path seenPartFilePath = new Path(filePath + "/" + getPartFileName(seenFileName, openPart.get(seenFileName).intValue())); //Handle the case when restoring to a checkpoint where the current rolling file //already has a length greater than max length. if (fs.getFileStatus(seenPartFilePath).getLen() > maxLength) { LOG.debug("rotating file at setup."); rotate(seenFileName); } } catch (IOException e) { throw new RuntimeException(e); } catch (ExecutionException e) { throw new RuntimeException(e); } } } LOG.debug("setup completed"); LOG.debug("end-offsets {}", endOffsets); } catch (IOException e) { throw new RuntimeException(e); } catch (ExecutionException e) { throw new RuntimeException(e); } this.context = context; lastTimeStamp = System.currentTimeMillis(); fileCounters.setCounter(Counters.TOTAL_BYTES_WRITTEN, new MutableLong()); fileCounters.setCounter(Counters.TOTAL_TIME_ELAPSED, new MutableLong()); }
From source file:com.datatorrent.stram.FSRecoveryHandler.java
License:Apache License
@Override public Object restore() throws IOException { FileContext fc = FileContext.getFileContext(fs.getUri()); // recover from wherever it was left if (fc.util().exists(snapshotBackupPath)) { LOG.warn("Incomplete checkpoint, reverting to {}", snapshotBackupPath); fc.rename(snapshotBackupPath, snapshotPath, Rename.OVERWRITE); // combine logs (w/o append, create new file) Path tmpLogPath = new Path(basedir, "log.combined"); FSDataOutputStream fsOut = fc.create(tmpLogPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); try {/*w w w .j a va2 s. co m*/ FSDataInputStream fsIn = fc.open(logBackupPath); try { IOUtils.copy(fsIn, fsOut); } finally { fsIn.close(); } fsIn = fc.open(logPath); try { IOUtils.copy(fsIn, fsOut); } finally { fsIn.close(); } } finally { fsOut.close(); } fc.rename(tmpLogPath, logPath, Rename.OVERWRITE); fc.delete(logBackupPath, false); } else { // we have log backup, but no checkpoint backup // failure between log rotation and writing checkpoint if (fc.util().exists(logBackupPath)) { LOG.warn("Found {}, did checkpointing fail?", logBackupPath); fc.rename(logBackupPath, logPath, Rename.OVERWRITE); } } if (!fc.util().exists(snapshotPath)) { LOG.debug("No existing checkpoint."); return null; } LOG.debug("Reading checkpoint {}", snapshotPath); InputStream is = fc.open(snapshotPath); // indeterministic class loading behavior // http://stackoverflow.com/questions/9110677/readresolve-not-working-an-instance-of-guavas-serializedform-appears final ClassLoader loader = Thread.currentThread().getContextClassLoader(); ObjectInputStream ois = new ObjectInputStream(is) { @Override protected Class<?> resolveClass(ObjectStreamClass objectStreamClass) throws IOException, ClassNotFoundException { return Class.forName(objectStreamClass.getName(), true, loader); } }; //ObjectInputStream ois = new ObjectInputStream(is); try { return ois.readObject(); } catch (ClassNotFoundException cnfe) { throw new IOException("Failed to read checkpointed state", cnfe); } finally { ois.close(); } }