Example usage for org.apache.hadoop.fs FileContext open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileContext open.

Prototype

public FSDataInputStream open(final Path f)
        throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException 

Source Link

Document

Opens an FSDataInputStream at the indicated Path using default buffersize.

Usage

From source file:com.datatorrent.stram.FSRecoveryHandler.java

License:Apache License

@Override
public Object restore() throws IOException {
    FileContext fc = FileContext.getFileContext(fs.getUri());

    // recover from wherever it was left
    if (fc.util().exists(snapshotBackupPath)) {
        LOG.warn("Incomplete checkpoint, reverting to {}", snapshotBackupPath);
        fc.rename(snapshotBackupPath, snapshotPath, Rename.OVERWRITE);

        // combine logs (w/o append, create new file)
        Path tmpLogPath = new Path(basedir, "log.combined");
        FSDataOutputStream fsOut = fc.create(tmpLogPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE));
        try {/*from ww w .j  a v  a  2 s .c o  m*/
            FSDataInputStream fsIn = fc.open(logBackupPath);
            try {
                IOUtils.copy(fsIn, fsOut);
            } finally {
                fsIn.close();
            }

            fsIn = fc.open(logPath);
            try {
                IOUtils.copy(fsIn, fsOut);
            } finally {
                fsIn.close();
            }
        } finally {
            fsOut.close();
        }

        fc.rename(tmpLogPath, logPath, Rename.OVERWRITE);
        fc.delete(logBackupPath, false);
    } else {
        // we have log backup, but no checkpoint backup
        // failure between log rotation and writing checkpoint
        if (fc.util().exists(logBackupPath)) {
            LOG.warn("Found {}, did checkpointing fail?", logBackupPath);
            fc.rename(logBackupPath, logPath, Rename.OVERWRITE);
        }
    }

    if (!fc.util().exists(snapshotPath)) {
        LOG.debug("No existing checkpoint.");
        return null;
    }

    LOG.debug("Reading checkpoint {}", snapshotPath);
    InputStream is = fc.open(snapshotPath);
    // indeterministic class loading behavior
    // http://stackoverflow.com/questions/9110677/readresolve-not-working-an-instance-of-guavas-serializedform-appears
    final ClassLoader loader = Thread.currentThread().getContextClassLoader();
    ObjectInputStream ois = new ObjectInputStream(is) {
        @Override
        protected Class<?> resolveClass(ObjectStreamClass objectStreamClass)
                throws IOException, ClassNotFoundException {
            return Class.forName(objectStreamClass.getName(), true, loader);
        }
    };
    //ObjectInputStream ois = new ObjectInputStream(is);
    try {
        return ois.readObject();
    } catch (ClassNotFoundException cnfe) {
        throw new IOException("Failed to read checkpointed state", cnfe);
    } finally {
        ois.close();
    }
}

---

From source file:org.apache.apex.malhar.lib.utils.IOUtilsTest.java

License:Apache License

private void testCopyPartialHelper(int dataSize, int offset, long size) throws IOException {
    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
    File file = new File("target/IOUtilsTest/testCopyPartial/input");
    createDataFile(file, dataSize);//from   w  w w.  j  av a2  s . c o m

    FileContext fileContext = FileContext.getFileContext();
    DataInputStream inputStream = fileContext.open(new Path(file.getAbsolutePath()));

    Path output = new Path("target/IOUtilsTest/testCopyPartial/output");
    DataOutputStream outputStream = fileContext.create(output,
            EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
            Options.CreateOpts.CreateParent.createParent());

    if (offset == 0) {
        IOUtils.copyPartial(inputStream, size, outputStream);
    } else {
        IOUtils.copyPartial(inputStream, offset, size, outputStream);
    }

    outputStream.close();

    Assert.assertTrue("output exists", fileContext.util().exists(output));
    Assert.assertEquals("output size", size, fileContext.getFileStatus(output).getLen());
    //    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
}

---