Example usage for org.apache.hadoop.fs FileStatus FileStatus

List of usage examples for org.apache.hadoop.fs FileStatus FileStatus

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. FileStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus FileStatus.

Prototype

public FileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time,
        long access_time, FsPermission permission, String owner, String group, Path path)

Source Link

Document

Constructor for file systems on which symbolic links are not supported

Usage

From source file:org.apache.nifi.processors.hadoop.TestListHDFS.java

License:Apache License

@Test
public void testNoListUntilUpdateFromRemoteOnPrimaryNodeChange() throws IOException, InterruptedException {
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 1999L, 0L, create777(),
            "owner", "group", new Path("/test/testFile.txt")));

    // first iteration will not pick up files because it has to instead check timestamps.
    // We must then wait long enough to ensure that the listing can be performed safely and
    // run the Processor again.
    runner.run();/* w w  w  .  j  a  v  a 2 s .  co m*/
    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 1);

    final MockFlowFile mff1 = runner.getFlowFilesForRelationship(ListHDFS.REL_SUCCESS).get(0);
    mff1.assertAttributeEquals("path", "/test");
    mff1.assertAttributeEquals("filename", "testFile.txt");

    runner.clearTransferState();

    // add new file to pull
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 2000L, 0L, create777(),
            "owner", "group", new Path("/test/testFile2.txt")));

    // cause calls to service to fail
    service.failOnCalls = true;

    runner.getStateManager().setFailOnStateGet(Scope.CLUSTER, true);

    // Should fail to perform @OnScheduled methods.
    try {
        runner.run();
        Assert.fail("Processor ran successfully");
    } catch (final AssertionError e) {
    }

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 0);

    // Should fail to perform @OnScheduled methods.
    try {
        runner.run();
        Assert.fail("Processor ran successfully");
    } catch (final AssertionError e) {
    }

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 0);

    service.failOnCalls = false;
    runner.getStateManager().setFailOnStateGet(Scope.CLUSTER, false);
    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));

    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 0);
    Map<String, String> newState = runner.getStateManager().getState(Scope.CLUSTER).toMap();
    assertEquals("2000", newState.get(ListHDFS.LISTING_TIMESTAMP_KEY));
    assertEquals("1999", newState.get(ListHDFS.EMITTED_TIMESTAMP_KEY));

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    newState = runner.getStateManager().getState(Scope.CLUSTER).toMap();
    assertEquals("2000", newState.get(ListHDFS.LISTING_TIMESTAMP_KEY));
    assertEquals("2000", newState.get(ListHDFS.EMITTED_TIMESTAMP_KEY));

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 1);
}

From source file:org.apache.nifi.processors.hadoop.TestListHDFS.java

License:Apache License

@Test
public void testOnlyNewestEntriesHeldBack() throws InterruptedException {
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(),
            "owner", "group", new Path("/test/testFile.txt")));
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 8L, 0L, create777(),
            "owner", "group", new Path("/test/testFile2.txt")));

    // this is a directory, so it won't be counted toward the entries
    proc.fileSystem.addFileStatus(new Path("/test"),
            new FileStatus(1L, true, 1, 1L, 8L, 0L, create777(), "owner", "group", new Path("/test/testDir")));
    proc.fileSystem.addFileStatus(new Path("/test/testDir"), new FileStatus(1L, false, 1, 1L, 100L, 0L,
            create777(), "owner", "group", new Path("/test/testDir/1.txt")));
    proc.fileSystem.addFileStatus(new Path("/test/testDir"), new FileStatus(1L, false, 1, 1L, 100L, 0L,
            create777(), "owner", "group", new Path("/test/testDir/2.txt")));

    // The first iteration should pick up 2 files with the smaller timestamps.
    runner.run();//from  w  w w .  j  a  v  a  2  s  . co m
    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2);

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    // Next iteration should pick up the other 2 files, since nothing else was added.
    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 4);

    proc.fileSystem.addFileStatus(new Path("/test/testDir"), new FileStatus(1L, false, 1, 1L, 110L, 0L,
            create777(), "owner", "group", new Path("/test/testDir/3.txt")));
    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 4);

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 5);
}

From source file:org.apache.nifi.processors.hadoop.TestListHDFS.java

License:Apache License

@Test
public void testMinAgeMaxAge() throws IOException, InterruptedException {
    long now = new Date().getTime();
    long oneHourAgo = now - 3600000;
    long twoHoursAgo = now - 2 * 3600000;
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, now, now, create777(),
            "owner", "group", new Path("/test/willBeIgnored.txt")));
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, now - 5, now - 5,
            create777(), "owner", "group", new Path("/test/testFile.txt")));
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, oneHourAgo, oneHourAgo,
            create777(), "owner", "group", new Path("/test/testFile1.txt")));
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, twoHoursAgo, twoHoursAgo,
            create777(), "owner", "group", new Path("/test/testFile2.txt")));

    // all files/*from ww w.  j a v a  2  s  .c  om*/
    runner.run();
    runner.assertValid();
    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 3);
    runner.clearTransferState();
    runner.getStateManager().clear(Scope.CLUSTER);

    // invalid min_age > max_age
    runner.setProperty(ListHDFS.MIN_AGE, "30 sec");
    runner.setProperty(ListHDFS.MAX_AGE, "1 sec");
    runner.assertNotValid();

    // only one file (one hour ago)
    runner.setProperty(ListHDFS.MIN_AGE, "30 sec");
    runner.setProperty(ListHDFS.MAX_AGE, "90 min");
    runner.assertValid();
    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run(); // will ignore the file for this cycle
    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 0);

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    // Next iteration should pick up the file, since nothing else was added.
    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 1);
    runner.getFlowFilesForRelationship(ListHDFS.REL_SUCCESS).get(0).assertAttributeEquals("filename",
            "testFile1.txt");
    runner.clearTransferState();
    runner.getStateManager().clear(Scope.CLUSTER);

    // two files (one hour ago and two hours ago)
    runner.setProperty(ListHDFS.MIN_AGE, "30 sec");
    runner.removeProperty(ListHDFS.MAX_AGE);
    runner.assertValid();

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 1);

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2);
    runner.clearTransferState();
    runner.getStateManager().clear(Scope.CLUSTER);

    // two files (now and one hour ago)
    runner.setProperty(ListHDFS.MIN_AGE, "0 sec");
    runner.setProperty(ListHDFS.MAX_AGE, "90 min");
    runner.assertValid();

    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2);
}

From source file:org.elasticsearch.hadoop.mr.NTFSLocalFileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(Path f) throws IOException {
    // it's the RawFS in place which messes things up as it dynamically returns the permissions...
    // workaround by doing a copy
    FileStatus fs = super.getFileStatus(f);

    // work-around for Hive 0.14
    if (SCRATCH_DIR.equals(f.toString())) {
        System.out.println("Faking scratch dir permissions on Windows...");

        return new FileStatus(fs.getLen(), fs.isDir(), fs.getReplication(), fs.getBlockSize(),
                fs.getModificationTime(), fs.getAccessTime(), SCRATCH_DIR_PERMS, fs.getOwner(), fs.getGroup(),
                fs.getPath());/*from   www . ja v  a  2  s  .co m*/
        // this doesn't work since the RawFS impl has its own algo that does the lookup dynamically
        //fs.getPermission().fromShort((short) 777);
    }
    return fs;
}

From source file:org.gridgain.grid.ggfs.hadoop.v1.GridGgfsHadoopFileSystem.java

License:Open Source License

/**
 * Convert GGFS file information into Hadoop file status.
 *
 * @param file GGFS file information.// w  ww  .  ja  v a2s. c  o m
 * @return Hadoop file status.
 */
@SuppressWarnings("deprecation")
private FileStatus convert(GridGgfsFile file) {
    return new FileStatus(file.length(), file.isDirectory(), getDefaultReplication(), file.groupBlockSize(),
            file.modificationTime(), file.accessTime(), permission(file),
            file.property(PROP_USER_NAME, DFLT_USER_NAME), file.property(PROP_GROUP_NAME, "users"),
            convert(file.path())) {
        @Override
        public String toString() {
            return "FileStatus [path=" + getPath() + ", isDir=" + isDir() + ", len=" + getLen() + ", mtime="
                    + getModificationTime() + ", atime=" + getAccessTime() + ']';
        }
    };
}

From source file:org.gridgain.grid.ggfs.hadoop.v2.GridGgfsHadoopFileSystem.java

License:Open Source License

/**
 * Convert GGFS file information into Hadoop file status.
 *
 * @param file GGFS file information./*w ww.  j  a v a  2s.c o  m*/
 * @return Hadoop file status.
 */
private FileStatus convert(GridGgfsFile file) {
    return new FileStatus(file.length(), file.isDirectory(), dfltReplication, file.groupBlockSize(),
            file.modificationTime(), file.accessTime(), permission(file),
            file.property(PROP_USER_NAME, DFLT_USER_NAME), file.property(PROP_GROUP_NAME, "users"),
            convert(file.path())) {
        @Override
        public String toString() {
            return "FileStatus [path=" + getPath() + ", isDir=" + isDirectory() + ", len=" + getLen() + "]";
        }
    };
}

From source file:org.xtreemfs.common.clients.hadoop.XtreemFSFileSystem.java

License:BSD License

@Override
public FileStatus[] listStatus(Path path) throws IOException {
    if (path == null) {
        return null;
    }/*from  w  ww.  j av a2 s  . c  om*/
    Volume xtreemfsVolume = getVolumeFromPath(path);
    final String pathString = preparePath(path, xtreemfsVolume);

    if (Logging.isDebug()) {
        Logging.logMessage(Logging.LEVEL_DEBUG, this, "ls: " + pathString);
    }

    if (isXtreemFSDirectory(pathString, xtreemfsVolume) == false) {
        return null;
    }

    DirectoryEntries dirEntries = xtreemfsVolume.readDir(userCredentials, pathString, 0, 0, false);
    statistics.incrementLargeReadOps(1);
    ArrayList<FileStatus> fileStatus = new ArrayList<FileStatus>(dirEntries.getEntriesCount() - 2);
    for (DirectoryEntry entry : dirEntries.getEntriesList()) {
        if (entry.getName().equals("..") || entry.getName().equals(".")) {
            continue;
        }
        final Stat stat = entry.getStbuf();
        final boolean isDir = isXtreemFSDirectory(stat);
        if (isDir) {
            // for directories, set blocksize to 0
            fileStatus.add(new FileStatus(0, isDir, 1, 0, (long) (stat.getMtimeNs() / 1e6),
                    (long) (stat.getAtimeNs() / 1e6), new FsPermission((short) stat.getMode()),
                    stat.getUserId(), stat.getGroupId(), new Path(makeAbsolute(path), entry.getName())));
        } else {
            // for files, set blocksize to stripeSize of the volume
            fileStatus
                    .add(new FileStatus(stat.getSize(), isDir, 1,
                            xtreemfsVolume.statFS(userCredentials).getDefaultStripingPolicy().getStripeSize()
                                    * 1024,
                            (long) (stat.getMtimeNs() / 1e6), (long) (stat.getAtimeNs() / 1e6),
                            new FsPermission((short) stat.getMode()), stat.getUserId(), stat.getGroupId(),
                            new Path(makeAbsolute(path), entry.getName())));
        }
    }
    return fileStatus.toArray(new FileStatus[fileStatus.size()]);
}

From source file:org.xtreemfs.common.clients.hadoop.XtreemFSFileSystem.java

License:BSD License

@Override
public FileStatus getFileStatus(Path path) throws IOException {
    Volume xtreemfsVolume = getVolumeFromPath(path);
    final String pathString = preparePath(path, xtreemfsVolume);
    if (Logging.isDebug()) {
        Logging.logMessage(Logging.LEVEL_DEBUG, this, "getting file status for file %s", pathString);
    }//  w  w w .  ja  va  2s  . c  om
    Stat stat = null;
    try {
        stat = xtreemfsVolume.getAttr(userCredentials, pathString);
    } catch (PosixErrorException pee) {
        if (pee.getPosixError().equals(POSIXErrno.POSIX_ERROR_ENOENT)) {
            throw new FileNotFoundException();
        }
        throw pee;
    }
    final boolean isDir = isXtreemFSDirectory(stat);
    if (isDir) {
        // for directories, set blocksize to 0
        return new FileStatus(0, isDir, 1, 0, (long) (stat.getMtimeNs() / 1e6),
                (long) (stat.getAtimeNs() / 1e6), new FsPermission((short) stat.getMode()), stat.getUserId(),
                stat.getGroupId(), makeQualified(path));
    } else {
        // for files, set blocksize to stripesize of the volume
        return new FileStatus(stat.getSize(), isDir, 1,
                xtreemfsVolume.statFS(userCredentials).getDefaultStripingPolicy().getStripeSize() * 1024,
                (long) (stat.getMtimeNs() / 1e6), (long) (stat.getAtimeNs() / 1e6),
                new FsPermission((short) stat.getMode()), stat.getUserId(), stat.getGroupId(),
                makeQualified(path));
    }
}

From source file:ras.test.hadoop.fs.InMemoryFileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(Path path) throws IOException {
    path = makeAbsolute(path);// www .  ja  v a2  s.c  o  m
    Node node = getPathMapNode(getConf(), name, path);
    if (node == null) {
        throw new FileNotFoundException("'" + path + "' not found!");
    }
    long length = 0;
    boolean isDir = true;
    if (node instanceof FileNode) {
        length = ((FileNode) node).bytes.length;
        isDir = false;
    }

    FileStatus status = new FileStatus(length, isDir, 1, 0, 0, 0, node.getPermissions(), node.getOwner(),
            node.getGroup(), path);
    return status;
}

From source file:tachyon.hadoop.AbstractTFS.java

License:Apache License

/**
 * {@inheritDoc}//from  ww  w  . j  a  v  a2 s . co m
 *
 * If the file does not exist in Tachyon, query it from HDFS.
 */
@Override
public FileStatus getFileStatus(Path path) throws IOException {
    TachyonURI tPath = new TachyonURI(Utils.getPathWithoutScheme(path));
    Path hdfsPath = Utils.getHDFSPath(tPath, mUnderFSAddress);

    LOG.info("getFileStatus(" + path + "): HDFS Path: " + hdfsPath + " TPath: " + mTachyonHeader + tPath);
    if (mStatistics != null) {
        mStatistics.incrementReadOps(1);
    }
    if (useHdfs()) {
        fromHdfsToTachyon(tPath);
    }
    TachyonFile file;
    try {
        file = mTFS.getFile(tPath);
    } catch (IOException ioe) {
        LOG.info("File does not exist: " + path);
        throw new FileNotFoundException("File does not exist: " + path);
    }

    FileStatus ret = new FileStatus(file.length(), file.isDirectory(), file.getDiskReplication(),
            file.getBlockSizeByte(), file.getCreationTimeMs(), file.getCreationTimeMs(), null, null, null,
            new Path(mTachyonHeader + tPath));
    return ret;
}