Example usage for org.apache.hadoop.fs FileStatus getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime()

Source Link

Document

Get the modification time of the file.

Usage

From source file:org.apache.asterix.metadata.utils.ExternalIndexingOperations.java

License:Apache License

private static void handleFile(Dataset dataset, List<ExternalFile> files, FileSystem fs, FileStatus fileStatus,
        int nextFileNumber) throws IOException {
    if (fileStatus.isDirectory()) {
        listSubFiles(dataset, fs, fileStatus, files);
    } else {/*from  ww w.j  ava 2 s .c  om*/
        files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(), nextFileNumber,
                fileStatus.getPath().toUri().getPath(), new Date(fileStatus.getModificationTime()),
                fileStatus.getLen(), ExternalFilePendingOp.NO_OP));
    }
}

From source file:org.apache.beam.sdk.io.hdfs.HadoopFileSystem.java

License:Apache License

private Metadata toMetadata(FileStatus fileStatus) {
    URI uri = dropEmptyAuthority(fileStatus.getPath().toUri().toString());
    return Metadata.builder().setResourceId(new HadoopResourceId(uri)).setIsReadSeekEfficient(true)
            .setSizeBytes(fileStatus.getLen()).setLastModifiedMillis(fileStatus.getModificationTime()).build();
}

From source file:org.apache.blur.command.BaseCommandManager.java

License:Apache License

protected BigInteger checkContents(FileStatus fileStatus, FileSystem fileSystem) throws IOException {
    if (fileStatus.isDir()) {
        LOG.debug("Scanning directory [{0}].", fileStatus.getPath());
        BigInteger count = BigInteger.ZERO;
        Path path = fileStatus.getPath();
        FileStatus[] listStatus = fileSystem.listStatus(path);
        for (FileStatus fs : listStatus) {
            count = count.add(checkContents(fs, fileSystem));
        }//  w ww  . ja  v a2s  . com
        return count;
    } else {
        int hashCode = fileStatus.getPath().toString().hashCode();
        long modificationTime = fileStatus.getModificationTime();
        long len = fileStatus.getLen();
        BigInteger bi = BigInteger.valueOf(hashCode)
                .add(BigInteger.valueOf(modificationTime).add(BigInteger.valueOf(len)));
        LOG.debug("File path hashcode [{0}], mod time [{1}], len [{2}] equals file code [{3}].",
                Integer.toString(hashCode), Long.toString(modificationTime), Long.toString(len),
                bi.toString(Character.MAX_RADIX));
        return bi;
    }
}

From source file:org.apache.blur.mapreduce.lib.update.MapperForNewData.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    InputSplit inputSplit = context.getInputSplit();
    FileSplit fileSplit = getFileSplit(inputSplit);
    Path path = fileSplit.getPath();
    FileSystem fileSystem = path.getFileSystem(context.getConfiguration());
    FileStatus fileStatus = fileSystem.getFileStatus(path);
    _timestamp = fileStatus.getModificationTime();
    _newRecords = context.getCounter("Blur", "New Records Read");
}

From source file:org.apache.blur.store.hdfs.HdfsDirectory.java

License:Apache License

private void addToCache(FileStatus fileStatus) throws IOException {
    if (!fileStatus.isDir()) {
        Path p = fileStatus.getPath();
        String name = p.getName();
        long lastMod;
        long length;
        String resolvedName;/*from  ww  w. jav a 2  s .  c  o  m*/
        if (name.endsWith(LNK)) {
            resolvedName = getRealFileName(name);
            Path resolvedPath = getPath(resolvedName);
            FileStatus resolvedFileStatus = _fileSystem.getFileStatus(resolvedPath);
            lastMod = resolvedFileStatus.getModificationTime();
        } else {
            resolvedName = name;
            lastMod = fileStatus.getModificationTime();
        }
        length = length(resolvedName);
        _fileStatusMap.put(resolvedName, new FStat(lastMod, length));
    }
}

From source file:org.apache.blur.store.hdfs.HdfsDirectory.java

License:Apache License

protected long fileModified(String name) throws IOException {
    Path path = getPath(name);/*from w  ww. j a va 2s  .c  o  m*/
    Tracer trace = Trace.trace("filesystem - fileModified", Trace.param("path", path));
    try {
        FileStatus fileStatus = _fileSystem.getFileStatus(path);
        if (_useCache) {
            _fileStatusMap.put(name, new FStat(fileStatus));
        }
        return fileStatus.getModificationTime();
    } finally {
        trace.done();
    }
}

From source file:org.apache.crunch.io.SourceTargetHelper.java

License:Apache License

public static long getLastModifiedAt(FileSystem fs, Path path) throws IOException {
    FileStatus[] stati = fs.globStatus(path);
    if (stati == null || stati.length == 0) {
        return -1L;
    }//from  w w  w .j a va  2  s  .  c  om
    long lastMod = -1;
    for (FileStatus status : stati) {
        if (lastMod < status.getModificationTime()) {
            lastMod = status.getModificationTime();
        }
    }
    return lastMod;
}

From source file:org.apache.distributedlog.fs.TestDLFileSystem.java

License:Apache License

@Test
public void testListStatuses() throws Exception {
    Path parentPath = new Path("/path/to/" + runtime.getMethodName());
    assertFalse(fs.exists(parentPath));//from ww w.  java2  s  . co  m
    try (FSDataOutputStream parentOut = fs.create(parentPath)) {
        parentOut.writeBytes("parent");
        parentOut.flush();
    }
    assertTrue(fs.exists(parentPath));

    int numLogs = 3;
    for (int i = 0; i < numLogs; i++) {
        Path path = new Path("/path/to/" + runtime.getMethodName() + "/" + runtime.getMethodName() + "-" + i);
        assertFalse(fs.exists(path));
        try (FSDataOutputStream out = fs.create(path)) {
            out.writeBytes("line");
            out.flush();
        }
        assertTrue(fs.exists(path));
    }
    FileStatus[] files = fs.listStatus(new Path("/path/to/" + runtime.getMethodName()));

    assertEquals(3, files.length);
    for (int i = 0; i < numLogs; i++) {
        FileStatus file = files[i];
        assertEquals(4, file.getLen());
        assertFalse(file.isDirectory());
        assertEquals(3, file.getReplication());
        assertEquals(0L, file.getModificationTime());
        assertEquals(new Path("/path/to/" + runtime.getMethodName() + "/" + runtime.getMethodName() + "-" + i),
                file.getPath());
    }
}

From source file:org.apache.drill.exec.planner.sql.handlers.AnalyzeTableHandler.java

License:Apache License

private boolean isStatsStale(DrillFileSystem fs, Path statsFilePath) throws IOException {
    long statsFileModifyTime = fs.getFileStatus(statsFilePath).getModificationTime();
    Path parentPath = statsFilePath.getParent();
    FileStatus directoryStatus = fs.getFileStatus(parentPath);
    // Parent directory modified after stats collection?
    return directoryStatus.getModificationTime() > statsFileModifyTime
            || tableModified(fs, parentPath, statsFileModifyTime);
}

From source file:org.apache.drill.exec.planner.sql.handlers.AnalyzeTableHandler.java

License:Apache License

private boolean tableModified(DrillFileSystem fs, Path parentPath, long statsModificationTime)
        throws IOException {
    for (final FileStatus file : fs.listStatus(parentPath)) {
        // If directory or files within it are modified
        if (file.getModificationTime() > statsModificationTime) {
            return true;
        }/*from www.j  a v  a2  s . c o  m*/
        // For a directory, we should recursively check sub-directories
        if (file.isDirectory() && tableModified(fs, file.getPath(), statsModificationTime)) {
            return true;
        }
    }
    return false;
}