Example usage for org.apache.hadoop.fs FileStatus getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime() 

Source Link

Document

Get the modification time of the file.

Usage

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *///  ww w .  j a v a2  s. c  o  m
public void testUtimes() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/utimetest");
        Runtime r = Runtime.getRuntime();
        String cmd = "touch " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.exists(path));

        FileStatus foo = fileSys.getFileStatus(path);
        long oldTime = foo.getModificationTime();
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }

        cmd = "touch " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }
        foo = fileSys.getFileStatus(path);
        long newTime = foo.getModificationTime();

        assertTrue(newTime > oldTime);

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
    }
}

---

From source file:AggregatedLogsPurger.java

License:Apache License

public boolean purge() throws IOException {
    LocalDateTime now = LocalDateTime.now();
    LocalDateTime deleteLogsOlderThanTime = now.minusDays(deleteOlderThanDays);

    //Identify which log dirs should be deleted
    FileSystem fs = rootLogDir.getFileSystem(conf);
    try {/*from www. j  a va  2 s . co  m*/

        long totalBytes = 0;
        for (FileStatus userDir : fs.listStatus(rootLogDir)) {
            if (userDir.isDirectory()) {
                Path userDirPath = new Path(userDir.getPath(), suffix);
                System.out.println("Checking for userDir : " + userDirPath);
                for (FileStatus appDir : fs.listStatus(userDirPath)) {
                    LocalDateTime appDirDate = getAppDirDateTime(appDir.getModificationTime());
                    if (appDirDate.isBefore(deleteLogsOlderThanTime)) {
                        long size = getLengthRecursively(fs, appDir.getPath());
                        System.out.println(appDir.getPath() + ", " + appDir.getOwner() + ", "
                                + appDirDate.toString() + ", size=" + size);
                        totalBytes += size;
                        if (shouldDelete) {
                            System.out.println("Deleting " + appDir.getPath());
                            fs.delete(appDir.getPath(), true);
                        }
                    }
                }
            }
        }
        System.out.println("Savings : " + totalBytes);
    } catch (IOException e) {
        e.printStackTrace();
        return false;
    } finally {
        fs.close();
    }
    return true;
}

---

From source file:alluxio.hadoop.HadoopUtils.java

License:Apache License

/**
 * Returns a string representation of a Hadoop {@link FileStatus}.
 *
 * @param fs Hadoop {@link FileStatus}/*  ww  w  .j av a2 s  .c  o  m*/
 * @return its string representation
 */
public static String toStringHadoopFileStatus(FileStatus fs) {
    StringBuilder sb = new StringBuilder();
    sb.append("HadoopFileStatus: Path: ").append(fs.getPath());
    sb.append(" , Length: ").append(fs.getLen());
    // Use isDir instead of isDirectory for compatibility with hadoop 1.
    sb.append(" , IsDir: ").append(fs.isDir());
    sb.append(" , BlockReplication: ").append(fs.getReplication());
    sb.append(" , BlockSize: ").append(fs.getBlockSize());
    sb.append(" , ModificationTime: ").append(fs.getModificationTime());
    sb.append(" , AccessTime: ").append(fs.getAccessTime());
    sb.append(" , Permission: ").append(fs.getPermission());
    sb.append(" , Owner: ").append(fs.getOwner());
    sb.append(" , Group: ").append(fs.getGroup());
    return sb.toString();
}

---

From source file:alluxio.underfs.hdfs.HdfsUnderFileSystem.java

License:Apache License

@Override
public long getModificationTimeMs(String path) throws IOException {
    Path tPath = new Path(path);
    if (!mFileSystem.exists(tPath)) {
        throw new FileNotFoundException(path);
    }//from www . j a va  2s  .  co m
    FileStatus fs = mFileSystem.getFileStatus(tPath);
    return fs.getModificationTime();
}

---

From source file:alluxio.yarn.YarnUtils.java

License:Apache License

/**
 * Creates a local resource for a file on HDFS.
 *
 * @param yarnConf YARN configuration/*  w w w. j a v  a  2  s  .  c o m*/
 * @param resource the path to a resource file on HDFS
 * @throws IOException if the file can not be found on HDFS
 * @return the created local resource
 */
public static LocalResource createLocalResourceOfFile(YarnConfiguration yarnConf, String resource)
        throws IOException {
    LocalResource localResource = Records.newRecord(LocalResource.class);

    Path resourcePath = new Path(resource);

    FileStatus jarStat = FileSystem.get(resourcePath.toUri(), yarnConf).getFileStatus(resourcePath);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
    return localResource;
}

---

From source file:azkaban.reportal.util.StreamProviderHDFS.java

License:Apache License

public String[] getOldFiles(String pathString, long thresholdTime) throws Exception {
    FileStatus[] statusList = getFileStatusList(pathString);

    List<String> oldFiles = new ArrayList<String>();

    for (FileStatus fs : statusList) {
        if (fs.getModificationTime() < thresholdTime) {
            oldFiles.add(fs.getPath().getName());
        }//ww w  .ja v  a  2 s .  c  o  m
    }

    return oldFiles.toArray(new String[0]);
}

---

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

/**
 * Copies paths from one local path to a remote path. If syncTimes is true, both modification and access time are
 * changed to match the local 'from' path.
 * <p/>/*ww  w  .  j a  v  a2  s.c o  m*/
 * Returns a map of file-name to remote modification times if the remote time is different than the local time.
 *
 * @param config
 * @param commonPaths
 * @param syncTimes
 */
public static Map<String, Long> syncPaths(Configuration config, Map<Path, Path> commonPaths,
        boolean syncTimes) {
    if (commonPaths == null)
        return Collections.emptyMap();

    Map<String, Long> timestampMap = new HashMap<>();

    Map<Path, Path> copyPaths = getCopyPaths(config, commonPaths); // tests remote file existence or if stale

    LocalFileSystem localFS = getLocalFS(config);
    FileSystem remoteFS = getDefaultFS(config);

    for (Map.Entry<Path, Path> entry : copyPaths.entrySet()) {
        Path localPath = entry.getKey();
        Path remotePath = entry.getValue();

        try {
            LOG.info("copying from: {}, to: {}", localPath, remotePath);
            remoteFS.copyFromLocalFile(localPath, remotePath);

            if (!syncTimes) {
                timestampMap.put(remotePath.getName(),
                        remoteFS.getFileStatus(remotePath).getModificationTime());
                continue;
            }
        } catch (IOException exception) {
            throw new FlowException("unable to copy local: " + localPath + " to remote: " + remotePath,
                    exception);
        }

        FileStatus localFileStatus = null;

        try {
            // sync the modified times so we can lazily upload jars to hdfs after job is started
            // otherwise modified time will be local to hdfs
            localFileStatus = localFS.getFileStatus(localPath);
            remoteFS.setTimes(remotePath, localFileStatus.getModificationTime(), -1); // don't set the access time
        } catch (IOException exception) {
            LOG.info(
                    "unable to set local modification time on remote file: {}, 'dfs.namenode.accesstime.precision' may be set to 0 on HDFS.",
                    remotePath);

            if (localFileStatus != null)
                timestampMap.put(remotePath.getName(), localFileStatus.getModificationTime());
        }
    }

    return timestampMap;
}

---

From source file:cascading.flow.tez.util.TezUtil.java

License:Open Source License

protected static void addResource(Map<String, LocalResource> localResources, Map<String, String> environment,
        String fileName, FileStatus stats, Path fullPath, LocalResourceType type) throws IOException {
    if (localResources.containsKey(fileName))
        throw new FlowException("duplicate filename added to classpath resources: " + fileName);

    URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(fullPath);
    long len = stats.getLen();
    long modificationTime = stats.getModificationTime();

    LocalResource resource = LocalResource.newInstance(yarnUrlFromPath, type,
            LocalResourceVisibility.APPLICATION, len, modificationTime);

    if (type == LocalResourceType.PATTERN) {
        // todo: parametrize this for dynamic inclusion below
        String pattern = "(?:classes/|lib/).*";

        resource.setPattern(pattern);//from  w w  w.j  a  v  a 2s.  c o  m

        if (environment != null) {
            String current = "";

            current += PWD.$$() + File.separator + fileName + File.separator + "*" + CLASS_PATH_SEPARATOR;
            current += PWD.$$() + File.separator + fileName + File.separator + "lib" + File.separator + "*"
                    + CLASS_PATH_SEPARATOR;
            current += PWD.$$() + File.separator + fileName + File.separator + "classes" + File.separator + "*"
                    + CLASS_PATH_SEPARATOR;

            String classPath = environment.get(CLASSPATH.name());

            if (classPath == null)
                classPath = "";
            else if (!classPath.startsWith(CLASS_PATH_SEPARATOR))
                classPath += CLASS_PATH_SEPARATOR;

            classPath += current;

            LOG.info("adding to cluster side classpath: {} ", classPath);

            environment.put(CLASSPATH.name(), classPath);
        }
    }

    localResources.put(fileName, resource);
}

---

From source file:cascading.tap.hadoop.Hfs.java

License:Open Source License

@Override
public long getModifiedTime(Configuration conf) throws IOException {
    if (!resourceExists(conf))
        return 0;

    FileStatus fileStatus = getFileSystem(conf).getFileStatus(getPath());

    if (!fileStatus.isDir())
        return fileStatus.getModificationTime();

    // todo: this should ignore the _temporary path, or not cache if found in the array
    makeStatuses(conf);/*w  w w .j  a v a2 s  .  c o m*/

    // statuses is empty, return 0
    if (statuses == null || statuses.length == 0)
        return 0;

    long date = 0;

    // filter out directories as we don't recurs into sub dirs
    for (FileStatus status : statuses) {
        if (!status.isDir())
            date = Math.max(date, status.getModificationTime());
    }

    return date;
}

---

From source file:cascading.tap.Hfs.java

License:Open Source License

@Override
public long getPathModified(JobConf conf) throws IOException {
    FileStatus fileStatus = getFileSystem(conf).getFileStatus(getPath());

    if (!fileStatus.isDir())
        return fileStatus.getModificationTime();

    makeStatuses(conf);/*from w w w. j  a  v  a  2  s  .c om*/

    // statuses is empty, return 0
    if (statuses == null || statuses.length == 0)
        return 0;

    long date = 0;

    // filter out directories as we don't recurse into sub dirs
    for (FileStatus status : statuses) {
        if (!status.isDir())
            date = Math.max(date, status.getModificationTime());
    }

    return date;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »