Example usage for org.apache.hadoop.fs FileStatus getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime() 

Source Link

Document

Get the modification time of the file.

Usage

From source file:org.apache.falcon.entity.FileSystemStorageTest.java

License:Apache License

@SuppressWarnings("MagicConstant")
private List<FeedInstanceStatus> prepareData(FileSystem fs, Feed feed, Date start, Date end) throws Exception {
    fs.delete(new Path("/" + TEST_FEED_LISTING), true);
    Random random = new Random();
    List<FeedInstanceStatus> instances = new ArrayList<FeedInstanceStatus>();
    String basePath = feed.getLocations().getLocations().get(0).getPath();
    Frequency frequency = feed.getFrequency();
    TimeZone tz = feed.getTimezone();
    for (Cluster cluster : feed.getClusters().getClusters()) {
        Date dataStart = EntityUtil.getNextStartTime(cluster.getValidity().getStart(), feed.getFrequency(), tz,
                new Date(start.getTime()));
        String clusterLocationPath = null;
        if (cluster.getLocations() != null && cluster.getLocations().getLocations().get(0).getPath() != null) {
            basePath = clusterLocationPath == null ? basePath : clusterLocationPath;
        }//from w ww .  j av  a2 s . c o m
        Date dataEnd = new Date(end.getTime());
        while (dataStart.before(dataEnd)) {
            Properties properties = ExpressionHelper.getTimeVariables(dataStart, tz);
            String path = ExpressionHelper.substitute(basePath, properties);
            FeedInstanceStatus instance = new FeedInstanceStatus(path);
            instance.setStatus(FeedInstanceStatus.AvailabilityStatus.MISSING);
            instance.setSize(-1);
            instance.setCreationTime(0);
            Date date = FeedHelper.getDate(basePath, new Path(path), tz);
            instance.setInstance(SchemaHelper.formatDateUTC(date));
            Calendar cal = Calendar.getInstance();
            cal.setTime(dataStart);
            cal.add(frequency.getTimeUnit().getCalendarUnit(), frequency.getFrequencyAsInt());
            dataStart.setTime(cal.getTimeInMillis());
            if (random.nextBoolean()) {
                OutputStream out = fs.create(new Path(path, "file"));
                out.write("Hello World\n".getBytes());
                out.close();
                instance.setSize(12);
                if (feed.getAvailabilityFlag() == null
                        || (feed.getAvailabilityFlag() != null && random.nextBoolean())) {
                    //If availability is not present or if ok to create availability file, mark as available
                    instance.setStatus(FeedInstanceStatus.AvailabilityStatus.AVAILABLE);
                    if (feed.getAvailabilityFlag() != null) {
                        fs.create(new Path(path, feed.getAvailabilityFlag())).close();
                    }
                } else if (feed.getAvailabilityFlag() != null) {
                    //If availability is present or not ok to create availability file, mark as partial
                    fs.mkdirs(new Path(path));
                    instance.setStatus(FeedInstanceStatus.AvailabilityStatus.PARTIAL);
                }
            } else {
                if (feed.getAvailabilityFlag() == null && random.nextBoolean()) {
                    //If availability is not present or ok to create dir, mark as empty
                    fs.mkdirs(new Path(path));
                    instance.setStatus(FeedInstanceStatus.AvailabilityStatus.EMPTY);
                    instance.setSize(0);
                } else if (feed.getAvailabilityFlag() != null && random.nextBoolean()) {
                    //If availability is present and ok to create dir, mark as partial
                    fs.mkdirs(new Path(path));
                    instance.setStatus(FeedInstanceStatus.AvailabilityStatus.PARTIAL);
                } else if (feed.getAvailabilityFlag() != null) {
                    //If availability is present and ok to create empty instance
                    fs.create(new Path(path, feed.getAvailabilityFlag())).close();
                    instance.setStatus(FeedInstanceStatus.AvailabilityStatus.EMPTY);
                    instance.setSize(0);
                }
            }
            try {
                FileStatus fileStatus = fs.getFileStatus(new Path(path));
                instance.setCreationTime(fileStatus.getModificationTime());
            } catch (IOException e) {
                //ignore
            }
            instances.add(instance);
        }
    }
    return instances;
}

---

From source file:org.apache.falcon.hadoop.JailedFileSystem.java

License:Apache License

@Override
public FileStatus[] listStatus(Path f) throws IOException {
    FileStatus[] fileStatuses = localFS.listStatus(toLocalPath(f));
    if (fileStatuses == null || fileStatuses.length == 0) {
        return fileStatuses;
    } else {/*from ww  w .  j  a  va 2s.c  o  m*/
        FileStatus[] jailFileStatuses = new FileStatus[fileStatuses.length];
        for (int index = 0; index < fileStatuses.length; index++) {
            FileStatus status = fileStatuses[index];
            jailFileStatuses[index] = new FileStatus(status.getLen(), status.isDirectory(),
                    status.getReplication(), status.getBlockSize(), status.getModificationTime(),
                    status.getAccessTime(), status.getPermission(), status.getOwner(), status.getGroup(),
                    fromLocalPath(status.getPath()).makeQualified(this.getUri(), this.getWorkingDirectory()));
        }
        return jailFileStatuses;
    }
}

---

From source file:org.apache.falcon.hadoop.JailedFileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(Path f) throws IOException {
    FileStatus status = localFS.getFileStatus(toLocalPath(f));
    if (status == null) {
        return null;
    }/*from   w  w  w .j a  va2s .c om*/
    return new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(), status.getBlockSize(),
            status.getModificationTime(), status.getAccessTime(), status.getPermission(), status.getOwner(),
            status.getGroup(),
            fromLocalPath(status.getPath()).makeQualified(this.getUri(), this.getWorkingDirectory()));
}

---

From source file:org.apache.falcon.snapshots.replication.HdfsSnapshotReplicator.java

License:Apache License

private String findLatestReplicatedSnapshot(DistributedFileSystem sourceFs, DistributedFileSystem targetFs,
        String sourceDir, String targetDir) throws FalconException {
    try {//from w w w.j  a  va 2s .  c  o m
        FileStatus[] sourceSnapshots = sourceFs.listStatus(new Path(getSnapshotDir(sourceDir)));
        Set<String> sourceSnapshotNames = new HashSet<>();
        for (FileStatus snapshot : sourceSnapshots) {
            sourceSnapshotNames.add(snapshot.getPath().getName());
        }

        FileStatus[] targetSnapshots = targetFs.listStatus(new Path(getSnapshotDir(targetDir)));
        if (targetSnapshots.length > 0) {
            //sort target snapshots in desc order of creation time.
            Arrays.sort(targetSnapshots, new Comparator<FileStatus>() {
                @Override
                public int compare(FileStatus f1, FileStatus f2) {
                    return Long.compare(f2.getModificationTime(), f1.getModificationTime());
                }
            });

            // get most recent snapshot name that exists in source.
            for (FileStatus targetSnapshot : targetSnapshots) {
                String name = targetSnapshot.getPath().getName();
                if (sourceSnapshotNames.contains(name)) {
                    return name;
                }
            }
            // If control reaches here,
            // there are snapshots on target, but none are replicated from source. Return null.
        } // No target snapshots, return null
        return null;
    } catch (IOException e) {
        LOG.error("Unable to find latest snapshot on targetDir {} {}", targetDir, e.getMessage());
        throw new FalconException("Unable to find latest snapshot on targetDir " + targetDir, e);
    }
}

---

From source file:org.apache.falcon.snapshots.retention.HdfsSnapshotEvictor.java

License:Apache License

protected static void evictSnapshots(DistributedFileSystem fs, String dirName, String ageLimit,
        int numSnapshots) throws FalconException {
    try {// w w  w.j a  v  a2 s .c  o m
        LOG.info("Started evicting snapshots on dir {}{} using policy {}, agelimit {}, numSnapshot {}",
                fs.getUri(), dirName, ageLimit, numSnapshots);

        long evictionTime = System.currentTimeMillis() - EvictionHelper.evalExpressionToMilliSeconds(ageLimit);

        dirName = StringUtils.removeEnd(dirName, Path.SEPARATOR);
        String snapshotDir = dirName + Path.SEPARATOR + HdfsSnapshotUtil.SNAPSHOT_DIR_PREFIX + Path.SEPARATOR;
        FileStatus[] snapshots = fs.listStatus(new Path(snapshotDir));
        if (snapshots.length <= numSnapshots) {
            // no eviction needed
            return;
        }

        // Sort by last modified time, ascending order.
        Arrays.sort(snapshots, new Comparator<FileStatus>() {
            @Override
            public int compare(FileStatus f1, FileStatus f2) {
                return Long.compare(f1.getModificationTime(), f2.getModificationTime());
            }
        });

        for (int i = 0; i < (snapshots.length - numSnapshots); i++) {
            // delete if older than ageLimit while retaining numSnapshots
            if (snapshots[i].getModificationTime() < evictionTime) {
                fs.deleteSnapshot(new Path(dirName), snapshots[i].getPath().getName());
            }
        }

    } catch (ELException ele) {
        LOG.warn("Unable to parse retention age limit {} {}", ageLimit, ele.getMessage());
        throw new FalconException("Unable to parse retention age limit " + ageLimit, ele);
    } catch (IOException ioe) {
        LOG.warn("Unable to evict snapshots from dir {} {}", dirName, ioe);
        throw new FalconException("Unable to evict snapshots from dir " + dirName, ioe);
    }

}

---

From source file:org.apache.flink.tez.client.TezExecutor.java

License:Apache License

private static void addLocalResource(TezConfiguration tezConf, Path jarPath, DAG dag) {

    try {//  w  ww  .j a v  a2 s.  c om
        org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(tezConf);

        LOG.info("Jar path received is " + jarPath.toString());

        String jarFile = jarPath.getName();

        Path remoteJarPath = null;

        /*
        if (tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR) == null) {
           LOG.info("Tez staging directory is null, setting it.");
           Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
           LOG.info("Setting Tez staging directory to " + stagingDir.toString());
           tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
           LOG.info("Set Tez staging directory to " + stagingDir.toString());
        }
        Path stagingDir = new Path(tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR));
        LOG.info("Ensuring that Tez staging directory exists");
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging directory exists and is " + stagingDir.toString());
        */

        Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf);
        LOG.info("Tez staging path is " + stagingDir);
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging dir exists");

        remoteJarPath = fs.makeQualified(new Path(stagingDir, jarFile));
        LOG.info("Copying " + jarPath.toString() + " to " + remoteJarPath.toString());
        fs.copyFromLocalFile(jarPath, remoteJarPath);

        FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
        Credentials credentials = new Credentials();
        TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, tezConf);

        Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();
        LocalResource jobJar = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
                remoteJarStatus.getModificationTime());
        localResources.put(jarFile.toString(), jobJar);

        dag.addTaskLocalFiles(localResources);

        LOG.info("Added job jar as local resource.");
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        System.exit(-1);
    }
}

---

From source file:org.apache.giraph.yarn.YarnUtils.java

License:Apache License

/**
 * Boilerplate to add a file to the local resources..
 * @param localResources the LocalResources map to populate.
 * @param fs handle to the HDFS file system.
 * @param target the file to send to the remote container.
 *//*from w  ww.  jav  a  2 s .c  om*/
public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target)
        throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus destStatus = fs.getFileStatus(target);
    resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri()));
    resource.setSize(destStatus.getLen());
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setType(LocalResourceType.FILE); // use FILE, even for jars!
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResources.put(target.getName(), resource);
    LOG.info("Registered file in LocalResources :: " + target);
}

---

From source file:org.apache.gobblin.data.management.copy.replication.SourceHadoopFsEndPoint.java

License:Apache License

@Override
public synchronized Optional<ComparableWatermark> getWatermark() {
    if (this.initialized) {
        return this.cachedWatermark;
    }/* w w  w.  j  ava  2s .  co  m*/
    try {
        long curTs = -1;
        FileSystem fs = FileSystem.get(rc.getFsURI(), new Configuration());

        Collection<Path> validPaths = ReplicationDataValidPathPicker.getValidPaths(this);
        for (Path p : validPaths) {
            this.allFileStatus.addAll(FileListUtils.listFilesRecursively(fs, p, super.getPathFilter(),
                    super.isApplyFilterToDirectories()));
        }

        for (FileStatus f : this.allFileStatus) {
            if (f.getModificationTime() > curTs) {
                curTs = f.getModificationTime();
            }
        }

        ComparableWatermark result = new LongWatermark(curTs);
        this.cachedWatermark = Optional.of(result);

        if (this.cachedWatermark.isPresent()) {
            this.initialized = true;
        }

        return this.cachedWatermark;
    } catch (IOException e) {
        log.error("Error while retrieve the watermark for " + this);
        return this.cachedWatermark;
    }
}

---

From source file:org.apache.gobblin.data.management.retention.Avro2OrcStaleDatasetCleaner.java

License:Apache License

private void deletePath(FileStatus fileStatus, long graceTimeInMillis, boolean recursively) {
    long modificationTime = fileStatus.getModificationTime();
    long currentTime = System.currentTimeMillis();
    if ((currentTime - modificationTime) < 0) {
        log.error("Modification time cannot be greater than current time: " + fileStatus.getPath());
        return;//  www  .ja  v a  2s.co  m
    }
    if ((currentTime - modificationTime) < graceTimeInMillis) {
        log.info("Modification time is still within grace time for deletion: " + fileStatus.getPath());
        return;
    }
    try {
        this.fs.delete(fileStatus.getPath(), recursively);
        log.info("Deleted path " + fileStatus.getPath());
    } catch (IOException e) {
        log.error("Unable to delete directory " + fileStatus.getPath(), e);
    }
}

---

From source file:org.apache.gobblin.runtime.metastore.filesystem.FsDatasetStateStoreEntryManager.java

License:Apache License

public FsDatasetStateStoreEntryManager(FileStatus fileStatus, FsDatasetStateStore stateStore) {
    super(fileStatus.getPath().getParent().getName(), fileStatus.getPath().getName(),
            fileStatus.getModificationTime(),
            new DatasetStateStore.TableNameParser(fileStatus.getPath().getName()), stateStore);
    this.stateStore = stateStore;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »