Example usage for org.apache.hadoop.fs FileStatus getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime() 

Source Link

Document

Get the modification time of the file.

Usage

From source file:crunch.MaxTemperature.java

License:Apache License

@Test
    public void fileStatusForFile() throws IOException {
        Path file = new Path("/dir/file"); // XXX new Path creates the file
        FileStatus stat = fs.getFileStatus(file);
        assertThat(stat.getPath().toUri().getPath(), is("/dir/file")); // XXX FileStatus.getPath().toUri() -> URI .getPath()
        assertThat(stat.isDir(), is(false)); // XXX assertThat(actual, Matcher) Matcher provides matches method
        assertThat(stat.getLen(), is(7L));
        assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
        assertThat(stat.getReplication(), is((short) 1)); // XXX Matcher<Short> is(Short value) -> o.h.core.Is.<Short>is(Short) static factory method from oh.core.Is
        // XXX which calls the constructor is(Matcher<Short> matcher) with the matcher equalTo(Short) hamcrest-all-1.3-source.java:Is.java:65
        assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L));
        assertThat(stat.getOwner(), is(System.getProperty("user.name")));
        assertThat(stat.getGroup(), is("supergroup"));
        assertThat(stat.getPermission().toString(), is("rw-r--r--"));
    }//from   www  . j  a va  2  s .c  om

---

From source file:crunch.MaxTemperature.java

License:Apache License

@Test
    public void fileStatusForDirectory() throws IOException {
        Path dir = new Path("/dir"); // XXX new Path creates the directory
        FileStatus stat = fs.getFileStatus(dir);
        assertThat(stat.getPath().toUri().getPath(), is("/dir"));
        assertThat(stat.isDir(), is(true));
        assertThat(stat.getLen(), is(0L));
        assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
        assertThat(stat.getReplication(), is((short) 0));
        assertThat(stat.getBlockSize(), is(0L));
        assertThat(stat.getOwner(), is(System.getProperty("user.name")));
        assertThat(stat.getGroup(), is("supergroup"));
        assertThat(stat.getPermission().toString(), is("rwxr-xr-x"));
    }//  w  w  w  .  j  a va  2 s.c  om

---

From source file:de.gesundkrank.wikipedia.hadoop.io.WikiDumpLoader.java

License:Open Source License

/**
 * Adds inputpath to a given hadoop job/*  www .java2s.  c  om*/
 *
 * @param job         hadoop job
 * @param basePathStr
 * @param locale      Language of the wikidump
 * @throws java.io.IOException
 */
public void addWikiDump(Job job, String basePathStr, Locale locale) throws IOException {
    Path basePath = new Path(basePathStr, locale.getLanguage());
    FileSystem fs = FileSystem.get(job.getConfiguration());
    FileStatus latestLocalDumpStatus = checkLocalDumps(fs, basePath);
    Path latestDump = null;
    if (latestLocalDumpStatus != null) {
        latestDump = latestLocalDumpStatus.getPath();
    } else {
        logger.info("Could not find a valid dump. Loading new version.");
    }

    if (checkNew || latestDump == null) {
        long latestDumpTime = checkNewDump(locale);
        if (latestLocalDumpStatus == null || latestDumpTime > latestLocalDumpStatus.getModificationTime()) {
            latestDump = loadNewDump(fs, basePath, latestDumpTime, locale);
        } else {
            throw new IOException("failed to get latest dump");
        }
    }

    FileInputFormat.addInputPath(job, latestDump);
}

---

From source file:de.gesundkrank.wikipedia.hadoop.io.WikiDumpLoader.java

License:Open Source License

/**
 * Returns {@link FileStatus} of the latest dump in the HDFS
 *
 * @param fs       HDFS/*from  w  w  w  .j  a  va2  s . com*/
 * @param basepath Base path of hdfs wikidumps
 * @return
 */
private FileStatus checkLocalDumps(FileSystem fs, Path basepath) {

    long lastLocalChange = 0;
    FileStatus lastLocalDump = null;
    try {
        if (!fs.exists(basepath)) {
            fs.mkdirs(basepath);
            return null;
        }

        FileStatus[] stati = fs.listStatus(basepath);

        for (FileStatus status : stati) {
            long fileChange = status.getModificationTime();
            if (fileChange > lastLocalChange) {
                lastLocalDump = status;
            }
        }
    } catch (IOException e) {
        logger.error(e);
    }
    return lastLocalDump;

}

---

From source file:de.huberlin.wbi.hiway.common.Data.java

License:Apache License

/** Adds this object to the given map (used in the workflow driver?). */
public void addToLocalResourceMap(Map<String, LocalResource> localResources) throws IOException {
    Path dest = getHdfsPath();/*from   w  ww. ja va 2s.com*/

    // Record is a Hadoop YARN util class
    LocalResource rsrc = Records.newRecord(LocalResource.class);
    rsrc.setType(LocalResourceType.FILE);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc.setResource(ConverterUtils.getYarnUrlFromPath(dest));

    FileStatus status = hdfs.getFileStatus(dest);
    rsrc.setTimestamp(status.getModificationTime());
    rsrc.setSize(status.getLen());

    localResources.put(getLocalPath().toString(), rsrc);
}

---

From source file:eagle.security.hdfs.entity.FileStatusEntity.java

License:Apache License

public FileStatusEntity(FileStatus status) throws IOException {
    //this.path = status.getPath();
    this.length = status.getLen();
    this.isdir = status.isDirectory();
    this.block_replication = status.getReplication();
    this.blocksize = status.getBlockSize();
    this.modification_time = status.getModificationTime();
    this.access_time = status.getAccessTime();
    this.permission = status.getPermission();
    this.owner = status.getOwner();
    this.group = status.getGroup();
    if (status.isSymlink()) {
        this.symlink = status.getSymlink();
    }/*from ww w . j  a v  a 2s .co m*/
}

---

From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java

License:Apache License

public static LocalResource addLocalResource(String filePath) throws Exception {
    File file = new File(filePath);
    String fullFilePath = file.getAbsolutePath();

    FileSystem fs = FileSystem.get(IO.getConf());
    Path src = new Path(fullFilePath);
    String pathSuffix = "local-tmp/" + file.getName();
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    fs.copyFromLocalFile(false, true, src, dst);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource resource = Records.newRecord(LocalResource.class);

    resource.setType(LocalResourceType.FILE);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setSize(destStatus.getLen());

    return resource;
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

/**
 * Sets up the parameters for the Asterix config.
 * //from  ww w  . j a  v a  2s.c  o  m
 * @throws IOException
 */
private void distributeAsterixConfig() throws IOException {
    FileSystem fs = FileSystem.get(conf);
    String pathSuffix = instanceConfPath + File.separator + ASTERIX_CONF_NAME;
    Path dst = new Path(dfsBasePath, pathSuffix);
    URI paramLocation = dst.toUri();
    FileStatus paramFileStatus = fs.getFileStatus(dst);
    Long paramLen = paramFileStatus.getLen();
    Long paramTimestamp = paramFileStatus.getModificationTime();
    LocalResource asterixParamLoc = Records.newRecord(LocalResource.class);
    asterixParamLoc.setType(LocalResourceType.FILE);
    asterixParamLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixParamLoc.setResource(ConverterUtils.getYarnUrlFromURI(paramLocation));
    asterixParamLoc.setTimestamp(paramTimestamp);
    asterixParamLoc.setSize(paramLen);
    localResources.put(ASTERIX_CONF_NAME, asterixParamLoc);

}

---

From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

/**
 * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client
 * filesystem of the nodes.//ww w.  java 2 s.co  m
 * 
 * @throws IOException
 */
private void localizeDFSResources() throws IOException {
    //if performing an 'offline' task, skip a lot of resource distribution
    if (obliterate || backup || restore) {
        if (appMasterJar == null || ("").equals(appMasterJar)) {
            //this can happen in a jUnit testing environment. we don't need to set it there. 
            if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
                throw new IllegalStateException("AM jar not provided in environment.");
            } else {
                return;
            }
        }
        FileSystem fs = FileSystem.get(conf);
        FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar);
        LocalResource obliteratorJar = Records.newRecord(LocalResource.class);
        obliteratorJar.setType(LocalResourceType.FILE);
        obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE);
        obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar));
        obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime());
        obliteratorJar.setSize(appMasterJarStatus.getLen());
        localResources.put("asterix-yarn.jar", obliteratorJar);
        LOG.info(localResources.values());
        return;
    }
    //otherwise, distribute evertything to start up asterix

    LocalResource asterixZip = Records.newRecord(LocalResource.class);

    //this un-tar's the asterix distribution
    asterixZip.setType(LocalResourceType.ARCHIVE);

    asterixZip.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath)));

    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath);
        throw new IOException(e);
    }

    asterixZip.setTimestamp(asterixZipTimestamp);
    asterixZip.setSize(asterixZipLen);
    localResources.put(ASTERIX_ZIP_NAME, asterixZip);

    //now let's do the same for the cluster description XML
    LocalResource asterixConf = Records.newRecord(LocalResource.class);
    asterixConf.setType(LocalResourceType.FILE);

    asterixConf.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath)));
    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath);
        throw new IOException(e);
    }
    //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node. 
    asterixConf.setTimestamp(asterixConfTimestamp);
    asterixConf.setSize(asterixConfLen);
    localResources.put("cluster-config.xml", asterixConf);
    //now add the libraries if there are any
    try {
        FileSystem fs = FileSystem.get(conf);
        Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR);
        if (fs.exists(p)) {
            FileStatus[] dataverses = fs.listStatus(p);
            for (FileStatus d : dataverses) {
                if (!d.isDirectory())
                    throw new IOException("Library configuration directory structure is incorrect");
                FileStatus[] libraries = fs.listStatus(d.getPath());
                for (FileStatus l : libraries) {
                    if (l.isDirectory())
                        throw new IOException("Library configuration directory structure is incorrect");
                    LocalResource lr = Records.newRecord(LocalResource.class);
                    lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri()));
                    lr.setSize(l.getLen());
                    lr.setTimestamp(l.getModificationTime());
                    lr.setType(LocalResourceType.ARCHIVE);
                    lr.setVisibility(LocalResourceVisibility.PRIVATE);
                    localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR
                            + l.getPath().getName().split("\\.")[0], lr);
                    LOG.info("Found library: " + l.getPath().toString());
                    LOG.info(l.getPath().getName());
                }
            }
        }
    } catch (FileNotFoundException e) {
        LOG.info("No external libraries present");
        //do nothing, it just means there aren't libraries. that is possible and ok
        // it should be handled by the fs.exists(p) check though.
    }
    LOG.info(localResources.values());

}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Upload the Asterix cluster description on to the DFS. This will persist the state of the instance.
 * //from  www.  ja  v a 2  s  .  co  m
 * @return
 * @throws YarnException
 * @throws IOException
 */
private List<DFSResourceCoordinate> deployConfig() throws YarnException, IOException {

    FileSystem fs = FileSystem.get(conf);
    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);

    String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME;
    Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix);
    FileStatus destStatus;
    try {
        destStatus = fs.getFileStatus(dstConf);
    } catch (IOException e) {
        throw new YarnException("Asterix instance by that name does not appear to exist in DFS");
    }
    LocalResource asterixConfLoc = Records.newRecord(LocalResource.class);
    asterixConfLoc.setType(LocalResourceType.FILE);
    asterixConfLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixConfLoc.setResource(ConverterUtils.getYarnUrlFromPath(dstConf));
    asterixConfLoc.setTimestamp(destStatus.getModificationTime());

    DFSResourceCoordinate conf = new DFSResourceCoordinate();
    conf.envs.put(dstConf.toUri().toString(), AConstants.CONFLOCATION);
    conf.envs.put(Long.toString(asterixConfLoc.getSize()), AConstants.CONFLEN);
    conf.envs.put(Long.toString(asterixConfLoc.getTimestamp()), AConstants.CONFTIMESTAMP);
    conf.name = CONFIG_DEFAULT_NAME;
    conf.res = asterixConfLoc;
    resources.add(conf);

    return resources;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »