Example usage for org.apache.hadoop.fs FileStatus getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime() 

Source Link

Document

Get the modification time of the file.

Usage

From source file:org.elasticsearch.hadoop.yarn.client.YarnLauncher.java

License:Apache License

private Map<String, LocalResource> setupEsYarnJar() {
    Map<String, LocalResource> resources = new LinkedHashMap<String, LocalResource>();
    LocalResource esYarnJar = Records.newRecord(LocalResource.class);
    Path p = new Path(clientCfg.jarHdfsPath());
    FileStatus fsStat;
    try {//from   ww  w.  j a  v a2s.  c o m
        fsStat = FileSystem.get(client.getConfiguration()).getFileStatus(p);
    } catch (IOException ex) {
        throw new IllegalArgumentException(String.format(
                "Cannot find jar [%s]; make sure the artifacts have been properly provisioned and the correct permissions are in place.",
                clientCfg.jarHdfsPath()), ex);
    }
    // use the normalized path as otherwise YARN chokes down the line
    esYarnJar.setResource(ConverterUtils.getYarnUrlFromPath(fsStat.getPath()));
    esYarnJar.setSize(fsStat.getLen());
    esYarnJar.setTimestamp(fsStat.getModificationTime());
    esYarnJar.setType(LocalResourceType.FILE);
    esYarnJar.setVisibility(LocalResourceVisibility.PUBLIC);

    resources.put(clientCfg.jarName(), esYarnJar);
    return resources;
}

---

From source file:org.etosha.cumulusonyarn.CumulusRDFRunner.java

License:Apache License

/**
 * Main run function for the client//from  ww  w.j a  v a 2  s  . c o m
 * 
 * @return true if application completed successfully
 * @throws IOException
 * @throws YarnException
 */
public boolean run() throws IOException, YarnException {

    LOG.info("Running Client");
    yarnClient.start();

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());

    List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM");
    for (NodeReport node : clusterNodeReports) {
        LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                + node.getNumContainers());
    }

    QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
            + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
            + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
            + queueInfo.getChildQueues().size());

    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                    + userAcl.name());
        }
    }

    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setApplicationName(appName);

    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    FileSystem fs = FileSystem.get(conf);
    Path src = new Path(appJar);
    String pathSuffix = appName + File.separator + appId.getId() + File.separator
            + CumulusConstants.JBOSS_ON_YARN_APP;
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    jbossAppUri = dst.toUri().toString();
    fs.copyFromLocalFile(false, true, src, dst);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);

    amJarRsrc.setType(LocalResourceType.FILE);
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    localResources.put(CumulusConstants.JBOSS_ON_YARN_APP, amJarRsrc);

    if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        localResources.put("log4j.properties", log4jRsrc);
    }

    amContainer.setLocalResources(localResources);

    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");

    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());

    amContainer.setEnvironment(env);

    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    vargs.add("-Xmx" + amMemory + "m");
    vargs.add(appMasterMainClass);
    vargs.add("--container_memory " + String.valueOf(containerMemory));
    vargs.add("--num_containers " + String.valueOf(numContainers));
    vargs.add("--priority " + String.valueOf(shellCmdPriority));
    vargs.add("--admin_user " + adminUser);
    vargs.add("--admin_password " + adminPassword);
    vargs.add("--jar " + jbossAppUri);

    if (debugFlag) {
        vargs.add("--debug");
    }

    vargs.add("1>" + CumulusConstants.JBOSS_CONTAINER_LOG_DIR + "/JBossApplicationMaster.stdout");
    vargs.add("2>" + CumulusConstants.JBOSS_CONTAINER_LOG_DIR + "/JBossApplicationMaster.stderr");

    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    amContainer.setCommands(commands);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMemory);
    appContext.setResource(capability);

    appContext.setAMContainerSpec(amContainer);

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    appContext.setQueue(amQueue);

    LOG.info("Submitting the application to ASM");

    yarnClient.submitApplication(appContext);

    return monitorApplication(appId);
}

---

From source file:org.exem.flamingo.agent.nn.hdfs.HdfsFileInfo.java

License:Apache License

public HdfsFileInfo(FileStatus fileStatus, ContentSummary contentSummary) {
    this.fullyQualifiedPath = fileStatus.getPath().toUri().getPath();
    this.filename = isEmpty(getFilename(fullyQualifiedPath)) ? getDirectoryName(fullyQualifiedPath)
            : getFilename(fullyQualifiedPath);
    this.length = fileStatus.isFile() ? fileStatus.getLen() : contentSummary.getLength();
    this.path = getPath(fullyQualifiedPath);
    this.directory = fileStatus.isDirectory();
    this.file = !fileStatus.isDirectory();
    this.owner = fileStatus.getOwner();
    this.group = fileStatus.getGroup();
    this.blockSize = fileStatus.getBlockSize();
    this.replication = fileStatus.getReplication();
    this.modificationTime = fileStatus.getModificationTime();
    if (contentSummary != null) {
        this.spaceConsumed = contentSummary.getSpaceConsumed();
        this.spaceQuota = contentSummary.getSpaceQuota();
        this.quota = contentSummary.getQuota();
        this.directoryCount = contentSummary.getDirectoryCount();
        this.fileCount = contentSummary.getFileCount();
    }//from w  w w  .  jav  a2s  . c o m
    this.accessTime = fileStatus.getAccessTime();
    this.permission = fileStatus.getPermission().toString();
}

---

From source file:org.exem.flamingo.agent.nn.hdfs.HdfsFileOnlyInfo.java

License:Apache License

public HdfsFileOnlyInfo(FileStatus fileStatus, ContentSummary contentSummary) {
    String qualifiedPath = fileStatus.getPath().toUri().getPath();
    this.filename = isEmpty(getFilename(qualifiedPath)) ? getDirectoryName(qualifiedPath)
            : getFilename(qualifiedPath);
    this.length = fileStatus.getLen();
    this.path = getPath(qualifiedPath);
    this.directory = fileStatus.isDirectory();
    this.modificationTime = fileStatus.getModificationTime();
    this.file = !fileStatus.isDirectory();
    this.replication = fileStatus.getReplication();
    this.owner = fileStatus.getOwner();
    this.group = fileStatus.getGroup();
    this.permission = fileStatus.getPermission().toString();
    if (contentSummary != null) {
        this.spaceConsumed = contentSummary.getSpaceConsumed();
        this.spaceQuota = contentSummary.getSpaceQuota();
    }/*from   ww w  . java2  s . c  om*/
}

---

From source file:org.godhuli.rhipe.FileUtils.java

License:Apache License

private void ls__(String path, ArrayList<String> lsco, boolean dorecurse)
        throws IOException, FileNotFoundException, URISyntaxException {

    Path spath = null;/*from w  w w.  j  av  a  2s  .  c o  m*/
    spath = new Path(path);
    FileSystem srcFS = spath.getFileSystem(getConf());
    FileStatus[] srcs;
    srcs = srcFS.globStatus(spath);
    if (srcs == null || srcs.length == 0) {
        throw new FileNotFoundException("Cannot access " + path + ": No such file or directory.");
    }
    if (srcs.length == 1 && srcs[0].isDir())
        srcs = srcFS.listStatus(srcs[0].getPath());
    Calendar c = Calendar.getInstance();
    for (FileStatus status : srcs) {
        StringBuilder sb = new StringBuilder();
        boolean idir = status.isDir();
        String x = idir ? "d" : "-";
        if (dorecurse && idir)
            ls__(status.getPath().toUri().getPath(), lsco, dorecurse);
        else {
            sb.append(x);
            sb.append(status.getPermission().toString());
            sb.append(fsep);

            sb.append(status.getOwner());
            sb.append(fsep);

            sb.append(status.getGroup());
            sb.append(fsep);

            sb.append(status.getLen());
            sb.append(fsep);

            Date d = new Date(status.getModificationTime());
            sb.append(formatter.format(d));
            sb.append(fsep);

            sb.append(status.getPath().toUri().getPath());
            lsco.add(sb.toString());
        }
    }
}

---

From source file:org.gridgain.grid.kernal.ggfs.hadoop.GridGgfsHadoopFileSystemWrapper.java

License:Open Source License

/** {@inheritDoc} */
@Override/* w  ww. j a v a  2  s . com*/
public GridGgfsFile info(final GridGgfsPath path) throws GridException {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new GridGgfsFile() {
            @Override
            public GridGgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                return (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };

    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.hdl.tensorflow.yarn.util.Utils.java

License:Apache License

private static void addToLocalResources(FileSystem fs, String key, Path dst,
        Map<String, LocalResource> localResources) throws IOException {
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource resource = LocalResource.newInstance(URL.fromURI(dst.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(key, resource);/*from   w  w w .ja  va 2s  .c  o  m*/
}

---

From source file:org.honu.datacollection.writer.localfs.LocalToRemoteHdfsMover.java

License:Apache License

protected void cleanup() throws Exception {
    try {// ww w .jav a 2s  .  c o m
        int rotateInterval = conf.getInt("honu.collector." + group + "rotateInterval", 1000 * 60 * 5);// defaults to 5 minutes

        Path pLocalOutputDir = new Path(localOutputDir);
        FileStatus[] files = localFs.listStatus(pLocalOutputDir);
        String fileName = null;
        for (FileStatus file : files) {
            fileName = file.getPath().getName();
            if (fileName.endsWith(".done")) {
                moveFile(localOutputDir + fileName);
            } else if (fileName.endsWith(".chukwa")) {
                long lastPeriod = System.currentTimeMillis() - rotateInterval - (2 * 60 * 1000);
                if (file.getModificationTime() < lastPeriod) {
                    log.info(group + "- Moving .chukwa file over, " + localOutputDir + fileName);
                    moveFile(localOutputDir + fileName);
                }
            }
        }
    } catch (Exception e) {
        log.warn("Cannot copy to the remote HDFS", e);
        throw e;
    }
}

---

From source file:org.hortonworks.dovetail.client.Client.java

License:Apache License

/**
 * Main run function for the client//from  w  w w .  j  av  a  2 s. co m
 * 
 * @return true if application completed successfully
 * @throws IOException
 * @throws YarnException
 */
private boolean run() throws IOException, YarnException {

    FileSystem fs = FileSystem.get(conf);
    Deployer deployer = new Deployer(fs, conf);
    deployer.deployArtifacts();
    Path amPath = deployer.getAppMasterPath();
    List<Path> configPaths = deployer.getConfigPaths();

    LOG.info("Running Client");
    yarnClient.start();

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());

    List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM");
    for (NodeReport node : clusterNodeReports) {
        LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                + node.getNumContainers());
    }

    QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
            + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
            + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
            + queueInfo.getChildQueues().size());

    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                    + userAcl.name());
        }
    }

    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setApplicationName(appName);

    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    FileStatus amStatus = fs.getFileStatus(amPath);
    LocalResource amJarFileResource = Records.newRecord(LocalResource.class);

    amJarFileResource.setType(LocalResourceType.FILE);
    amJarFileResource.setVisibility(LocalResourceVisibility.APPLICATION);
    amJarFileResource.setResource(ConverterUtils.getYarnUrlFromPath(amPath));
    amJarFileResource.setTimestamp(amStatus.getModificationTime());
    amJarFileResource.setSize(amStatus.getLen());
    localResources.put(conf.get(DovetailConfiguration.DOVETAIL_AM_JAR), amJarFileResource);

    for (Path path : configPaths) {

        FileStatus configFileStatus = fs.getFileStatus(path);
        LocalResource configFileResource = Records.newRecord(LocalResource.class);
        configFileResource.setType(LocalResourceType.FILE);
        configFileResource.setVisibility(LocalResourceVisibility.APPLICATION);
        configFileResource.setResource(ConverterUtils.getYarnUrlFromURI(path.toUri()));
        configFileResource.setTimestamp(configFileStatus.getModificationTime());
        configFileResource.setSize(configFileStatus.getLen());

        localResources.put(path.getName(), configFileResource);
    }

    fs.close();

    amContainer.setLocalResources(localResources);

    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }

    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());

    env.put("JAVA_HOME", System.getProperty("dovetail.java.home"));
    LOG.info("JAVA_HOME=" + System.getenv("JAVA_HOME"));
    amContainer.setEnvironment(env);

    amContainer.setCommands(AppMasterCommand.getCommands(conf));

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMemory);
    appContext.setResource(capability);

    appContext.setAMContainerSpec(amContainer);

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    appContext.setQueue(amQueue);

    LOG.info("Submitting the application to ASM");

    yarnClient.submitApplication(appContext);

    return monitorApplication(appId);
}

---

From source file:org.kitesdk.data.spi.filesystem.FileSystemView.java

License:Apache License

@Override
public long getLastModified() {
    long lastMod = -1;
    for (Iterator<Path> i = dirIterator(); i.hasNext();) {
        Path dir = i.next();//w ww  .  j  a  v  a  2s . co  m
        try {
            for (FileStatus st : fs.listStatus(dir)) {
                if (lastMod < st.getModificationTime()) {
                    lastMod = st.getModificationTime();
                }
            }
        } catch (IOException e) {
            throw new DatasetIOException("Cannot find last modified time of of " + dir, e);
        }
    }

    // if view was marked ready more recently count it as the modified time
    if (signalManager != null) {
        long readyTimestamp = signalManager.getReadyTimestamp(getConstraints());
        if (lastMod < readyTimestamp) {
            lastMod = readyTimestamp;
        }
    }

    return lastMod;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »