Example usage for org.apache.hadoop.fs FileStatus getModificationTime

List of usage examples for org.apache.hadoop.fs FileStatus getModificationTime

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime()

Source Link

Document

Get the modification time of the file.

Usage

From source file:org.apache.gobblin.util.PullFileLoader.java

License:Apache License

private List<PathWithTimeStamp> fetchJobFilePathsRecursivelyHelper(Path path) {
    List<PathWithTimeStamp> paths = Lists.newArrayList();
    try {/*w w w .j av a 2 s.c  o m*/
        FileStatus[] statuses = this.fs.listStatus(path);
        if (statuses == null) {
            log.error("Path does not exist: " + path);
            return paths;
        }

        for (FileStatus status : statuses) {
            if (status.isDirectory()) {
                paths.addAll(fetchJobFilePathsRecursivelyHelper(status.getPath()));
            } else if (this.javaPropsPullFileFilter.accept(status.getPath())) {
                log.debug("modification time of {} is {}", status.getPath(), status.getModificationTime());
                paths.add(new PathWithTimeStamp(status.getModificationTime(), status.getPath()));
            } else if (this.hoconPullFileFilter.accept(status.getPath())) {
                log.debug("modification time of {} is {}", status.getPath(), status.getModificationTime());
                paths.add(new PathWithTimeStamp(status.getModificationTime(), status.getPath()));
            }
        }

        return paths;
    } catch (IOException ioe) {
        log.error("Could not load properties at path: " + path, ioe);
        return Lists.newArrayList();
    }
}

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

@Override
protected RunningJob launchJob(BSPJobID jobId, BSPJob normalJob, Path submitJobFile, FileSystem pFs)
        throws IOException {
    YARNBSPJob job = (YARNBSPJob) normalJob;

    LOG.info("Submitting job...");
    if (getConf().get("bsp.child.mem.in.mb") == null) {
        LOG.warn("BSP Child memory has not been set, YARN will guess your needs or use default values.");
    }// w ww .  j  a  v a 2 s. c o  m

    FileSystem fs = pFs;
    if (fs == null) {
        fs = FileSystem.get(getConf());
    }

    if (getConf().get("bsp.user.name") == null) {
        String s = getUnixUserName();
        getConf().set("bsp.user.name", s);
        LOG.debug("Retrieved username: " + s);
    }

    yarnClient.start();
    try {
        YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
        LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers="
                + clusterMetrics.getNumNodeManagers());

        List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
        LOG.info("Got Cluster node info from ASM");
        for (NodeReport node : clusterNodeReports) {
            LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                    + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                    + node.getNumContainers());
        }

        QueueInfo queueInfo = yarnClient.getQueueInfo("default");
        LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
                + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
                + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
                + queueInfo.getChildQueues().size());

        List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
        for (QueueUserACLInfo aclInfo : listAclInfo) {
            for (QueueACL userAcl : aclInfo.getUserAcls()) {
                LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                        + userAcl.name());
            }
        }

        // Get a new application id
        YarnClientApplication app = yarnClient.createApplication();

        // Create a new ApplicationSubmissionContext
        //ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
        ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();

        id = appContext.getApplicationId();

        // set the application name
        appContext.setApplicationName(job.getJobName());

        // Create a new container launch context for the AM's container
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

        // Define the local resources required
        Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
        // Lets assume the jar we need for our ApplicationMaster is available in
        // HDFS at a certain known path to us and we want to make it available to
        // the ApplicationMaster in the launched container
        if (job.getJar() == null) {
            throw new IllegalArgumentException("Jar must be set in order to run the application!");
        }

        Path jarPath = new Path(job.getJar());
        jarPath = fs.makeQualified(jarPath);
        getConf().set("bsp.jar", jarPath.makeQualified(fs.getUri(), jarPath).toString());

        FileStatus jarStatus = fs.getFileStatus(jarPath);
        LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
        amJarRsrc.setType(LocalResourceType.FILE);
        amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
        amJarRsrc.setTimestamp(jarStatus.getModificationTime());
        amJarRsrc.setSize(jarStatus.getLen());

        // this creates a symlink in the working directory
        localResources.put(YARNBSPConstants.APP_MASTER_JAR_PATH, amJarRsrc);

        // add hama related jar files to localresources for container
        List<File> hamaJars;
        if (System.getProperty("hama.home.dir") != null)
            hamaJars = localJarfromPath(System.getProperty("hama.home.dir"));
        else
            hamaJars = localJarfromPath(getConf().get("hama.home.dir"));
        String hamaPath = getSystemDir() + "/hama";
        for (File fileEntry : hamaJars) {
            addToLocalResources(fs, fileEntry.getCanonicalPath(), hamaPath, fileEntry.getName(),
                    localResources);
        }

        // Set the local resources into the launch context
        amContainer.setLocalResources(localResources);

        // Set up the environment needed for the launch context
        Map<String, String> env = new HashMap<String, String>();
        // Assuming our classes or jars are available as local resources in the
        // working directory from which the command will be run, we need to append
        // "." to the path.
        // By default, all the hadoop specific classpaths will already be available
        // in $CLASSPATH, so we should be careful not to overwrite it.
        StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$())
                .append(File.pathSeparatorChar).append("./*");
        for (String c : yarnConf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
            classPathEnv.append(File.pathSeparatorChar);
            classPathEnv.append(c.trim());
        }

        env.put(YARNBSPConstants.HAMA_YARN_LOCATION, jarPath.toUri().toString());
        env.put(YARNBSPConstants.HAMA_YARN_SIZE, Long.toString(jarStatus.getLen()));
        env.put(YARNBSPConstants.HAMA_YARN_TIMESTAMP, Long.toString(jarStatus.getModificationTime()));

        env.put(YARNBSPConstants.HAMA_LOCATION, hamaPath);
        env.put("CLASSPATH", classPathEnv.toString());
        amContainer.setEnvironment(env);

        // Set the necessary command to execute on the allocated container
        Vector<CharSequence> vargs = new Vector<CharSequence>(5);
        vargs.add("${JAVA_HOME}/bin/java");
        vargs.add("-cp " + classPathEnv + "");
        vargs.add(ApplicationMaster.class.getCanonicalName());
        vargs.add(submitJobFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

        vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/hama-appmaster.stdout");
        vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/hama-appmaster.stderr");

        // Get final commmand
        StringBuilder command = new StringBuilder();
        for (CharSequence str : vargs) {
            command.append(str).append(" ");
        }

        List<String> commands = new ArrayList<String>();
        commands.add(command.toString());
        amContainer.setCommands(commands);

        LOG.debug("Start command: " + command);

        Resource capability = Records.newRecord(Resource.class);
        // we have at least 3 threads, which comsumes 1mb each, for each bsptask and
        // a base usage of 100mb
        capability.setMemory(3 * job.getNumBspTask() + getConf().getInt("hama.appmaster.memory.mb", 100));
        LOG.info("Set memory for the application master to " + capability.getMemory() + "mb!");

        // Set the container launch content into the ApplicationSubmissionContext
        appContext.setResource(capability);

        // Setup security tokens
        if (UserGroupInformation.isSecurityEnabled()) {
            // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
            Credentials credentials = new Credentials();
            String tokenRenewer = yarnConf.get(YarnConfiguration.RM_PRINCIPAL);
            if (tokenRenewer == null || tokenRenewer.length() == 0) {
                throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
            }

            // For now, only getting tokens for the default file-system.
            final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
            if (tokens != null) {
                for (Token<?> token : tokens) {
                    LOG.info("Got dt for " + fs.getUri() + "; " + token);
                }
            }
            DataOutputBuffer dob = new DataOutputBuffer();
            credentials.writeTokenStorageToStream(dob);
            ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
            amContainer.setTokens(fsTokens);
        }

        appContext.setAMContainerSpec(amContainer);

        // Create the request to send to the ApplicationsManager
        ApplicationId appId = appContext.getApplicationId();
        yarnClient.submitApplication(appContext);

        return monitorApplication(appId) ? new NetworkedJob() : null;
    } catch (YarnException e) {
        e.printStackTrace();
        return null;
    }
}

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName,
        Map<String, LocalResource> localResources) throws IOException {
    Path dstPath = new Path(fileDstPath, fileName);
    dstPath = fs.makeQualified(dstPath);
    fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath);
    FileStatus fileStatus = fs.getFileStatus(dstPath);
    LocalResource localRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dstPath.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(),
            fileStatus.getModificationTime());
    localResources.put(fileName, localRsrc);
}

From source file:org.apache.hcatalog.templeton.tool.HDFSCleanup.java

License:Apache License

/**
 * Loop through all the files, deleting any that are older than
 * maxage./*from w  ww . j  a  v a  2s. c om*/
 * 
 * @param fs
 * @throws IOException
 */
private void checkFiles(FileSystem fs) throws IOException {
    long now = new Date().getTime();
    for (Type type : Type.values()) {
        try {
            for (FileStatus status : fs.listStatus(new Path(HDFSStorage.getPath(type, storage_root)))) {
                if (now - status.getModificationTime() > maxage) {
                    LOG.info("Deleting " + status.getPath().toString());
                    fs.delete(status.getPath(), true);
                }
            }
        } catch (Exception e) {
            // Nothing to find for this type.
        }
    }
}

From source file:org.apache.helix.provisioning.yarn.AppLauncher.java

License:Apache License

private LocalResource setupLocalResource(FileSystem fs, Path dst) throws Exception {
    URI uri = dst.toUri();/*from   ww w.j a v  a 2  s  .  co m*/
    String extension = uri.getPath().substring(uri.getPath().lastIndexOf(".") + 1);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
    // Set the type of resource - file or archive
    // archives are untarred at destination
    // we don't need the jar file to be untarred for now
    if (isArchive(extension)) {
        amJarRsrc.setType(LocalResourceType.ARCHIVE);
    } else {
        amJarRsrc.setType(LocalResourceType.FILE);
    }
    // Set visibility of the resource
    // Setting to most private option
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    return amJarRsrc;
}

From source file:org.apache.helix.provisioning.yarn.YarnProvisioner.java

License:Apache License

private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container,
        Participant participant) throws Exception {

    ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class);

    // Map<String, String> envs = System.getenv();
    String appName = applicationMasterConfig.getAppName();
    int appId = applicationMasterConfig.getAppId();
    String serviceName = _resourceConfig.getId().stringify();
    String serviceClasspath = applicationMasterConfig.getClassPath(serviceName);
    String mainClass = applicationMasterConfig.getMainClass(serviceName);
    String zkAddress = applicationMasterConfig.getZKAddress();

    // set the localresources needed to launch container
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LocalResource servicePackageResource = Records.newRecord(LocalResource.class);
    YarnConfiguration conf = new YarnConfiguration();
    FileSystem fs;/*from   w  w  w  .j a  v a2s  .c om*/
    fs = FileSystem.get(conf);
    String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar";
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    FileStatus destStatus = fs.getFileStatus(dst);

    // Set the type of resource - file or archive
    // archives are untarred at destination
    // we don't need the jar file to be untarred for now
    servicePackageResource.setType(LocalResourceType.ARCHIVE);
    // Set visibility of the resource
    // Setting to most private option
    servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    servicePackageResource.setTimestamp(destStatus.getModificationTime());
    servicePackageResource.setSize(destStatus.getLen());
    LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName);
    localResources.put(serviceName, servicePackageResource);

    // Set local resource info into app master container launch context
    participantContainer.setLocalResources(localResources);

    // Set the necessary security tokens as needed
    // amContainer.setContainerTokens(containerToken);

    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    env.put(serviceName, dst.getName());
    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    classPathEnv.append(File.pathSeparatorChar);
    classPathEnv.append(serviceClasspath);
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
    LOG.info("Setting classpath for service:\n" + classPathEnv.toString());
    env.put("CLASSPATH", classPathEnv.toString());

    participantContainer.setEnvironment(env);

    if (applicationMaster.allTokens != null) {
        LOG.info("Setting tokens: " + applicationMaster.allTokens);
        participantContainer.setTokens(applicationMaster.allTokens);
    }

    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx" + 4096 + "m");
    // Set class name
    vargs.add(ParticipantLauncher.class.getCanonicalName());
    // Set params for container participant
    vargs.add("--zkAddress " + zkAddress);
    vargs.add("--cluster " + appName);
    vargs.add("--participantId " + participant.getId().stringify());
    vargs.add("--participantClass " + mainClass);

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up  container launch command " + command.toString() + " with arguments \n"
            + vargs);
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    participantContainer.setCommands(commands);
    return participantContainer;
}

From source file:org.apache.hive.common.util.MockFileSystem.java

License:Apache License

public void touch(MockFile file) {
    if (fileStatusMap.containsKey(file)) {
        FileStatus fileStatus = fileStatusMap.get(file);
        FileStatus fileStatusNew = new FileStatus(fileStatus.getLen(), fileStatus.isDirectory(),
                fileStatus.getReplication(), fileStatus.getBlockSize(), fileStatus.getModificationTime() + 1,
                fileStatus.getAccessTime(), fileStatus.getPermission(), fileStatus.getOwner(),
                fileStatus.getGroup(), fileStatus.getPath());
        fileStatusMap.put(file, fileStatusNew);
    }/*from   w  ww .  j av  a 2s .c o  m*/
}

From source file:org.apache.hoya.tools.CoreFileSystem.java

License:Apache License

/**
 * Create an AM resource from the//from w  w w.j  av  a  2  s .  co  m
 *
 * @param destPath     dest path in filesystem
 * @param resourceType resource type
 * @return the resource set up wih application-level visibility and the
 * timestamp & size set from the file stats.
 */
public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
    FileStatus destStatus = fileSystem.getFileStatus(destPath);
    LocalResource amResource = Records.newRecord(LocalResource.class);
    amResource.setType(resourceType);
    // Set visibility of the resource
    // Setting to most private option
    amResource.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    amResource.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    amResource.setTimestamp(destStatus.getModificationTime());
    amResource.setSize(destStatus.getLen());
    return amResource;
}

From source file:org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem.java

License:Apache License

/** {@inheritDoc} */
@Override//from   w  w  w. j av a 2s .c o  m
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                // By convention directory has blockSize == 0, while file has blockSize > 0:
                return isDirectory() ? 0 : (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };
    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

From source file:org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.java

License:Apache License

/**
 * Convert a file status obtained from the secondary file system to a status of the primary file system.
 *
 * @param status Secondary file system status.
 * @return Primary file system status.// w  w  w .  ja  v  a2 s  .  c om
 */
@SuppressWarnings("deprecation")
private FileStatus toPrimary(FileStatus status) {
    return status != null
            ? new FileStatus(status.getLen(), status.isDir(), status.getReplication(), status.getBlockSize(),
                    status.getModificationTime(), status.getAccessTime(), status.getPermission(),
                    status.getOwner(), status.getGroup(), toPrimary(status.getPath()))
            : null;
}