Example usage for org.apache.hadoop.fs FileStatus getModificationTime

List of usage examples for org.apache.hadoop.fs FileStatus getModificationTime

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime()

Source Link

Document

Get the modification time of the file.

Usage

From source file:org.kitesdk.data.spi.filesystem.SignalManager.java

License:Apache License

/**
 * Check the last time the specified constraints have been signaled as ready.
 *
 * @param viewConstraints The constraints to check for a signal.
 *
 * @return the timestamp of the last time the constraints were signaled as ready.
 *          if the constraints have never been signaled, -1 will be returned.
 *
 * @throws DatasetException if the signals could not be accessed.
 *//*w ww.  j a  v a 2  s.  c o  m*/
public long getReadyTimestamp(Constraints viewConstraints) {
    String normalizedConstraints = getNormalizedConstraints(viewConstraints);

    Path signalPath = new Path(signalDirectory, normalizedConstraints);
    // check if the signal exists
    try {
        try {
            FileStatus signalStatus = rootFileSystem.getFileStatus(signalPath);
            return signalStatus.getModificationTime();
        } catch (final FileNotFoundException ex) {
            // empty, will be thrown when the signal path doesn't exist
        }
        return -1;
    } catch (IOException e) {
        throw new DatasetIOException("Could not access signal path: " + signalPath, e);
    }
}

From source file:org.lobid.lodmill.hadoop.IntegrationTestCollectSubjects.java

License:Open Source License

private void writeZippedMapFile() throws IOException {
    long time = System.currentTimeMillis();
    final Path[] outputFiles = FileUtil.stat2Paths(
            getFileSystem().listStatus(new Path(HDFS_OUT), new Utils.OutputFileUtils.OutputFilesFilter()));
    final Path zipOutputLocation = new Path(HDFS_OUT + "/" + CollectSubjects.MAP_FILE_ZIP);
    CollectSubjects.asZippedMapFile(hdfs, outputFiles[0], zipOutputLocation);
    final FileStatus fileStatus = hdfs.getFileStatus(zipOutputLocation);
    assertTrue(fileStatus.getModificationTime() >= time);
}

From source file:org.moya.core.yarn.Client.java

License:Apache License

/**
 * Main run function for the client/*from   w ww  .j a  v  a 2s . c o  m*/
 * 
 * @return true if application completed successfully
 * @throws IOException
 * @throws YarnException
 */
public boolean run() throws IOException, YarnException {

    LOG.info("Running Client");
    yarnClient.start();

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());

    List<NodeReport> clusterNodeReports = yarnClient.getNodeReports();
    LOG.info("Got Cluster node info from ASM");
    for (NodeReport node : clusterNodeReports) {
        LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                + node.getNumContainers());
    }

    QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
            + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
            + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
            + queueInfo.getChildQueues().size());

    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                    + userAcl.name());
        }
    }

    // Get a new application id
    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    // TODO get min/max resource capabilities from RM and change memory ask
    // if needed
    // If we do not have min/max, we may not be able to correctly request
    // the required resources from the RM for the app master
    // Memory ask has to be a multiple of min and less than max.
    // Dump out information about cluster capability as seen by the resource
    // manager
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    // A resource ask cannot exceed the max.
    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    // set the application name
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setApplicationName(appName);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of
    // the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    FileSystem fs = FileSystem.get(conf);
    Path src = new Path(appMasterJar);
    String pathSuffix = appName + "/" + appId.getId() + "/AppMaster.jar";
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    fs.copyFromLocalFile(false, true, src, dst);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);

    // Set the type of resource - file or archive
    // archives are untarred at destination
    // we don't need the jar file to be untarred 
    amJarRsrc.setType(LocalResourceType.FILE);
    // Set visibility of the resource
    // Setting to most private option
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    localResources.put("AppMaster.jar", amJarRsrc);

    // Setup App Master Constants
    String amJarLocation = "";
    long amJarLen = 0;
    long amJarTimestamp = 0;

    // adding info so we can add the jar to the App master container path
    amJarLocation = dst.toUri().toString();
    FileStatus shellFileStatus = fs.getFileStatus(dst);
    amJarLen = shellFileStatus.getLen();
    amJarTimestamp = shellFileStatus.getModificationTime();

    // ADD libs needed that will be untared
    // Keep it all archived for now so add it as a file...
    src = new Path(localLibJar);
    pathSuffix = appName + "/" + appId.getId() + "/Runnable.jar";
    dst = new Path(fs.getHomeDirectory(), pathSuffix);
    fs.copyFromLocalFile(false, true, src, dst);
    destStatus = fs.getFileStatus(dst);
    LocalResource libsJarRsrc = Records.newRecord(LocalResource.class);
    libsJarRsrc.setType(LocalResourceType.FILE);
    libsJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    libsJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    libsJarRsrc.setTimestamp(destStatus.getModificationTime());
    localResources.put("Runnable.jar", libsJarRsrc);

    // Setup Libs Constants
    String libsLocation = "";
    long libsLen = 0;
    long libsTimestamp = 0;

    // adding info so we can add the jar to the App master container path
    libsLocation = dst.toUri().toString();
    FileStatus libsFileStatus = fs.getFileStatus(dst);
    libsLen = libsFileStatus.getLen();
    libsTimestamp = libsFileStatus.getModificationTime();

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        localResources.put("log4j.properties", log4jRsrc);
    }

    // Set local resource info into app master container launch context
    amContainer.setLocalResources(localResources);

    // Set the env variables to be setup in the env where the application
    // master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    // put the AM jar into env and MOYA Runnable
    // using the env info, the application master will create the correct
    // local resource for the
    // eventual containers that will be launched to execute the shell
    // scripts
    env.put(MConstants.APPLICATIONMASTERJARLOCATION, amJarLocation);
    env.put(MConstants.APPLICATIONMASTERJARTIMESTAMP, Long.toString(amJarTimestamp));
    env.put(MConstants.APPLICATIONMASTERJARLEN, Long.toString(amJarLen));

    env.put(MConstants.LIBSLOCATION, libsLocation);
    env.put(MConstants.LIBSTIMESTAMP, Long.toString(libsTimestamp));
    env.put(MConstants.LIBSLEN, Long.toString(libsLen));

    env.put(MConstants.ZOOKEEPERHOSTS, ZKHosts);

    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());

    amContainer.setEnvironment(env);

    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx" + amMemory + "m");
    // Set class name
    vargs.add(appMasterMainClass);
    // Set params for Application Master
    vargs.add("--container_memory " + String.valueOf(containerMemory));
    vargs.add("--num_containers " + String.valueOf(numContainers));
    vargs.add("--priority " + String.valueOf(moyaPriority));
    if (!localLibJar.isEmpty()) {
        vargs.add("--lib " + localLibJar + "");
    }
    if (debugFlag) {
        vargs.add("--debug");
    }

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    amContainer.setCommands(commands);

    // Set up resource type requirements
    // For now, only memory is supported so we set memory requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMemory);
    appContext.setResource(capability);

    // Service data is a binary blob that can be passed to the application
    // Not needed in this scenario
    // amContainer.setServiceData(serviceData);

    // The following are not required for launching an application master
    // amContainer.setContainerId(containerId);

    appContext.setAMContainerSpec(amContainer);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    // TODO - what is the range for priority? how to decide?
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Submit the application to the applications manager
    // SubmitApplicationResponse submitResp =
    // applicationsManager.submitApplication(appRequest);
    // Ignore the response as either a valid response object is returned on
    // success
    // or an exception thrown to denote some form of a failure
    LOG.info("Submitting application to ASM");

    yarnClient.submitApplication(appContext);

    // TODO
    // Try submitting the same request again
    // app submission failure?

    // Monitor the application
    return monitorApplication(appId);

}

From source file:org.mrgeo.cmd.mrsimageinfo.MrsImageInfo.java

License:Apache License

private void printFileInfo(final Path pfile, PrintStream out) throws IOException {
    // TODO: The following is HDFS-sepcific; needs to be re-factored
    final FileSystem fs = pfile.getFileSystem(config);
    final FileStatus stat = fs.getFileStatus(pfile);

    out.print("    date: " + DateTimeFormat.shortDateTime().print(stat.getModificationTime()));
    out.println("  size: " + human(stat.getLen()));

    final FsPermission p = stat.getPermission();

    if (debug) {/* w  ww.jav a2s  .c  o m*/
        out.print("    ");
        out.print(stat.isDir() ? "d" : "f");
        out.print(" u: " + stat.getOwner() + " (" + p.getUserAction().toString().toLowerCase() + ")");
        out.print(" g: " + stat.getGroup() + " (" + p.getGroupAction().toString().toLowerCase() + ")");
        out.print(" o: " + "(" + p.getOtherAction().toString().toLowerCase() + ")");

        out.print(" blk: " + human(stat.getBlockSize()));
        out.println(" repl: " + stat.getReplication());
    }
}

From source file:org.mrgeo.utils.DependencyLoader.java

License:Apache License

private static void addFileToClasspath(Configuration conf, Set<String> existing, FileSystem fs, Path hdfsBase,
        File file) throws IOException {
    Path hdfsPath = new Path(hdfsBase, file.getName());
    if (!existing.contains(hdfsPath.toString())) {
        if (fs.exists(hdfsPath)) {
            // check the timestamp and exit if the one in hdfs is "newer"
            FileStatus status = fs.getFileStatus(hdfsPath);

            if (file.lastModified() <= status.getModificationTime()) {
                log.debug(file.getPath() + " up to date");
                DistributedCache.addFileToClassPath(hdfsPath, conf, fs);

                existing.add(hdfsPath.toString());
                return;
            }/*from  w ww  .  j  a v a  2 s .  c o m*/
        }

        // copy the file...
        log.debug("Copying " + file.getPath() + " to HDFS for distribution");

        fs.copyFromLocalFile(new Path(file.getCanonicalFile().toURI()), hdfsPath);
        DistributedCache.addFileToClassPath(hdfsPath, conf, fs);
        existing.add(hdfsPath.toString());
    }
}

From source file:org.notmysock.tez.BroadcastTest.java

License:Apache License

private LocalResource createLocalResource(FileSystem fs, Path file) throws IOException {

    final LocalResourceType type = LocalResourceType.FILE;
    final LocalResourceVisibility visibility = LocalResourceVisibility.APPLICATION;

    FileStatus fstat = fs.getFileStatus(file);

    org.apache.hadoop.yarn.api.records.URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
    long resourceSize = fstat.getLen();
    long resourceModificationTime = fstat.getModificationTime();

    LocalResource lr = Records.newRecord(LocalResource.class);
    lr.setResource(resourceURL);//from  w  w  w . j  a  v  a 2s.  co  m
    lr.setType(type);
    lr.setSize(resourceSize);
    lr.setVisibility(visibility);
    lr.setTimestamp(resourceModificationTime);

    return lr;
}

From source file:org.opencloudengine.flamingo.mapreduce.util.SortableFileStatus.java

License:Apache License

@Override
public int compareTo(SortableFileStatus other) {
    FileStatus otherFileStatus = other.getFileStatus();
    if (fileStatus.getModificationTime() > otherFileStatus.getModificationTime()) {
        return 1;
    } else if (fileStatus.getModificationTime() < otherFileStatus.getModificationTime()) {
        return -1;
    } else {/*w w  w  .  j  ava  2 s . c o  m*/
        return 0;
    }
}

From source file:org.opencloudengine.garuda.model.HdfsFileInfo.java

License:Open Source License

public HdfsFileInfo(FileStatus fileStatus, ContentSummary contentSummary) {
    this.fullyQualifiedPath = fileStatus.getPath().toUri().getPath();
    this.filename = isEmpty(getFilename(fullyQualifiedPath)) ? getDirectoryName(fullyQualifiedPath)
            : getFilename(fullyQualifiedPath);
    this.length = fileStatus.getLen();
    this.path = getPath(fullyQualifiedPath);
    this.directory = fileStatus.isDirectory();
    this.file = !fileStatus.isDirectory();
    this.owner = fileStatus.getOwner();
    this.group = fileStatus.getGroup();
    this.blockSize = fileStatus.getBlockSize();
    this.replication = fileStatus.getReplication();
    this.modificationTime = fileStatus.getModificationTime();
    if (contentSummary != null) {
        this.spaceConsumed = contentSummary.getSpaceConsumed();
        this.quota = contentSummary.getQuota();
        this.spaceQuota = contentSummary.getSpaceQuota();
        this.directoryCount = contentSummary.getDirectoryCount();
        this.fileCount = contentSummary.getFileCount();
    }//from w  w  w  . j  ava  2  s  . com
    this.accessTime = fileStatus.getAccessTime();
    this.permission = fileStatus.getPermission().toString();
}

From source file:org.openflamingo.fs.hdfs.HdfsFileInfo.java

License:Apache License

/**
 *  ??.//from ww  w .  j a v  a  2s  .  com
 *
 * @param fileStatus HDFS File Status
 */
public HdfsFileInfo(FileStatus fileStatus) {
    this.fullyQualifiedPath = fileStatus.getPath().toUri().getPath();
    this.filename = StringUtils.isEmpty(FileUtils.getFilename(fullyQualifiedPath))
            ? FileUtils.getDirectoryName(fullyQualifiedPath)
            : FileUtils.getFilename(fullyQualifiedPath);
    this.length = fileStatus.getLen();
    this.path = FileUtils.getPath(fullyQualifiedPath);
    this.directory = fileStatus.isDir();
    this.file = !fileStatus.isDir();
    this.owner = fileStatus.getOwner();
    this.group = fileStatus.getGroup();
    this.blockSize = fileStatus.getBlockSize();
    this.replication = fileStatus.getReplication();
    this.modificationTime = fileStatus.getModificationTime();
    this.accessTime = fileStatus.getAccessTime();
    this.setText(this.filename);
    this.setLeaf(file ? true : false);
    this.setCls(directory ? "folder" : "file");
    this.setId(fullyQualifiedPath);
    this.permission = fileStatus.getPermission().toString();
}

From source file:org.smartfrog.services.hadoop.operations.dfs.DfsListDirImpl.java

License:Open Source License

/**
 * do the work/*from   w ww . j  av  a  2  s  . co m*/
 *
 * @param fileSystem the filesystem; this is closed afterwards
 * @param conf       the configuration driving this operation
 * @throws Exception on any failure
 */
@Override
protected void performDfsOperation(FileSystem fileSystem, ManagedConfiguration conf) throws Exception {
    Path path = getPath();
    if (path == null) {
        throw new SmartFrogLivenessException("No path for the DfsListDir operation", this);
    }
    int minFileCount = sfResolve(ATTR_MIN_FILE_COUNT, 0, true);
    int maxFileCount = sfResolve(ATTR_MAX_FILE_COUNT, 0, true);
    long minTotalFileSize = sfResolve(ATTR_MIN_TOTAL_FILE_SIZE, 0L, true);
    long maxTotalFileSize = sfResolve(ATTR_MAX_TOTAL_FILE_SIZE, 0L, true);
    try {
        long size = 0;
        FileStatus[] stats = fileSystem.listStatus(path);
        if (stats == null) {
            throw new SmartFrogLivenessException("Path not found in the remote filesystem: " + path, this);
        }
        StringBuilder builder = new StringBuilder();
        builder.append("Listing of ").append(path).append("/\n");
        for (FileStatus file : stats) {
            size += file.getLen();
            builder.append(file.getPath().getName());
            builder.append("\n  size=").append(file.getLen());
            builder.append("\n  replication=").append(file.getReplication());
            builder.append("\n  last modified=").append(new Date(file.getModificationTime()).toString());
            builder.append("\n  owner=").append(file.getOwner());
            builder.append("\n  group=").append(file.getGroup());
            builder.append("\n  permissions=").append(file.getPermission()).append('\n');
        }
        String listing = builder.toString();
        sfLog().info(listing);
        int count = stats.length;
        sfLog().info("Files: " + count + "  total size=" + size);
        if (count < minFileCount) {
            throw new SmartFrogLivenessException("File count " + count + " is below the minFileCount value of "
                    + minFileCount + "\n" + listing, this);
        }
        if (maxFileCount > -1 && count > maxFileCount) {
            throw new SmartFrogLivenessException("File count " + count + " is above the maxFileCount value of "
                    + minFileCount + "\n" + listing, this);
        }
        if (size < minTotalFileSize) {
            throw new SmartFrogLivenessException("File size " + size
                    + " is below the minTotalFileSize value of " + minTotalFileSize + "\n" + listing, this);
        }
        if (maxFileCount > -1 && size > maxFileCount) {
            throw new SmartFrogLivenessException("File size " + size
                    + " is above the maxTotalFileSize value of " + maxTotalFileSize + "\n" + listing, this);
        }

    } catch (IOException e) {
        if (isIdempotent()) {
            sfLog().info("Failed to stat " + path, e);
        } else {
            throw e;
        }
    }
}