Example usage for org.apache.hadoop.fs FileStatus getModificationTime

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getModificationTime.

Prototype

public long getModificationTime() 

Source Link

Document

Get the modification time of the file.

Usage

From source file:org.apache.spark.simr.Simr.java

License:Apache License

public boolean isMaster() throws IOException {
    String electionDirName = conf.get("simr_tmp_dir") + "/" + ELECTIONDIR;

    try {//from w ww  . ja va  2s . co m
        fs.mkdirs(new Path(electionDirName)); // create election directory
    } catch (Exception ex) {
    }

    String myTaskId = context.getTaskAttemptID().getTaskID().toString();

    Path myIpFile = new Path(electionDirName + "/" + myTaskId);
    FSDataOutputStream outf = fs.create(myIpFile, true);
    outf.close();

    // look for file with smallest timestamp
    long firstMapperTime = Long.MAX_VALUE;
    String firstMapperId = "";
    for (FileStatus fstat : fs.listStatus(new Path(electionDirName + "/"))) {
        long modTime = fstat.getModificationTime();
        if (modTime < firstMapperTime) {
            firstMapperTime = modTime;
            firstMapperId = fstat.getPath().getName();
        }
    }
    return myTaskId.equals(firstMapperId);
}

---

From source file:org.apache.sysml.yarn.DMLYarnClient.java

License:Apache License

private Map<String, LocalResource> constructLocalResourceMap(YarnConfiguration yconf) throws IOException {
    Map<String, LocalResource> rMap = new HashMap<>();
    Path path = new Path(_hdfsJarFile);

    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus jarStat = IOUtilFunctions.getFileSystem(path, yconf).getFileStatus(path);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(path));
    resource.setSize(jarStat.getLen());/*from  w w  w  .  j  a  va 2s .  c  o m*/
    resource.setTimestamp(jarStat.getModificationTime());
    resource.setType(LocalResourceType.FILE);
    resource.setVisibility(LocalResourceVisibility.PUBLIC);

    rMap.put(DML_JAR_NAME, resource);
    return rMap;
}

---

From source file:org.apache.tajo.master.TaskRunnerLauncherImpl.java

License:Apache License

private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
        throws IOException {
    LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
    FileStatus rsrcStat = fs.getFileStatus(p);
    rsrc.setResource(// w w w. j  av  a  2s  .c  o  m
            ConverterUtils.getYarnUrlFromPath(fs.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
    rsrc.setSize(rsrcStat.getLen());
    rsrc.setTimestamp(rsrcStat.getModificationTime());
    rsrc.setType(type);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    return rsrc;
}

---

From source file:org.apache.tajo.master.YarnContainerProxy.java

License:Apache License

private static LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
        throws IOException {
    LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
    FileStatus rsrcStat = fs.getFileStatus(p);
    rsrc.setResource(/*from   w ww.j a v a2  s  .co m*/
            ConverterUtils.getYarnUrlFromPath(fs.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
    rsrc.setSize(rsrcStat.getLen());
    rsrc.setTimestamp(rsrcStat.getModificationTime());
    rsrc.setType(type);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    return rsrc;
}

---

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupLocalResources(ContainerLaunchContext amContainer, FileSystem fs, ApplicationId appId)
        throws IOException {
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path

    String appMasterJar = findContainingJar(ApplicationMaster.class);
    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources,
            LocalResourceType.FILE);//ww  w.j a  v  a 2 s. c om

    addToLocalResources(fs, libDir, libDir, appId.getId(), localResources, LocalResourceType.FILE);

    addToLocalResources(fs, tajoArchive, "tajo", appId.getId(), localResources, LocalResourceType.ARCHIVE);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources,
                LocalResourceType.FILE);
    }

    //    addToLocalResources(fs, confDir, "conf", appId.getId(),
    //        localResources, LocalResourceType.FILE);

    // Tajo master conf
    Configuration tajoMasterConf = new Configuration(false);
    tajoMasterConf.addResource(new Path(confDir, "tajo-site.xml"));
    String suffix = appName + "/" + appId.getId() + "/master-conf";
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoMasterConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);

    amContainer.setLocalResources(localResources);
}

---

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

/**
 * @return Destinate n/*from  ww w.  j av  a 2s .  c  o m*/
 */
private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, int appId,
        Map<String, LocalResource> localResources, LocalResourceType type) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileSrcPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), type,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return dst;
}

---

From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java

License:Apache License

@Override
public ContainerLaunchContext getLaunchContext(Container container) throws IOException {
    // create a container launch context
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
    UserGroupInformation user = UserGroupInformation.getCurrentUser();
    try {/* w  w  w  . j  a  v a  2s  .  c o  m*/
        Credentials credentials = user.getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        launchContext.setTokens(securityTokens);
    } catch (IOException e) {
        LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    FileSystem fs = FileSystem.get(appContext.getConfiguration());

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    String suffix = "Tajo" + "/" + appContext.getApplicationId().getId();
    Path parentPath = new Path(fs.getHomeDirectory(), suffix);

    // tar ball
    Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH));
    FileStatus archiveFs = fs.getFileStatus(archivePath);
    LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()),
            LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(),
            archiveFs.getModificationTime());
    localResources.put("tajo", archiveRsrc);

    Configuration tajoWorkerConf = new Configuration(false);
    tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml"));
    tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001");
    tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005");
    Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf");
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoWorkerConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);
    launchContext.setLocalResources(localResources);

    // Set the environment
    setupEnv(launchContext);

    // Set the necessary command to execute on the allocated container
    Vector<CharSequence> vargs = new Vector<CharSequence>(5);

    // Set executable command
    // Set args for the shell command if any
    vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo");
    vargs.add("--config");
    vargs.add("${" + Constants.TAJO_CONF_DIR + "}");
    vargs.add("worker");
    // Add log redirect params
    // Add log redirect params
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    launchContext.setCommands(commands);
    return launchContext;
}

---

From source file:org.apache.tez.benchmark.SessionTest.java

License:Apache License

protected LocalResource createLocalResource(FileSystem fs, Path file) throws IOException {
    final LocalResourceType type = LocalResourceType.FILE;
    final LocalResourceVisibility visibility = LocalResourceVisibility.APPLICATION;
    FileStatus fstat = fs.getFileStatus(file);
    org.apache.hadoop.yarn.api.records.URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
    long resourceSize = fstat.getLen();
    long resourceModificationTime = fstat.getModificationTime();
    LocalResource lr = Records.newRecord(LocalResource.class);
    lr.setResource(resourceURL);//from   w ww  .  ja v a 2 s .  c o  m
    lr.setType(type);
    lr.setSize(resourceSize);
    lr.setVisibility(visibility);
    lr.setTimestamp(resourceModificationTime);
    return lr;
}

---

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

/**
 * Setup LocalResource map for Tez jars based on provided Configuration
 * /*from  w w  w  .j  a v  a  2 s  .c o m*/
 * @param conf
 *          Configuration to use to access Tez jars' locations
 * @param credentials
 *          a credentials instance into which tokens for the Tez local
 *          resources will be populated
 * @param tezJarResources Map of LocalResources to use for AM and DAGs
 * @return Whether the archive-based deployment of Tez was used.
 * @throws IOException
 */
static boolean setupTezJarsLocalResources(TezConfiguration conf, Credentials credentials,
        Map<String, LocalResource> tezJarResources) throws IOException {
    Preconditions.checkNotNull(credentials, "A non-null credentials object should be specified");
    boolean usingTezArchive = false;

    if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) {
        LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since  '"
                + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true");
    } else {
        // Add tez jars to local resource
        String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS);

        if (tezJarUris == null || tezJarUris.length == 0) {
            throw new TezUncheckedException("Invalid configuration of tez jars" + ", "
                    + TezConfiguration.TEZ_LIB_URIS + " is not defined in the configuration");
        }

        LOG.info("Using tez.lib.uris value from configuration: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

        if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz")
                || tezJarUris[0].endsWith(".zip") || tezJarUris[0].endsWith(".tar"))) {
            String fileName = tezJarUris[0];

            FileStatus fStatus = getLRFileStatus(fileName, conf)[0];
            LocalResourceVisibility lrVisibility;
            if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE)
                    && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
                lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
                lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME,
                    LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                            LocalResourceType.ARCHIVE, lrVisibility, fStatus.getLen(),
                            fStatus.getModificationTime()));
            Path[] tezJarPaths = { fStatus.getPath() };
            // obtain credentials
            TokenCache.obtainTokensForFileSystems(credentials, tezJarPaths, conf);
            usingTezArchive = true;
        } else { // Treat as non-archives
            addLocalResources(conf, tezJarUris, tezJarResources, credentials);
        }

        if (tezJarResources.isEmpty()) {
            throw new TezUncheckedException("No files found in locations specified in "
                    + TezConfiguration.TEZ_LIB_URIS + " . Locations: " + StringUtils.join(tezJarUris, ','));
        }
    }

    // Add aux uris to local resources
    addLocalResources(conf, conf.getStrings(TezConfiguration.TEZ_AUX_URIS), tezJarResources, credentials);

    return usingTezArchive;
}

---

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

private static void addLocalResources(Configuration conf, String[] configUris,
        Map<String, LocalResource> tezJarResources, Credentials credentials) throws IOException {
    if (configUris == null || configUris.length == 0) {
        return;//from   ww  w . j  a  v  a  2  s  . c  om
    }
    List<Path> configuredPaths = Lists.newArrayListWithCapacity(configUris.length);
    for (String configUri : configUris) {
        boolean ancestorsHavePermission = checkAncestorPermissionsForAllUsers(conf, configUri,
                FsAction.EXECUTE);
        FileStatus[] fileStatuses = getLRFileStatus(configUri, conf);
        for (FileStatus fStatus : fileStatuses) {
            if (fStatus.isDirectory()) {
                // Skip directories - no recursive search support.
                continue;
            }
            LocalResourceVisibility lrVisibility;
            if (ancestorsHavePermission && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
                lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
                lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            String rsrcName = fStatus.getPath().getName();
            if (tezJarResources.containsKey(rsrcName)) {
                String message = "Duplicate resource found" + ", resourceName=" + rsrcName + ", existingPath="
                        + tezJarResources.get(rsrcName).getResource().toString() + ", newPath="
                        + fStatus.getPath();
                LOG.warn(message);
            }
            tezJarResources.put(rsrcName,
                    LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                            LocalResourceType.FILE, lrVisibility, fStatus.getLen(),
                            fStatus.getModificationTime()));
            configuredPaths.add(fStatus.getPath());
        }
    }
    // Obtain credentials.
    if (!configuredPaths.isEmpty()) {
        TokenCache.obtainTokensForFileSystems(credentials,
                configuredPaths.toArray(new Path[configuredPaths.size()]), conf);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »