Example usage for org.apache.hadoop.yarn.api.records LocalResource getTimestamp

List of usage examples for org.apache.hadoop.yarn.api.records LocalResource getTimestamp

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResource
  6. getTimestamp

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource getTimestamp.

Prototype

@Public
@Stable
public abstract long getTimestamp();

Source Link

Document

Get the original timestamp of the resource to be localized, used for verification.

Usage

From source file:com.continuuity.weave.yarn.YarnWeavePreparer.java

License:Open Source License

private void saveLocalFiles(Map<String, LocalResource> localResources, Set<String> keys) throws IOException {
    Map<String, LocalFile> localFiles = Maps.transformEntries(
            Maps.filterKeys(localResources, Predicates.in(keys)),
            new Maps.EntryTransformer<String, LocalResource, LocalFile>() {
                @Override// ww w  .ja  v a  2s  .co  m
                public LocalFile transformEntry(String key, LocalResource value) {
                    try {
                        return new DefaultLocalFile(key,
                                ConverterUtils.getPathFromYarnURL(value.getResource()).toUri(),
                                value.getTimestamp(), value.getSize(),
                                value.getType() != LocalResourceType.FILE, value.getPattern());
                    } catch (URISyntaxException e) {
                        throw Throwables.propagate(e);
                    }
                }
            });

    LOG.debug("Create and copy localFiles.json");
    Location location = createTempLocation("localFiles", ".json");
    Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
    try {
        new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec()).create()
                .toJson(localFiles.values(), new TypeToken<List<LocalFile>>() {
                }.getType(), writer);
    } finally {
        writer.close();
    }
    LOG.debug("Done localFiles.json");
    localResources.put("localFiles.json", YarnUtils.createLocalResource(location));
}

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Upload the Asterix cluster description on to the DFS. This will persist the state of the instance.
 * //  ww w . j  av  a  2  s. c  o  m
 * @return
 * @throws YarnException
 * @throws IOException
 */
private List<DFSResourceCoordinate> deployConfig() throws YarnException, IOException {

    FileSystem fs = FileSystem.get(conf);
    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);

    String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME;
    Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix);
    FileStatus destStatus;
    try {
        destStatus = fs.getFileStatus(dstConf);
    } catch (IOException e) {
        throw new YarnException("Asterix instance by that name does not appear to exist in DFS");
    }
    LocalResource asterixConfLoc = Records.newRecord(LocalResource.class);
    asterixConfLoc.setType(LocalResourceType.FILE);
    asterixConfLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixConfLoc.setResource(ConverterUtils.getYarnUrlFromPath(dstConf));
    asterixConfLoc.setTimestamp(destStatus.getModificationTime());

    DFSResourceCoordinate conf = new DFSResourceCoordinate();
    conf.envs.put(dstConf.toUri().toString(), AConstants.CONFLOCATION);
    conf.envs.put(Long.toString(asterixConfLoc.getSize()), AConstants.CONFLEN);
    conf.envs.put(Long.toString(asterixConfLoc.getTimestamp()), AConstants.CONFTIMESTAMP);
    conf.name = CONFIG_DEFAULT_NAME;
    conf.res = asterixConfLoc;
    resources.add(conf);

    return resources;

}

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Uploads binary resources to HDFS for use by the AM
 * @return//www  .  j a  v a2  s.c o  m
 * @throws IOException
 * @throws YarnException
 */
public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException {

    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    FileSystem fs = FileSystem.get(conf);
    Path src, dst;
    FileStatus destStatus;
    String pathSuffix;

    // adding info so we can add the jar to the App master container path

    // Add the asterix tarfile to HDFS for easy distribution
    // Keep it all archived for now so add it as a file...

    pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip";
    dst = new Path(fs.getHomeDirectory(), pathSuffix);
    if (refresh) {
        if (fs.exists(dst)) {
            fs.delete(dst, false);
        }
    }
    if (!fs.exists(dst)) {
        src = new Path(asterixZip);
        LOG.info("Copying Asterix distributable to DFS");
        fs.copyFromLocalFile(false, true, src, dst);
    }
    destStatus = fs.getFileStatus(dst);
    LocalResource asterixTarLoc = Records.newRecord(LocalResource.class);
    asterixTarLoc.setType(LocalResourceType.ARCHIVE);
    asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    asterixTarLoc.setTimestamp(destStatus.getModificationTime());

    // adding info so we can add the tarball to the App master container path
    DFSResourceCoordinate tar = new DFSResourceCoordinate();
    tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION);
    tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN);
    tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP);
    tar.res = asterixTarLoc;
    tar.name = "asterix-server.zip";
    resources.add(tar);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        DFSResourceCoordinate l4j = new DFSResourceCoordinate();
        tar.res = log4jRsrc;
        tar.name = "log4j.properties";
        resources.add(l4j);
    }

    resources.addAll(installAmLibs());
    return resources;
}

From source file:org.apache.drill.yarn.core.LaunchSpec.java

License:Apache License

public void dump(PrintStream out) {
    if (command != null) {
        out.print("Command: ");
        out.println(command);/*from   w  w  w.  j  a  va  2 s .  c  o m*/
    }
    if (mainClass != null) {
        out.print("Main Class: ");
        out.println(mainClass);
        out.println("VM Args:");
        if (vmArgs.isEmpty()) {
            out.println("  None");
        } else {
            for (String item : vmArgs) {
                out.print("  ");
                out.println(item);
            }
        }
        out.println("Class Path:");
        if (classPath.isEmpty()) {
            out.println("  None");
        } else {
            for (String item : classPath) {
                out.print("  ");
                out.println(item);
            }
        }
    }
    out.println("Program Args:");
    if (cmdArgs.isEmpty()) {
        out.println("  None");
    } else {
        for (String item : cmdArgs) {
            out.print("  ");
            out.println(item);
        }
    }
    out.println("Environment:");
    if (env.isEmpty()) {
        out.println("  None");
    } else {
        for (String key : env.keySet()) {
            out.print("  ");
            out.print(key);
            out.print("=");
            out.println(env.get(key));
        }
    }
    out.println("Resources: ");
    if (resources.isEmpty()) {
        out.println("  None");
    } else {
        for (String key : resources.keySet()) {
            out.print("  Key: ");
            out.println(key);
            LocalResource resource = resources.get(key);
            out.print("   URL: ");
            out.println(resource.getResource().toString());
            out.print("   Size: ");
            out.println(resource.getSize());
            out.print("   Timestamp: ");
            out.println(DoYUtil.toIsoTime(resource.getTimestamp()));
            out.print("   Type: ");
            out.println(resource.getType().toString());
            out.print("   Visiblity: ");
            out.println(resource.getVisibility().toString());
        }
    }
}

From source file:org.apache.reef.bridge.client.JobResourceUploader.java

License:Apache License

/**
 * This class is invoked from Org.Apache.REEF.Client.Yarn.LegacyJobResourceUploader in .NET code.
 * Arguments:/*from   w  ww . j a  v  a  2s. c o  m*/
 * [0] : Local path for already generated archive
 * [1] : Path of job submission directory
 * [2] : File path for output with details of uploaded resource
 */
public static void main(final String[] args) throws InjectionException, IOException {
    Validate.isTrue(args.length == 3, "Job resource uploader requires 3 args");
    final File localFile = new File(args[0]);
    Validate.isTrue(localFile.exists(), "Local archive does not exist " + localFile.getAbsolutePath());
    final String jobSubmissionDirectory = args[1];
    final String localOutputPath = args[2];

    LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory "
            + jobSubmissionDirectory + " LocalOutputPath " + localOutputPath);
    final Configuration configuration = Configurations.merge(
            Tang.Factory.getTang().newConfigurationBuilder()
                    .bindNamedParameter(JobSubmissionDirectoryPrefix.class, jobSubmissionDirectory).build(),
            YarnClientConfiguration.CONF.build());

    final JobUploader jobUploader = Tang.Factory.getTang().newInjector(configuration)
            .getInstance(JobUploader.class);
    final LocalResource localResource = jobUploader.createJobFolder(jobSubmissionDirectory)
            .uploadAsLocalResource(localFile);

    // Output: <UploadedPath>;<LastModificationUnixTimestamp>;<ResourceSize>
    final URL resource = localResource.getResource();
    final String outputString = String.format("%s://%s:%d%s;%d;%d", resource.getScheme(), resource.getHost(),
            resource.getPort(), resource.getFile(), localResource.getTimestamp(), localResource.getSize());
    LOG.log(Level.INFO, "Writing output: " + outputString);
    try (Writer writer = new BufferedWriter(
            new OutputStreamWriter(new FileOutputStream(localOutputPath), "utf-8"))) {
        writer.write(outputString);
    }

    LOG.log(Level.FINER, "Done writing output file");
}

From source file:org.apache.tez.common.TezConverterUtils.java

License:Apache License

@Private
public static TezLocalResource convertYarnLocalResourceToTez(LocalResource lr) throws URISyntaxException {
    return new TezLocalResource(getURIFromYarnURL(lr.getResource()), lr.getSize(), lr.getTimestamp());
}

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) {
    List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size());
    for (Entry<String, LocalResource> entry : lrs.entrySet()) {
        PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
        String key = entry.getKey();
        LocalResource lr = entry.getValue();
        localResourcesBuilder.setName(key);
        localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
        localResourcesBuilder.setSize(lr.getSize());
        localResourcesBuilder.setTimeStamp(lr.getTimestamp());
        localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
        localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
        if (lr.getType() == LocalResourceType.PATTERN) {
            if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
                throw new TezUncheckedException(
                        "LocalResource type set to pattern" + " but pattern is null or empty");
            }/*  ww w  .  j ava 2s.  c  o m*/
            localResourcesBuilder.setPattern(lr.getPattern());
        }
        planLrs.add(localResourcesBuilder.build());
    }
    return planLrs;
}

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static PlanLocalResource convertLocalResourceToPlanLocalResource(String name, LocalResource lr) {
    PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
    localResourcesBuilder.setName(name);
    localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
    localResourcesBuilder.setSize(lr.getSize());
    localResourcesBuilder.setTimeStamp(lr.getTimestamp());
    localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
    localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
    if (lr.getType() == LocalResourceType.PATTERN) {
        if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
            throw new TezUncheckedException(
                    "LocalResource type set to pattern" + " but pattern is null or empty");
        }//  w ww .  j  a v a2  s . c o  m
        localResourcesBuilder.setPattern(lr.getPattern());
    }
    return localResourcesBuilder.build();
}