Example usage for org.apache.hadoop.yarn.api.records LocalResource getType

List of usage examples for org.apache.hadoop.yarn.api.records LocalResource getType

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResource
  6. getType

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource getType.

Prototype

@Public
@Stable
public abstract LocalResourceType getType();

Source Link

Document

Get the LocalResourceType of the resource to be localized.

Usage

From source file:com.continuuity.weave.yarn.YarnWeavePreparer.java

License:Open Source License

private void saveLocalFiles(Map<String, LocalResource> localResources, Set<String> keys) throws IOException {
    Map<String, LocalFile> localFiles = Maps.transformEntries(
            Maps.filterKeys(localResources, Predicates.in(keys)),
            new Maps.EntryTransformer<String, LocalResource, LocalFile>() {
                @Override//from  ww w .  ja  v  a  2 s .  c  o  m
                public LocalFile transformEntry(String key, LocalResource value) {
                    try {
                        return new DefaultLocalFile(key,
                                ConverterUtils.getPathFromYarnURL(value.getResource()).toUri(),
                                value.getTimestamp(), value.getSize(),
                                value.getType() != LocalResourceType.FILE, value.getPattern());
                    } catch (URISyntaxException e) {
                        throw Throwables.propagate(e);
                    }
                }
            });

    LOG.debug("Create and copy localFiles.json");
    Location location = createTempLocation("localFiles", ".json");
    Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
    try {
        new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec()).create()
                .toJson(localFiles.values(), new TypeToken<List<LocalFile>>() {
                }.getType(), writer);
    } finally {
        writer.close();
    }
    LOG.debug("Done localFiles.json");
    localResources.put("localFiles.json", YarnUtils.createLocalResource(location));
}

From source file:com.scaleoutsoftware.soss.hserver.hadoop.DistributedCacheManager.java

License:Apache License

/**
 * Set up the distributed cache by localizing the resources, and updating
 * the configuration with references to the localized resources.
 * @param conf job configuration//ww  w .ja v  a  2s .c  om
 * @throws IOException
 */
public void setup(Configuration conf) throws IOException {
    //If we are not 0th worker, wait for 0th worker to set up the cache
    if (InvocationWorker.getIgWorkerIndex() > 0 && InvocationWorker.getNumberOfWorkers() > 1) {
        try {
            InvocationWorker.getSynchronizationBarrier().waitForComplete(ACTION_NAME, SYNCHRONIZATION_WAIT_MS,
                    WAIT_GRANULARITY_MS);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return;
    }

    File workDir = new File(System.getProperty("user.dir"));

    // Generate YARN local resources objects corresponding to the distributed
    // cache configuration
    Map<String, LocalResource> localResources = new LinkedHashMap<String, LocalResource>();
    MRApps.setupDistributedCache(conf, localResources);

    //CODE CHANGE FROM ORIGINAL FILE:
    //We need to clear the resources from jar files, since they are distributed through the IG.
    //
    Iterator<Map.Entry<String, LocalResource>> iterator = localResources.entrySet().iterator();
    while (iterator.hasNext()) {
        Entry<String, LocalResource> entry = iterator.next();
        if (entry.getKey().endsWith(".jar")) {
            iterator.remove();
        }
    }

    // Generating unique numbers for FSDownload.

    AtomicLong uniqueNumberGenerator = new AtomicLong(System.currentTimeMillis());

    // Find which resources are to be put on the local classpath
    Map<String, Path> classpaths = new HashMap<String, Path>();
    Path[] archiveClassPaths = DistributedCache.getArchiveClassPaths(conf);
    if (archiveClassPaths != null) {
        for (Path p : archiveClassPaths) {
            FileSystem remoteFS = p.getFileSystem(conf);
            p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
            classpaths.put(p.toUri().getPath().toString(), p);
        }
    }

    Path[] fileClassPaths = DistributedCache.getFileClassPaths(conf);
    if (fileClassPaths != null) {
        for (Path p : fileClassPaths) {
            FileSystem remoteFS = p.getFileSystem(conf);
            p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
            classpaths.put(p.toUri().getPath().toString(), p);
        }
    }

    // Localize the resources
    LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LOCAL_DIR);
    FileContext localFSFileContext = FileContext.getLocalFSFileContext();
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

    ExecutorService exec = null;
    try {
        ThreadFactory tf = new ThreadFactoryBuilder()
                .setNameFormat("LocalDistributedCacheManager Downloader #%d").build();
        exec = Executors.newCachedThreadPool(tf);
        Path destPath = localDirAllocator.getLocalPathForWrite(".", conf);
        Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap();
        for (LocalResource resource : localResources.values()) {
            Callable<Path> download = new FSDownload(localFSFileContext, ugi, conf,
                    new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet())), resource);
            Future<Path> future = exec.submit(download);
            resourcesToPaths.put(resource, future);
        }
        for (Entry<String, LocalResource> entry : localResources.entrySet()) {
            LocalResource resource = entry.getValue();
            Path path;
            try {
                path = resourcesToPaths.get(resource).get();
            } catch (InterruptedException e) {
                throw new IOException(e);
            } catch (ExecutionException e) {
                throw new IOException(e);
            }
            String pathString = path.toUri().toString();
            String link = entry.getKey();
            String target = new File(path.toUri()).getPath();
            symlink(workDir, target, link);

            if (resource.getType() == LocalResourceType.ARCHIVE) {
                localArchives.add(pathString);
            } else if (resource.getType() == LocalResourceType.FILE) {
                localFiles.add(pathString);
            } else if (resource.getType() == LocalResourceType.PATTERN) {
                //PATTERN is not currently used in local mode
                throw new IllegalArgumentException(
                        "Resource type PATTERN is not " + "implemented yet. " + resource.getResource());
            }
            Path resourcePath;
            try {
                resourcePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
            } catch (URISyntaxException e) {
                throw new IOException(e);
            }
            LOG.info(String.format("Localized %s as %s", resourcePath, path));
            String cp = resourcePath.toUri().getPath();
            if (classpaths.keySet().contains(cp)) {
                localClasspaths.add(path.toUri().getPath().toString());
            }
        }
    } finally {
        if (exec != null) {
            exec.shutdown();
        }
    }
    // Update the configuration object with localized data.
    if (!localArchives.isEmpty()) {
        conf.set(MRJobConfig.CACHE_LOCALARCHIVES,
                StringUtils.arrayToString(localArchives.toArray(new String[localArchives.size()])));
    }
    if (!localFiles.isEmpty()) {
        conf.set(MRJobConfig.CACHE_LOCALFILES,
                StringUtils.arrayToString(localFiles.toArray(new String[localArchives.size()])));
    }
    setupCalled = true;

    //If we are  0th worker, signal action complete
    if (InvocationWorker.getIgWorkerIndex() == 0 && InvocationWorker.getNumberOfWorkers() > 1) {
        try {
            InvocationWorker.getSynchronizationBarrier().signalComplete(ACTION_NAME);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

}

From source file:org.apache.drill.yarn.core.LaunchSpec.java

License:Apache License

public void dump(PrintStream out) {
    if (command != null) {
        out.print("Command: ");
        out.println(command);/*from w ww . j  a  v  a 2  s. co m*/
    }
    if (mainClass != null) {
        out.print("Main Class: ");
        out.println(mainClass);
        out.println("VM Args:");
        if (vmArgs.isEmpty()) {
            out.println("  None");
        } else {
            for (String item : vmArgs) {
                out.print("  ");
                out.println(item);
            }
        }
        out.println("Class Path:");
        if (classPath.isEmpty()) {
            out.println("  None");
        } else {
            for (String item : classPath) {
                out.print("  ");
                out.println(item);
            }
        }
    }
    out.println("Program Args:");
    if (cmdArgs.isEmpty()) {
        out.println("  None");
    } else {
        for (String item : cmdArgs) {
            out.print("  ");
            out.println(item);
        }
    }
    out.println("Environment:");
    if (env.isEmpty()) {
        out.println("  None");
    } else {
        for (String key : env.keySet()) {
            out.print("  ");
            out.print(key);
            out.print("=");
            out.println(env.get(key));
        }
    }
    out.println("Resources: ");
    if (resources.isEmpty()) {
        out.println("  None");
    } else {
        for (String key : resources.keySet()) {
            out.print("  Key: ");
            out.println(key);
            LocalResource resource = resources.get(key);
            out.print("   URL: ");
            out.println(resource.getResource().toString());
            out.print("   Size: ");
            out.println(resource.getSize());
            out.print("   Timestamp: ");
            out.println(DoYUtil.toIsoTime(resource.getTimestamp()));
            out.print("   Type: ");
            out.println(resource.getType().toString());
            out.print("   Visiblity: ");
            out.println(resource.getVisibility().toString());
        }
    }
}

From source file:org.apache.tez.client.TezClient.java

License:Apache License

private DAGClient submitDAGSession(DAG dag) throws TezException, IOException {
    Preconditions.checkState(isSession == true,
            "submitDAG with additional resources applies to only session mode. "
                    + "In non-session mode please specify all resources in the initial configuration");

    verifySessionStateForSubmission();/*from w w w.  j  a v  a 2s  .c  o  m*/

    String dagId = null;
    LOG.info("Submitting dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId
            + ", dagName=" + dag.getName());

    if (!additionalLocalResources.isEmpty()) {
        for (LocalResource lr : additionalLocalResources.values()) {
            Preconditions.checkArgument(lr.getType() == LocalResourceType.FILE, "LocalResourceType: "
                    + lr.getType() + " is not supported, only " + LocalResourceType.FILE + " is supported");
        }
    }

    Map<String, String> aclConfigs = null;
    if (historyACLPolicyManager != null) {
        aclConfigs = historyACLPolicyManager.setupSessionDAGACLs(amConfig.getTezConfiguration(), sessionAppId,
                dag.getName(), dag.getDagAccessControls());
    }

    Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
    DAGPlan dagPlan = TezClientUtils.prepareAndCreateDAGPlan(dag, amConfig, tezJarResources,
            usingTezArchiveDeploy, sessionCredentials, aclConfigs);

    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder();
    requestBuilder.setDAGPlan(dagPlan).build();
    if (!additionalLocalResources.isEmpty()) {
        requestBuilder.setAdditionalAmResources(
                DagTypeConverters.convertFromLocalResources(additionalLocalResources));
    }

    additionalLocalResources.clear();

    DAGClientAMProtocolBlockingPB proxy = null;
    try {
        proxy = waitForProxy();
    } catch (InterruptedException e) {
        throw new IOException("Interrupted while trying to create a connection to the AM", e);
    }
    if (proxy == null) {
        try {
            LOG.warn("DAG submission to session timed out, stopping session");
            stop();
        } catch (Throwable t) {
            LOG.info("Got an exception when trying to stop session", t);
        }
        throw new DAGSubmissionTimedOut(
                "Could not submit DAG to Tez Session" + ", timed out after " + clientTimeout + " seconds");
    }

    try {
        SubmitDAGResponseProto response = proxy.submitDAG(null, requestBuilder.build());
        // the following check is only for testing since the final class
        // SubmitDAGResponseProto cannot be mocked
        if (response != null) {
            dagId = response.getDagId();
        }
    } catch (ServiceException e) {
        throw new TezException(e);
    }
    LOG.info("Submitted dag to TezSession" + ", sessionName=" + clientName + ", applicationId=" + sessionAppId
            + ", dagName=" + dag.getName());
    return new DAGClientImpl(sessionAppId, dagId, amConfig.getTezConfiguration(), frameworkClient);
}

From source file:org.apache.tez.dag.api.client.rpc.TestDAGClientAMProtocolBlockingPBServerImpl.java

License:Apache License

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
    int maxIPCMsgSize = 1024;
    String dagPlanName = "dagplan-name";
    File requestFile = tmpFolder.newFile("request-file");
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);

    byte[] randomBytes = new byte[2 * maxIPCMsgSize];
    (new Random()).nextBytes(randomBytes);
    UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
    Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
    DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);

    String lrName = "localResource";
    String scheme = "file";
    String host = "localhost";
    int port = 80;
    String path = "/test";
    URL lrURL = URL.newInstance(scheme, host, port, path);
    LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE,
            LocalResourceVisibility.PUBLIC, 1, 1);
    Map<String, LocalResource> localResources = new HashMap<>();
    localResources.put(lrName, localResource);

    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan)
            .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
    try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
        requestBuilder.build().writeTo(fileOutputStream);
    }/*from ww w .  j av a 2s .  c  om*/

    DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
    ACLManager aclManager = mock(ACLManager.class);
    DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(
            new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf)));
    when(dagClientHandler.getACLManager()).thenReturn(aclManager);
    when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id");
    when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);

    requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
    serverImpl.submitDAG(null, requestBuilder.build());

    ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
    verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
    dagPlan = dagPlanCaptor.getValue();
    localResources = localResourcesCaptor.getValue();

    assertEquals(dagPlan.getName(), dagPlanName);
    assertEquals(dagPlan.getVertexCount(), 1);
    assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
    assertArrayEquals(randomBytes,
            dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertEquals(localResources.size(), 1);
    assertTrue(localResources.containsKey(lrName));
    localResource = localResources.get(lrName);
    assertEquals(localResource.getType(), LocalResourceType.FILE);
    assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
    lrURL = localResource.getResource();
    assertEquals(lrURL.getScheme(), scheme);
    assertEquals(lrURL.getHost(), host);
    assertEquals(lrURL.getPort(), port);
    assertEquals(lrURL.getFile(), path);
}

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) {
    List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size());
    for (Entry<String, LocalResource> entry : lrs.entrySet()) {
        PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
        String key = entry.getKey();
        LocalResource lr = entry.getValue();
        localResourcesBuilder.setName(key);
        localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
        localResourcesBuilder.setSize(lr.getSize());
        localResourcesBuilder.setTimeStamp(lr.getTimestamp());
        localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
        localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
        if (lr.getType() == LocalResourceType.PATTERN) {
            if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
                throw new TezUncheckedException(
                        "LocalResource type set to pattern" + " but pattern is null or empty");
            }/*w w w.j  a va 2s  .c  o  m*/
            localResourcesBuilder.setPattern(lr.getPattern());
        }
        planLrs.add(localResourcesBuilder.build());
    }
    return planLrs;
}

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static PlanLocalResource convertLocalResourceToPlanLocalResource(String name, LocalResource lr) {
    PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
    localResourcesBuilder.setName(name);
    localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
    localResourcesBuilder.setSize(lr.getSize());
    localResourcesBuilder.setTimeStamp(lr.getTimestamp());
    localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
    localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
    if (lr.getType() == LocalResourceType.PATTERN) {
        if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
            throw new TezUncheckedException(
                    "LocalResource type set to pattern" + " but pattern is null or empty");
        }/*from   w  w  w.jav a2  s. c  om*/
        localResourcesBuilder.setPattern(lr.getPattern());
    }
    return localResourcesBuilder.build();
}

From source file:org.apache.tez.dag.app.ContainerContext.java

License:Apache License

private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
        Map<String, LocalResource> reqLRs) {
    Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
    for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
        LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
        if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Cannot match container: Attempting to use same target resource name: "
                        + srcLREntry.getKey() + ", but with different source resources. Already localized: "
                        + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
            }// w  w w.j a va2  s  .com
            return false;
        }
    }
    for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
        LocalResource lr = additionalLREntry.getValue();
        if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
            return false;
        }
    }
    return true;
}

From source file:org.apache.tez.dag.app.launcher.TezLocalCacheManager.java

License:Apache License

/**
 * Localize this instance's resources by downloading and symlinking them.
 *
 * @throws IOException when an error occurs in download or link
 *///ww w  .  j  a v a 2s . com
public void localize() throws IOException {
    String absPath = Paths.get(".").toAbsolutePath().normalize().toString();
    Path cwd = fileContext.makeQualified(new Path(absPath));
    ExecutorService threadPool = null;

    try {
        // construct new threads with helpful names
        ThreadFactory threadFactory = new ThreadFactoryBuilder()
                .setNameFormat("TezLocalCacheManager Downloader #%d").build();
        threadPool = Executors.newCachedThreadPool(threadFactory);

        // start all fetches
        for (Map.Entry<String, LocalResource> entry : resources.entrySet()) {
            String resourceName = entry.getKey();
            LocalResource resource = entry.getValue();

            if (resource.getType() == LocalResourceType.PATTERN) {
                throw new IllegalArgumentException("Resource type PATTERN not supported.");
            }

            // linkPath is the path we want to symlink the file/directory into
            Path linkPath = new Path(cwd, entry.getKey());

            if (resourceInfo.containsKey(resource)) {
                // We've already downloaded this resource and just need to add another link.
                resourceInfo.get(resource).linkPaths.add(linkPath);
            } else {
                // submit task to download the object
                java.nio.file.Path downloadDir = Files.createTempDirectory(tempDir, resourceName);
                Path dest = new Path(downloadDir.toAbsolutePath().toString());
                FSDownload downloader = new FSDownload(fileContext, ugi, conf, dest, resource);
                Future<Path> downloadedPath = threadPool.submit(downloader);
                resourceInfo.put(resource, new ResourceInfo(downloadedPath, linkPath));
            }
        }

        // Link each file
        for (Map.Entry<LocalResource, ResourceInfo> entry : resourceInfo.entrySet()) {
            LocalResource resource = entry.getKey();
            ResourceInfo resourceMeta = entry.getValue();

            for (Path linkPath : resourceMeta.linkPaths) {
                Path targetPath;

                try {
                    // this blocks on the download completing
                    targetPath = resourceMeta.downloadPath.get();
                } catch (InterruptedException | ExecutionException e) {
                    throw new IOException(e);
                }

                if (createSymlink(targetPath, linkPath)) {
                    LOG.info("Localized file: {} as {}", resource, linkPath);
                } else {
                    LOG.warn("Failed to create symlink: {} <- {}", targetPath, linkPath);
                }
            }
        }
    } finally {
        if (threadPool != null) {
            threadPool.shutdownNow();
        }
    }
}