Example usage for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer ContainerLocalizer APPCACHE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer ContainerLocalizer APPCACHE.

Prototype

String APPCACHE

To view the source code for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer ContainerLocalizer APPCACHE.

Click Source Link

Usage

From source file:com.mellanox.hadoop.mapred.UdaPluginSH.java

License:Apache License

static IndexRecordBridge getPathIndex(String jobIDStr, String mapId, int reduce) {
    String user = userRsrc.get(jobIDStr);

    ///////////////////////
    JobID jobID = JobID.forName(jobIDStr);
    ApplicationId appID = ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()), jobID.getId());
    final String base = ContainerLocalizer.USERCACHE + "/" + user + "/" + ContainerLocalizer.APPCACHE + "/"
            + ConverterUtils.toString(appID) + "/output" + "/" + mapId;
    if (LOG.isDebugEnabled()) {
        LOG.debug("DEBUG0 " + base);
    }/*from ww  w  .  j  a v a  2s . c om*/
    // Index file
    IndexRecordBridge data = null;
    try {
        Path indexFileName = lDirAlloc.getLocalPathToRead(base + "/file.out.index", mjobConf);
        // Map-output file
        Path mapOutputFileName = lDirAlloc.getLocalPathToRead(base + "/file.out", mjobConf);
        if (LOG.isDebugEnabled()) {
            LOG.debug("DEBUG1 " + base + " : " + mapOutputFileName + " : " + indexFileName);
        }

        ///////////////////////
        // TODO: is this correct ?? - why user and not runAsUserName like in hadoop-1 ?? 
        // on 2nd thought, this sounds correct, because probably we registered the runAsUser and not the "user"
        data = indexCache.getIndexInformationBridge(mapId, reduce, indexFileName, user);
        data.pathMOF = mapOutputFileName.toString();
    } catch (IOException e) {
        LOG.error("got an exception while retrieving the Index Info");
    }

    return data;

}

---

From source file:org.apache.tajo.pullserver.HttpDataServerHandler.java

License:Apache License

@Override
public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception {

    if (request.getMethod() != HttpMethod.GET) {
        sendError(ctx, HttpResponseStatus.METHOD_NOT_ALLOWED);
        return;/* w  w w .j av  a 2 s  . c  om*/
    }

    String base = ContainerLocalizer.USERCACHE + "/" + userName + "/" + ContainerLocalizer.APPCACHE + "/"
            + appId + "/output" + "/";

    final Map<String, List<String>> params = new QueryStringDecoder(request.getUri()).parameters();

    List<FileChunk> chunks = Lists.newArrayList();
    List<String> taskIds = splitMaps(params.get("ta"));
    int sid = Integer.valueOf(params.get("sid").get(0));
    int partitionId = Integer.valueOf(params.get("p").get(0));
    for (String ta : taskIds) {

        File file = new File(base + "/" + sid + "/" + ta + "/output/" + partitionId);
        FileChunk chunk = new FileChunk(file, 0, file.length());
        chunks.add(chunk);
    }

    FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);

    // Write the content.
    if (file == null) {
        HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NO_CONTENT);
        if (!HttpHeaders.isKeepAlive(request)) {
            ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
        } else {
            response.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
            ctx.writeAndFlush(response);
        }
    } else {
        HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        ChannelFuture writeFuture = null;
        long totalSize = 0;
        for (FileChunk chunk : file) {
            totalSize += chunk.length();
        }
        HttpHeaders.setContentLength(response, totalSize);

        if (HttpHeaders.isKeepAlive(request)) {
            response.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        }
        // Write the initial line and the header.
        writeFuture = ctx.write(response);

        for (FileChunk chunk : file) {
            writeFuture = sendFile(ctx, chunk);
            if (writeFuture == null) {
                sendError(ctx, HttpResponseStatus.NOT_FOUND);
                return;
            }
        }
        if (ctx.pipeline().get(SslHandler.class) == null) {
            writeFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
        } else {
            ctx.flush();
        }

        // Decide whether to close the connection or not.
        if (!HttpHeaders.isKeepAlive(request)) {
            // Close the connection when the whole content is written out.
            writeFuture.addListener(ChannelFutureListener.CLOSE);
        }
    }

}

---

From source file:oz.hadoop.yarn.test.cluster.InJvmContainerExecutor.java

License:Apache License

/**
 * Most of this code is copied from the super class's launchContainer method (unfortunately), since directory
 * and other preparation logic is tightly coupled with the actual container launch.
 * Would be nice if it was broken apart where launch method would be invoked when
 * everything is prepared//  ww w  .  j av  a  2  s. c om
 */
private void prepareContainerDirectories(Container container, Path nmPrivateContainerScriptPath,
        Path nmPrivateTokensPath, String userName, String appId, Path containerWorkDir, List<String> localDirs,
        List<String> logDirs) {

    FsPermission dirPerm = new FsPermission(APPDIR_PERM);
    ContainerId containerId = container.getContainerId();
    String containerIdStr = ConverterUtils.toString(containerId);
    String appIdStr = ConverterUtils.toString(containerId.getApplicationAttemptId().getApplicationId());

    try {
        for (String sLocalDir : localDirs) {
            Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
            Path userdir = new Path(usersdir, userName);
            Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE);
            Path appDir = new Path(appCacheDir, appIdStr);
            Path containerDir = new Path(appDir, containerIdStr);
            createDir(containerDir, dirPerm, true);
        }

        // Create the container log-dirs on all disks
        this.createLogDirs(appIdStr, containerIdStr, logDirs);

        Path tmpDir = new Path(containerWorkDir, YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
        createDir(tmpDir, dirPerm, false);

        // copy launch script to work dir
        Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
        fc.util().copy(nmPrivateContainerScriptPath, launchDst);

        // copy container tokens to work dir
        Path tokenDst = new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
        fc.util().copy(nmPrivateTokensPath, tokenDst);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to prepare container directories for container " + container,
                e);
    }
}

---