Example usage for org.apache.hadoop.io DataOutputBuffer getLength

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataOutputBuffer getLength.

Prototype

public int getLength() 

Source Link

Document

Returns the length of the valid data currently in the buffer.

Usage

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

/**
 * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 * specified by packagePath.//from  www  . j  a v  a 2  s  . co  m
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws IOException {
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    packageResource.setResource(packageUrl);
    log.debug("Set package resource in YarnContainerRunner for {}", packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens to start the container
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);

    // now remove the AM->RM token so that containers cannot access it
    Iterator iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
        TokenIdentifier token = ((org.apache.hadoop.security.token.Token) iter.next()).decodeIdentifier();
        if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    Map<String, LocalResource> localResourceMap = new HashMap<>();
    localResourceMap.put("__package", packageResource);

    // include the resources from the universal resource configurations
    LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config),
            yarnConfiguration);
    localResourceMap.putAll(resourceMapper.getResourceMap());

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(localResourceMap);

    if (UserGroupInformation.isSecurityEnabled()) {
        Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls();
        if (!acls.isEmpty()) {
            context.setApplicationACLs(acls);
        }
    }

    log.debug("Setting localResourceMap to {}", localResourceMap);
    log.debug("Setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);

    log.info(
            "Making an async start request for Container ID: {} on host: {} with local resource map: {} and context: {}",
            container.getId(), container.getNodeHttpAddress(), localResourceMap.toString(), context);
    nmClientAsync.startContainerAsync(container, context);
}

---

From source file:org.apache.samza.job.yarn.YarnContainerRunner.java

License:Apache License

/**
 *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 *    specified by packagePath.//from ww w .j av a  2s  . c o m
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws SamzaContainerLaunchException {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // TODO: SAMZA-1144 remove the customized approach for package resource and use the common one.
    // But keep it now for backward compatibility.
    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaContainerLaunchException(
                "IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    log.info("set package Resource in YarnContainerRunner for {}", packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        log.error("IOException when writing credentials.", ioe);
        throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer");
    }

    Map<String, LocalResource> localResourceMap = new HashMap<>();
    localResourceMap.put("__package", packageResource);

    // include the resources from the universal resource configurations
    LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config),
            yarnConfiguration);
    localResourceMap.putAll(resourceMapper.getResourceMap());

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(localResourceMap);

    log.debug("setting localResourceMap to {}", localResourceMap);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaContainerLaunchException(
                "Received YarnException when starting container: " + container.getId(), ye);
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaContainerLaunchException(
                "Received IOException when starting container: " + container.getId(), ioe);
    }
}

---

From source file:org.apache.slider.core.launch.AbstractLauncher.java

License:Apache License

/**
 * Complete the launch context (copy in env vars, etc).
 * @return the container to launch//from   w w  w.j av  a2  s  .c o  m
 */
public ContainerLaunchContext completeContainerLaunch() throws IOException {

    String cmdStr = SliderUtils.join(commands, " ", false);
    log.debug("Completed setting up container command {}", cmdStr);
    containerLaunchContext.setCommands(commands);

    //env variables
    if (log.isDebugEnabled()) {
        log.debug("Environment variables");
        for (Map.Entry<String, String> envPair : envVars.entrySet()) {
            log.debug("    \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
        }
    }
    containerLaunchContext.setEnvironment(env);

    //service data
    if (log.isDebugEnabled()) {
        log.debug("Service Data size");
        for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
            log.debug("\"{}\"=> {} bytes of data", entry.getKey(), entry.getValue().array().length);
        }
    }
    containerLaunchContext.setServiceData(serviceData);

    // resources
    dumpLocalResources();
    containerLaunchContext.setLocalResources(localResources);

    //tokens
    log.debug("{} tokens", credentials.numberOfTokens());
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer tokenBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    containerLaunchContext.setTokens(tokenBuffer);

    return containerLaunchContext;
}

---

From source file:org.apache.slider.core.launch.CredentialUtils.java

License:Apache License

/**
 * Save credentials to a byte buffer. Returns null if there were no
 * credentials to save//from   w  w  w  .ja v a 2s .  c om
 * @param credentials credential set
 * @return a byte buffer of serialized tokens
 * @throws IOException if the credentials could not be written to the stream
 */
public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException {
    ByteBuffer buffer = null;
    if (!credentials.getAllTokens().isEmpty()) {
        DataOutputBuffer dob = new DataOutputBuffer();
        try {
            credentials.writeTokenStorageToStream(dob);
        } finally {
            dob.close();
        }
        buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    }
    return buffer;
}

---

From source file:org.apache.slider.server.appmaster.SliderAppMaster.java

License:Apache License

private ByteBuffer getContainerCredentials() throws IOException {
    // a delegation token can be retrieved from filesystem since
    // the login is via a keytab (see above)
    Credentials credentials = new Credentials(containerCredentials);
    ByteBuffer tokens = null;//  w w w .j a  va2s. c o m
    Token<? extends TokenIdentifier>[] hdfsTokens = getClusterFS().getFileSystem()
            .addDelegationTokens(UserGroupInformation.getLoginUser().getShortUserName(), credentials);
    if (hdfsTokens.length > 0) {
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        dob.close();
        tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    }

    return tokens;
}

---

From source file:org.apache.sqoop.mapreduce.mainframe.TestMainframeDatasetInputSplit.java

License:Apache License

@Test
public void testWriteRead() {
    mfDatasetInputSplit.addDataset("dataSet1");
    mfDatasetInputSplit.addDataset("dataSet2");
    DataOutputBuffer dob = new DataOutputBuffer();
    DataInputBuffer dib = new DataInputBuffer();
    MainframeDatasetInputSplit mfReader = new MainframeDatasetInputSplit();
    try {//from w  w w .  j a  va 2 s. co m
        mfDatasetInputSplit.write(dob);
        dib.reset(dob.getData(), dob.getLength());
        mfReader.readFields(dib);
        Assert.assertNotNull("MFReader get data from tester", mfReader);
        Assert.assertEquals(2, mfReader.getLength());
        Assert.assertEquals("dataSet1", mfReader.getNextDataset());
        Assert.assertEquals("dataSet2", mfReader.getNextDataset());
    } catch (IOException ioe) {
        Assert.fail("No IOException should be thrown!");
    } catch (InterruptedException ie) {
        Assert.fail("No InterruptedException should be thrown!");
    }
}

---

From source file:org.apache.tajo.pullserver.PullServerAuxService.java

License:Apache License

/**
 * Serialize the shuffle port into a ByteBuffer for use later on.
 * @param port the port to be sent to the ApplciationMaster
 * @return the serialized form of the port.
 *//*  w  ww.  j  av a  2  s .c  om*/
public static ByteBuffer serializeMetaData(int port) throws IOException {
    //TODO these bytes should be versioned
    DataOutputBuffer port_dob = new DataOutputBuffer();
    port_dob.writeInt(port);
    return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());
}

---

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupSecurityTokens(ContainerLaunchContext amContainer, FileSystem fs) throws IOException {
    if (UserGroupInformation.isSecurityEnabled()) {
        Credentials credentials = new Credentials();
        String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }/*from   w ww. j av  a  2s .  co m*/

        // For now, only getting tokens for the default file-system.
        final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        amContainer.setTokens(fsTokens);
    }
}

---

From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java

License:Apache License

@Override
public ContainerLaunchContext getLaunchContext(Container container) throws IOException {
    // create a container launch context
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
    UserGroupInformation user = UserGroupInformation.getCurrentUser();
    try {/*from  ww  w  . java 2 s  .  c om*/
        Credentials credentials = user.getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        launchContext.setTokens(securityTokens);
    } catch (IOException e) {
        LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    FileSystem fs = FileSystem.get(appContext.getConfiguration());

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    String suffix = "Tajo" + "/" + appContext.getApplicationId().getId();
    Path parentPath = new Path(fs.getHomeDirectory(), suffix);

    // tar ball
    Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH));
    FileStatus archiveFs = fs.getFileStatus(archivePath);
    LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()),
            LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(),
            archiveFs.getModificationTime());
    localResources.put("tajo", archiveRsrc);

    Configuration tajoWorkerConf = new Configuration(false);
    tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml"));
    tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001");
    tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005");
    Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf");
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoWorkerConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);
    launchContext.setLocalResources(localResources);

    // Set the environment
    setupEnv(launchContext);

    // Set the necessary command to execute on the allocated container
    Vector<CharSequence> vargs = new Vector<CharSequence>(5);

    // Set executable command
    // Set args for the shell command if any
    vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo");
    vargs.add("--config");
    vargs.add("${" + Constants.TAJO_CONF_DIR + "}");
    vargs.add("worker");
    // Add log redirect params
    // Add log redirect params
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    launchContext.setCommands(commands);
    return launchContext;
}

---

From source file:org.apache.tez.auxservices.ShuffleHandler.java

License:Apache License

/**
 * A helper function to serialize the JobTokenIdentifier to be sent to the
 * ShuffleHandler as ServiceData.//from w w w  .  j a v  a  2 s.c o  m
 * @param jobToken the job token to be used for authentication of
 * shuffle data requests.
 * @return the serialized version of the jobToken.
 */
public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException {
    //TODO these bytes should be versioned
    DataOutputBuffer jobToken_dob = new DataOutputBuffer();
    jobToken.write(jobToken_dob);
    return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• »