Example usage for org.apache.hadoop.security Credentials addToken

List of usage examples for org.apache.hadoop.security Credentials addToken

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. Credentials
  6. addToken

Introduction

In this page you can find the example usage for org.apache.hadoop.security Credentials addToken.

Prototype

public void addToken(Text alias, Token<? extends TokenIdentifier> t)

Source Link

Document

Add a token in the storage (in memory).

Usage

From source file:com.cloudera.recordservice.hcatalog.common.TestHCatRSUtil.java

License:Apache License

@Test
public void copyCredentialsToJobConfTest() {
    JobConf conf = new JobConf();
    Credentials cred = new Credentials();
    cred.addToken(new Text("Alias"), new Token<TokenIdentifier>());
    HCatRSUtil.copyCredentialsToJobConf(cred, conf);
    assertEquals(1, conf.getCredentials().numberOfTokens());
}

From source file:com.cloudera.recordservice.hcatalog.mapreduce.HCatRSInputFormat.java

License:Apache License

/**
 * Initializes the input with a provided filter.
 * See {@link #setInput(Configuration, String, String, String)}
 *///from   w w w . ja v a2 s .  c o  m
public static HCatRSInputFormat setInput(Job job, String location, String filter) throws IOException {
    Configuration conf = job.getConfiguration();
    String kerberosPrincipal = conf.get(ConfVars.KERBEROS_PRINCIPAL_CONF.name);
    Pair<String, String> dbTablePair = HCatUtil.getDbAndTableName(location);
    dbTablePair = HCatRSUtil.cleanQueryPair(dbTablePair);
    String dbName = dbTablePair.first;
    String tableName = dbTablePair.second;
    if (location.toLowerCase().startsWith("select")) {
        RecordServiceConfig.setInputQuery(conf, location);
    } else {
        RecordServiceConfig.setInputTable(conf, dbName, tableName);
    }
    Credentials credentials = job.getCredentials();
    RecordServicePlannerClient.Builder builder = PlanUtil.getBuilder(conf);
    List<NetworkAddress> plannerHosts = PlanUtil.getPlannerHostPorts(conf);
    RecordServicePlannerClient planner = PlanUtil.getPlanner(conf, builder, plannerHosts, kerberosPrincipal,
            credentials);
    try {
        if (planner.isKerberosAuthenticated()) {
            Token<DelegationTokenIdentifier> delegationToken = TokenUtils
                    .fromTDelegationToken(planner.getDelegationToken(""));
            credentials.addToken(DelegationTokenIdentifier.DELEGATION_KIND, delegationToken);
        }
    } catch (RecordServiceException e) {
        throw new IOException(e);
    } finally {
        if (planner != null)
            planner.close();
    }
    job.setInputFormatClass(HCatRSInputFormat.class);
    return setInput(conf, dbName, tableName, filter);
}

From source file:com.cloudera.recordservice.mr.PlanUtil.java

License:Apache License

/**
 * This also handles authentication using credentials. If there is a delegation
 * token in the credentials, that will be used to authenticate the planner
 * connection. Otherwise, if kerberos is enabled, a token will be generated
 * and added to the credentials./*from w w w .  ja v  a 2 s .  com*/
 * TODO: is this behavior sufficient? Do we need to fall back and renew tokens
 * or does the higher level framework (i.e. oozie) do that?
 */
public static SplitsInfo getSplits(Configuration jobConf, Credentials credentials) throws IOException {
    Request request = PlanUtil.getRequest(jobConf);
    RecordServicePlannerClient.Builder builder = getBuilder(jobConf);
    List<NetworkAddress> plannerHostPorts = getPlannerHostPorts(jobConf);
    String kerberosPrincipal = jobConf.get(ConfVars.KERBEROS_PRINCIPAL_CONF.name);
    PlanRequestResult result = null;
    RecordServicePlannerClient planner = PlanUtil.getPlanner(jobConf, builder, plannerHostPorts,
            kerberosPrincipal, credentials);

    try {
        result = planner.planRequest(request);
        if (planner.isKerberosAuthenticated()) {
            // We need to get a delegation token and populate credentials (for the map tasks)
            // TODO: what to set as renewer?
            Token<DelegationTokenIdentifier> delegationToken = TokenUtils
                    .fromTDelegationToken(planner.getDelegationToken(""));
            credentials.addToken(DelegationTokenIdentifier.DELEGATION_KIND, delegationToken);
        }
    } catch (RecordServiceException e) {
        throw new IOException(e);
    } finally {
        if (planner != null)
            planner.close();
    }

    Schema schema = new Schema(result.schema);
    List<InputSplit> splits = new ArrayList<InputSplit>();
    for (Task t : result.tasks) {
        splits.add(new RecordServiceInputSplit(schema, new TaskInfo(t, result.hosts)));
    }
    LOG.debug(String.format("Generated %d splits.", splits.size()));

    // Randomize the order of the splits to mitigate skew.
    Collections.shuffle(splits);
    return new SplitsInfo(splits, schema);
}

From source file:com.continuuity.weave.internal.yarn.Hadoop20YarnAppClient.java

License:Apache License

private void addRMToken(ContainerLaunchContext context) {
    if (!UserGroupInformation.isSecurityEnabled()) {
        return;/*w  w w  . j  av a  2  s  .  c o  m*/
    }

    try {
        Credentials credentials = YarnUtils.decodeCredentials(context.getContainerTokens());

        Configuration config = yarnClient.getConfig();
        Token<TokenIdentifier> token = convertToken(
                yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))),
                YarnUtils.getRMAddress(config));

        LOG.info("Added RM delegation token {}", token);
        credentials.addToken(token.getService(), token);

        context.setContainerTokens(YarnUtils.encodeCredentials(credentials));

    } catch (Exception e) {
        LOG.error("Fails to create credentials.", e);
        throw Throwables.propagate(e);
    }
}

From source file:com.continuuity.weave.internal.yarn.Hadoop21YarnAppClient.java

License:Apache License

private void addRMToken(ContainerLaunchContext context) {
    if (!UserGroupInformation.isSecurityEnabled()) {
        return;//w  ww.j a  v a 2s.  c om
    }

    try {
        Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());

        Configuration config = yarnClient.getConfig();
        Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(
                yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))),
                YarnUtils.getRMAddress(config));

        LOG.info("Added RM delegation token {}", token);
        credentials.addToken(token.getService(), token);

        context.setTokens(YarnUtils.encodeCredentials(credentials));

    } catch (Exception e) {
        LOG.error("Fails to create credentials.", e);
        throw Throwables.propagate(e);
    }
}

From source file:com.datatorrent.stram.LaunchContainerRunnable.java

License:Apache License

public static ByteBuffer getTokens(UserGroupInformation ugi,
        Token<StramDelegationTokenIdentifier> delegationToken) {
    try {//w ww.j  a v a 2 s .  c o m
        Collection<Token<? extends TokenIdentifier>> tokens = ugi.getTokens();
        Credentials credentials = new Credentials();
        for (Token<? extends TokenIdentifier> token : tokens) {
            if (!token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                credentials.addToken(token.getService(), token);
                LOG.info("Passing container token {}", token);
            }
        }
        credentials.addToken(delegationToken.getService(), delegationToken);
        DataOutputBuffer dataOutput = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dataOutput);
        byte[] tokenBytes = dataOutput.getData();
        ByteBuffer cTokenBuf = ByteBuffer.wrap(tokenBytes);
        return cTokenBuf.duplicate();
    } catch (IOException e) {
        throw new RuntimeException("Error generating delegation token", e);
    }
}

From source file:com.datatorrent.stram.security.StramUserLogin.java

License:Apache License

public static long refreshTokens(long tokenLifeTime, String destinationDir, String destinationFile,
        final Configuration conf, String hdfsKeyTabFile, final Credentials credentials,
        final InetSocketAddress rmAddress, final boolean renewRMToken) throws IOException {
    long expiryTime = System.currentTimeMillis() + tokenLifeTime;
    //renew tokens
    final String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
    if (tokenRenewer == null || tokenRenewer.length() == 0) {
        throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
    }/*from  w ww.  j a  v a2 s. co m*/
    FileSystem fs = FileSystem.newInstance(conf);
    File keyTabFile;
    try {
        keyTabFile = FSUtil.copyToLocalFileSystem(fs, destinationDir, destinationFile, hdfsKeyTabFile, conf);
    } finally {
        fs.close();
    }
    UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
            UserGroupInformation.getCurrentUser().getUserName(), keyTabFile.getAbsolutePath());
    try {
        ugi.doAs(new PrivilegedExceptionAction<Object>() {
            @Override
            public Object run() throws Exception {
                FileSystem fs1 = FileSystem.newInstance(conf);
                YarnClient yarnClient = null;
                if (renewRMToken) {
                    yarnClient = YarnClient.createYarnClient();
                    yarnClient.init(conf);
                    yarnClient.start();
                }
                Credentials creds = new Credentials();
                try {
                    fs1.addDelegationTokens(tokenRenewer, creds);
                    if (renewRMToken) {
                        org.apache.hadoop.yarn.api.records.Token rmDelToken = yarnClient
                                .getRMDelegationToken(new Text(tokenRenewer));
                        Token<RMDelegationTokenIdentifier> rmToken = ConverterUtils.convertFromYarn(rmDelToken,
                                rmAddress);
                        creds.addToken(rmToken.getService(), rmToken);
                    }
                } finally {
                    fs1.close();
                    if (renewRMToken) {
                        yarnClient.stop();
                    }
                }
                credentials.addAll(creds);
                return null;
            }
        });
        UserGroupInformation.getCurrentUser().addCredentials(credentials);
    } catch (InterruptedException e) {
        LOG.error("Error while renewing tokens ", e);
        expiryTime = System.currentTimeMillis();
    } catch (IOException e) {
        LOG.error("Error while renewing tokens ", e);
        expiryTime = System.currentTimeMillis();
    }
    LOG.debug("number of tokens: {}", credentials.getAllTokens().size());
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.debug("updated token: {}", token);
    }
    keyTabFile.delete();
    return expiryTime;
}

From source file:com.datatorrent.stram.StramClient.java

License:Apache License

private void addRMDelegationToken(final String renewer, final Credentials credentials)
        throws IOException, YarnException {
    // Get the ResourceManager delegation rmToken
    final org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient
            .getRMDelegationToken(new Text(renewer));

    Token<RMDelegationTokenIdentifier> token;
    // TODO: Use the utility method getRMDelegationTokenService in ClientRMProxy to remove the separate handling of
    // TODO: HA and non-HA cases when hadoop dependency is changed to hadoop 2.4 or above
    if (conf.getBoolean(RM_HA_ENABLED, DEFAULT_RM_HA_ENABLED)) {
        LOG.info("Yarn Resource Manager HA is enabled");
        token = getRMHAToken(rmDelegationToken);
    } else {/*from   ww  w.  java2s .  c o m*/
        LOG.info("Yarn Resource Manager HA is not enabled");
        InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
                YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);

        token = ConverterUtils.convertFromYarn(rmDelegationToken, rmAddress);
    }

    LOG.info("RM dt {}", token);

    credentials.addToken(token.getService(), token);
}

From source file:eu.stratosphere.yarn.Utils.java

License:Apache License

public static void setTokensFor(ContainerLaunchContext amContainer, Path[] paths, Configuration conf)
        throws IOException {
    Credentials credentials = new Credentials();
    // for HDFS//from w  ww .  j  a va  2 s  .  co m
    TokenCache.obtainTokensForNamenodes(credentials, paths, conf);
    // for user
    UserGroupInformation currUsr = UserGroupInformation.getCurrentUser();

    Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens();
    for (Token<? extends TokenIdentifier> token : usrTok) {
        final Text id = new Text(token.getIdentifier());
        LOG.info("Adding user token " + id + " with " + token);
        credentials.addToken(id, token);
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength());

    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    amContainer.setTokens(securityTokens);
}

From source file:gobblin.hadoop.token.TokenUtils.java

License:Open Source License

private static void getJhToken(Configuration conf, Credentials cred) throws IOException {
    YarnRPC rpc = YarnRPC.create(conf);//from w  w  w .j  ava 2 s .  c  o  m
    final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);

    LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
    HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class,
            NetUtils.createSocketAddr(serviceAddr), conf);
    LOG.info("Pre-fetching JH token from job history server");

    Token<?> jhToken = null;
    try {
        jhToken = getDelegationTokenFromHS(hsProxy, conf);
    } catch (Exception exc) {
        throw new IOException("Failed to fetch JH token.", exc);
    }

    if (jhToken == null) {
        LOG.error("getDelegationTokenFromHS() returned null");
        throw new IOException("Unable to fetch JH token.");
    }

    LOG.info("Created JH token: " + jhToken.toString());
    LOG.info("Token kind: " + jhToken.getKind());
    LOG.info("Token id: " + jhToken.getIdentifier());
    LOG.info("Token service: " + jhToken.getService());

    cred.addToken(jhToken.getService(), jhToken);
}