Example usage for org.apache.hadoop.security Credentials Credentials

List of usage examples for org.apache.hadoop.security Credentials Credentials

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. Credentials
  6. Credentials

Introduction

In this page you can find the example usage for org.apache.hadoop.security Credentials Credentials.

Prototype

public Credentials()

Source Link

Document

Create an empty credentials instance.

Usage

From source file:org.apache.hive.hcatalog.templeton.SecureProxySupport.java

License:Apache License

/**
 * @param fsTokens not null//  w  w w .ja  v a 2  s  .  c om
 */
private void writeProxyDelegationTokens(final Token<?> fsTokens[], final Token<?> msToken,
        final Configuration conf, String user, final Path tokenPath) throws IOException, InterruptedException {

    LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName());
    final UserGroupInformation ugi = UgiFactory.getUgi(user);

    ugi.doAs(new PrivilegedExceptionAction<Object>() {
        public Object run() throws IOException {
            Credentials cred = new Credentials();
            for (Token<?> fsToken : fsTokens) {
                cred.addToken(fsToken.getService(), fsToken);
            }
            cred.addToken(msToken.getService(), msToken);
            cred.writeTokenStorageFile(tokenPath, conf);
            return null;
        }
    });

}

From source file:org.apache.ignite.yarn.IgniteYarnClient.java

License:Apache License

/**
 * Main methods has one mandatory parameter and one optional parameter.
 *
 * @param args Path to jar mandatory parameter and property file is optional.
 *//*from   w w  w  . j a  va 2s  .co  m*/
public static void main(String[] args) throws Exception {
    checkArguments(args);

    // Set path to app master jar.
    String pathAppMasterJar = args[0];

    ClusterProperties props = ClusterProperties.from(args.length == 2 ? args[1] : null);

    YarnConfiguration conf = new YarnConfiguration();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    FileSystem fs = FileSystem.get(conf);

    Path ignite;

    // Load ignite and jar
    if (props.ignitePath() == null)
        ignite = getIgnite(props, fs);
    else
        ignite = new Path(props.ignitePath());

    // Upload the jar file to HDFS.
    Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar,
            props.igniteWorkDir() + File.separator + IgniteYarnUtils.JAR_NAME);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    amContainer.setCommands(Collections
            .singletonList(Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName()
                    + IgniteYarnUtils.SPACE + ignite.toUri() + IgniteYarnUtils.YARN_LOG_OUT));

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = IgniteYarnUtils.setupFile(appJar, fs, LocalResourceType.FILE);

    amContainer.setLocalResources(Collections.singletonMap(IgniteYarnUtils.JAR_NAME, appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = props.toEnvs();

    setupAppMasterEnv(appMasterEnv, conf);

    amContainer.setEnvironment(appMasterEnv);

    // Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        Credentials creds = new Credentials();

        String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);

        if (tokRenewer == null || tokRenewer.length() == 0)
            throw new IOException("Master Kerberos principal for the RM is not set.");

        log.info("Found RM principal: " + tokRenewer);

        final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds);

        if (tokens != null)
            log.info("File system delegation tokens: " + Arrays.toString(tokens));

        amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds));
    }

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(512);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("ignition"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();

    yarnClient.submitApplication(appContext);

    log.log(Level.INFO, "Submitted application. Application id: {0}", appId);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();

    while (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING
            || appState == YarnApplicationState.SUBMITTED || appState == YarnApplicationState.ACCEPTED) {
        TimeUnit.SECONDS.sleep(1L);

        appReport = yarnClient.getApplicationReport(appId);

        if (appState != YarnApplicationState.ACCEPTED
                && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED)
            log.log(Level.INFO, "Application {0} is ACCEPTED.", appId);

        appState = appReport.getYarnApplicationState();
    }

    log.log(Level.INFO, "Application {0} is {1}.", new Object[] { appId, appState });
}

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

/**
 * Main run function for the client//from   w ww.j  a v a2  s. co m
 * @return true if application completed successfully
 * @throws IOException
 * @throws YarnException
 */
public boolean run() throws IOException, YarnException {

    LOG.info("Running Client");
    yarnClient.start();

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());

    List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM");
    for (NodeReport node : clusterNodeReports) {
        LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                + node.getNumContainers());
    }

    QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
            + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
            + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
            + queueInfo.getChildQueues().size());

    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                    + userAcl.name());
        }
    }

    if (domainId != null && domainId.length() > 0 && toCreateDomain) {
        prepareTimelineDomain();
    }

    // Get a new application id
    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    // TODO get min/max resource capabilities from RM and change memory ask if needed
    // If we do not have min/max, we may not be able to correctly request
    // the required resources from the RM for the app master
    // Memory ask has to be a multiple of min and less than max.
    // Dump out information about cluster capability as seen by the resource manager
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    // A resource ask cannot exceed the max.
    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);

    if (amVCores > maxVCores) {
        LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value."
                + ", specified=" + amVCores + ", max=" + maxVCores);
        amVCores = maxVCores;
    }

    // set the application name
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    appContext.setKeepContainersAcrossApplicationAttempts(keepContainers);
    appContext.setApplicationName(appName);

    if (attemptFailuresValidityInterval >= 0) {
        appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
    }

    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    FileSystem fs = FileSystem.get(conf);
    Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources,
            null);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null);
    }

    // Set the necessary security tokens as needed
    //amContainer.setContainerTokens(containerToken);

    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    // put location of shell script into env
    // using the env info, the application master will create the correct local resource for the
    // eventual containers that will be launched to execute the shell scripts
    if (domainId != null && domainId.length() > 0) {
        env.put(Constants.TIMELINEDOMAIN, domainId);
    }

    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$())
            .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());

    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx" + amMemory + "m");
    // Set class name
    vargs.add(appMasterMainClass);
    // Set params for Application Master
    vargs.add(ApplicationMaster.AMOptions.toArgs(ApplicationMaster.AMOptions.ZK_QUORUM.of(zkQuorum),
            ApplicationMaster.AMOptions.ZK_ROOT.of(zkRoot),
            ApplicationMaster.AMOptions.APP_JAR_PATH.of(ajPath.toString())));
    if (null != nodeLabelExpression) {
        appContext.setNodeLabelExpression(nodeLabelExpression);
    }
    for (Map.Entry<String, String> entry : shellEnv.entrySet()) {
        vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue());
    }

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null,
            null, null);

    // Set up resource type requirements
    // For now, both memory and vcores are supported, so we set memory and
    // vcores requirements
    Resource capability = Resource.newInstance(amMemory, amVCores);
    appContext.setResource(capability);

    // Service data is a binary blob that can be passed to the application
    // Not needed in this scenario

    // Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
        Credentials credentials = new Credentials();
        String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }

        // For now, only getting tokens for the default file-system.
        final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        amContainer.setTokens(fsTokens);
    }

    appContext.setAMContainerSpec(amContainer);

    // Set the priority for the application master
    Priority pri = Priority.newInstance(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Submit the application to the applications manager
    // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
    // Ignore the response as either a valid response object is returned on success
    // or an exception thrown to denote some form of a failure
    LOG.info("Submitting application to ASM");

    yarnClient.submitApplication(appContext);

    // Monitor the application
    return monitorApplication(appId);

}

From source file:org.apache.oozie.action.hadoop.TestHCatCredentials.java

License:Apache License

@Test
public void testAddToJobConfFromHCat() throws Exception {
    File hcatConfig = new File(OOZIE_HOME_DIR, "hcatConf.xml");
    Writer fw = new OutputStreamWriter(new FileOutputStream(hcatConfig), StandardCharsets.UTF_8);
    fw.write(getHiveConfig(TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2));
    fw.flush();/*ww w .jav  a2  s . c o m*/
    fw.close();
    @SuppressWarnings("deprecation")
    Configuration conf = services.getConf();
    conf.set(HCatAccessorService.HCAT_CONFIGURATION, OOZIE_HOME_DIR + "/hcatConf.xml");
    services.init();
    HCatCredentialHelper hcatCredHelper = Mockito.mock(HCatCredentialHelper.class);
    PowerMockito.whenNew(HCatCredentialHelper.class).withNoArguments().thenReturn(hcatCredHelper);
    CredentialsProperties credProps = new CredentialsProperties("", "");
    credProps.setProperties(new HashMap<String, String>());
    HCatCredentials hcatCred = new HCatCredentials();
    final JobConf jobConf = new JobConf(false);
    Credentials credentials = new Credentials();
    PowerMockito.doAnswer(new Answer<Void>() {
        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            Configuration jConf = (Configuration) args[1];
            jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[2]);
            jConf.set(HCAT_METASTORE_URI, (String) args[3]);
            return null;
        }
    }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2);
    hcatCred.updateCredentials(credentials, jobConf, credProps, null);
    assertEquals(TEST_HIVE_METASTORE_PRINCIPAL2, jobConf.get(HCAT_METASTORE_PRINCIPAL));
    assertEquals(TEST_HIVE_METASTORE_URI2, jobConf.get(HCAT_METASTORE_URI));
    assertNull(jobConf.get(HIVE_METASTORE_PRINCIPAL));
    assertNull(jobConf.get(HIVE_METASTORE_URI));
    hcatConfig.delete();
}

From source file:org.apache.oozie.action.hadoop.TestHCatCredentials.java

License:Apache License

@Test
public void testAddToJobConfFromHiveConf() throws Exception {
    services.init();/*from  ww  w . ja v  a  2  s.  c  o m*/
    CredentialsProperties credProps = new CredentialsProperties("", "");
    credProps.setProperties(new HashMap<String, String>());
    HCatCredentials hcatCred = new HCatCredentials();
    final JobConf jobConf = new JobConf(false);
    Credentials credentials = new Credentials();
    HCatCredentialHelper hcatCredHelper = Mockito.mock(HCatCredentialHelper.class);
    PowerMockito.whenNew(HCatCredentialHelper.class).withNoArguments().thenReturn(hcatCredHelper);
    PowerMockito.doAnswer(new Answer<Void>() {
        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            Configuration jConf = (Configuration) args[1];
            jConf.set(HIVE_METASTORE_PRINCIPAL, (String) args[2]);
            jConf.set(HIVE_METASTORE_URI, (String) args[3]);
            return null;
        }
    }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL, TEST_HIVE_METASTORE_URI);
    hcatCred.updateCredentials(credentials, jobConf, credProps, null);
    assertEquals(TEST_HIVE_METASTORE_PRINCIPAL, jobConf.get(HIVE_METASTORE_PRINCIPAL));
    assertEquals(TEST_HIVE_METASTORE_URI, jobConf.get(HIVE_METASTORE_URI));
    assertNull(jobConf.get(HCAT_METASTORE_PRINCIPAL));
    assertNull(jobConf.get(HCAT_METASTORE_URI));
}

From source file:org.apache.oozie.action.hadoop.TestHCatCredentials.java

License:Apache License

@Test
public void testAddToJobConfFromOozieConfig() throws Exception {
    services.init();/*from  w ww .  j av a2s. c o  m*/
    HCatCredentialHelper hcatCredHelper = Mockito.mock(HCatCredentialHelper.class);
    PowerMockito.whenNew(HCatCredentialHelper.class).withNoArguments().thenReturn(hcatCredHelper);
    CredentialsProperties credProps = new CredentialsProperties("", "");
    HashMap<String, String> prop = new HashMap<String, String>();
    prop.put("hcat.metastore.principal", TEST_HIVE_METASTORE_PRINCIPAL2);
    prop.put("hcat.metastore.uri", TEST_HIVE_METASTORE_URI2);
    credProps.setProperties(prop);
    HCatCredentials hcatCred = new HCatCredentials();
    final JobConf jobConf = new JobConf(false);
    Credentials credentials = new Credentials();
    PowerMockito.doAnswer(new Answer<Void>() {
        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            JobConf jConf = (JobConf) args[1];
            jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[2]);
            jConf.set(HCAT_METASTORE_URI, (String) args[3]);
            return null;
        }
    }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2);
    hcatCred.updateCredentials(credentials, jobConf, credProps, null);
    assertEquals(TEST_HIVE_METASTORE_PRINCIPAL2, jobConf.get(HCAT_METASTORE_PRINCIPAL));
    assertEquals(TEST_HIVE_METASTORE_URI2, jobConf.get(HCAT_METASTORE_URI));
    assertNull(jobConf.get(HIVE_METASTORE_PRINCIPAL));
    assertNull(jobConf.get(HIVE_METASTORE_URI));
}

From source file:org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan.java

License:Apache License

public TezOperPlan() {
    creds = new Credentials();
}

From source file:org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler.java

License:Apache License

public DAG buildDAG(TezOperPlan tezPlan, Map<String, LocalResource> localResources)
        throws IOException, YarnException {
    String jobName = pigContext.getProperties().getProperty(PigContext.JOB_NAME, "pig");
    DAG tezDag = new DAG(jobName + "-" + dagIdentifier);
    dagIdentifier++;//from  ww w  .  j a v a 2s .  com
    tezDag.setCredentials(new Credentials());
    TezDagBuilder dagBuilder = new TezDagBuilder(pigContext, tezPlan, tezDag, localResources);
    dagBuilder.visit();
    return tezDag;
}

From source file:org.apache.sqoop.connector.hdfs.security.SecurityUtils.java

License:Apache License

/**
 * Generate delegation tokens for current user (this code is suppose to run in doAs) and store them
 * serialized in given mutable context./*from w w  w . j av  a2s .c om*/
 */
static public void generateDelegationTokens(MutableContext context, Path path, Configuration configuration)
        throws IOException {
    if (!UserGroupInformation.isSecurityEnabled()) {
        LOG.info("Running on unsecured cluster, skipping delegation token generation.");
        return;
    }

    // String representation of all tokens that we will create (most likely single one)
    List<String> tokens = new LinkedList<>();

    Credentials credentials = new Credentials();
    TokenCache.obtainTokensForNamenodes(credentials, new Path[] { path }, configuration);
    for (Token token : credentials.getAllTokens()) {
        LOG.info("Generated token: " + token.toString());
        tokens.add(serializeToken(token));
    }

    // The context classes are transferred via "Credentials" rather then with jobconf, so we're not leaking the DT out here
    if (tokens.size() > 0) {
        context.setString(HdfsConstants.DELEGATION_TOKENS, StringUtils.join(tokens, " "));
    }
}

From source file:org.apache.storm.common.AbstractAutoCreds.java

License:Apache License

private Credentials doGetCredentials(Map<String, String> credentials, String configKey) {
    Credentials credential = null;//from  ww  w  .  ja v  a  2 s.c  om
    if (credentials != null && credentials.containsKey(getCredentialKey(configKey))) {
        try {
            byte[] credBytes = DatatypeConverter
                    .parseBase64Binary(credentials.get(getCredentialKey(configKey)));
            ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(credBytes));

            credential = new Credentials();
            credential.readFields(in);
        } catch (Exception e) {
            LOG.error("Could not obtain credentials from credentials map.", e);
        }
    }
    return credential;

}