Example usage for org.apache.hadoop.security Credentials Credentials

Introduction

In this page you can find the example usage for org.apache.hadoop.security Credentials Credentials.

Prototype

public Credentials() 

Source Link

Document

Create an empty credentials instance.

Usage

From source file:org.apache.tez.dag.app.dag.impl.TestVertexRecovery.java

License:Apache License

@Before
public void setUp() throws IOException {

    dispatcher = new DrainDispatcher();
    dispatcher.register(DAGEventType.class, mock(EventHandler.class));
    vertexEventHandler = new VertexEventHanlder();
    dispatcher.register(VertexEventType.class, vertexEventHandler);
    taskEventHandler = new TaskEventHandler();
    dispatcher.register(TaskEventType.class, taskEventHandler);
    dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventHandler());
    dispatcher.init(new Configuration());
    dispatcher.start();/*w w  w .  j av  a 2s .com*/

    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);

    DAGPlan dagPlan = createDAGPlan();
    dag = new DAGImpl(dagId, new Configuration(), dagPlan, dispatcher.getEventHandler(),
            mock(TaskAttemptListener.class), new Credentials(), new SystemClock(), user,
            mock(TaskHeartbeatHandler.class), mockAppContext);
    when(mockAppContext.getCurrentDAG()).thenReturn(dag);

    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
    LOG.info("finish setUp");
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexRecovery.java

License:Apache License

/**
 * vertex1(New) -> StartRecoveryTransition(SUCCEEDED)
 * @throws IOException //w  ww  .j  a  v  a  2s .  co m
 */
@Test(timeout = 5000)
public void testRecovery_Desired_SUCCEEDED_OnlySummaryLog() throws IOException {
    DAGPlan dagPlan = createDAGPlanSingleVertex();
    dag = new DAGImpl(dagId, new Configuration(), dagPlan, dispatcher.getEventHandler(),
            mock(TaskAttemptListener.class), new Credentials(), new SystemClock(), user,
            mock(TaskHeartbeatHandler.class), mockAppContext);
    when(mockAppContext.getCurrentDAG()).thenReturn(dag);
    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));

    VertexImpl vertex1 = (VertexImpl) dag.getVertex("vertex1");
    VertexFinishedEvent vertexFinishEvent = new VertexFinishedEvent();
    vertexFinishEvent
            .fromSummaryProtoStream(SummaryEventProto.newBuilder().setDagId(dag.getID().toString())
                    .setEventType(HistoryEventType.VERTEX_FINISHED.ordinal()).setTimestamp(100L)
                    .setEventPayload(VertexFinishStateProto.newBuilder().setNumTasks(2)
                            .setState(VertexState.SUCCEEDED.ordinal())
                            .setVertexId(vertex1.getVertexId().toString()).build().toByteString())
                    .build());
    VertexState recoveredState = vertex1.restoreFromEvent(vertexFinishEvent);
    // numTasks is recovered from summary log
    assertEquals(2, vertex1.numTasks);
    assertEquals(VertexState.SUCCEEDED, recoveredState);
    vertex1.handle(new VertexEventRecoverVertex(vertex1.getVertexId(), VertexState.SUCCEEDED));
    dispatcher.await();
    assertEquals(VertexState.SUCCEEDED, vertex1.getState());
    assertEquals(vertex1.numTasks, vertex1.succeededTaskCount);
    assertEquals(vertex1.numTasks, vertex1.completedTaskCount);
}

---

From source file:org.apache.tez.dag.app.DAGAppMaster.java

License:Apache License

/** Create and initialize (but don't start) a single dag. */
DAGImpl createDAG(DAGPlan dagPB, TezDAGID dagId) {
    if (dagId == null) {
        dagId = TezDAGID.getInstance(appAttemptID.getApplicationId(), dagCounter.incrementAndGet());
    }//from  w w w.j av  a2 s  . c  o m

    Credentials dagCredentials = null;
    if (dagPB.hasCredentialsBinary()) {
        dagCredentials = DagTypeConverters.convertByteStringToCredentials(dagPB.getCredentialsBinary());
        TezCommonUtils.logCredentials(LOG, dagCredentials, "dag");
    } else {
        dagCredentials = new Credentials();
    }
    // TODO Does this move to the client in case of work-preserving recovery.
    TokenCache.setSessionToken(sessionToken, dagCredentials);

    // create single dag
    DAGImpl newDag = new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(), taskAttemptListener,
            dagCredentials, clock, appMasterUgi.getShortUserName(), taskHeartbeatHandler, context);

    try {
        if (LOG.isDebugEnabled()) {
            LOG.info("JSON dump for submitted DAG, dagId=" + dagId.toString() + ", json="
                    + DAGUtils.generateSimpleJSONPlan(dagPB).toString());
        }
    } catch (JSONException e) {
        LOG.warn("Failed to generate json for DAG", e);
    }

    generateDAGVizFile(dagId, dagPB, logDirs);
    writePBTextFile(newDag);
    return newDag;
}

---

From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java

License:Apache License

/**
 * Create the common {@link ContainerLaunchContext} for all attempts.
 *
 * @param applicationACLs//from   w  w w.ja v  a2 s .c o  m
 */
private static ContainerLaunchContext createCommonContainerLaunchContext(
        Map<ApplicationAccessType, String> applicationACLs, Credentials credentials,
        Map<String, LocalResource> localResources) {

    // Application environment
    Map<String, String> environment = new HashMap<String, String>();

    // Service data
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();

    // Tokens

    // Setup up task credentials buffer
    ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
    try {
        Credentials containerCredentials = new Credentials();

        // All Credentials need to be set so that YARN can localize the resources
        // correctly, even though they may not be used by all tasks which will run
        // on this container.

        LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #" + credentials.numberOfSecretKeys()
                + " secret keys for NM use for launching container");
        containerCredentials.addAll(credentials);

        DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
        containerCredentials.writeTokenStorageToStream(containerTokens_dob);
        containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
                containerTokens_dob.getLength());

        // Add shuffle token
        LOG.info("Putting shuffle token in serviceData");
        serviceData.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
                serializeServiceData(TokenCache.getSessionToken(containerCredentials)));
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }
    // Construct the actual Container
    // The null fields are per-container and will be constructed for each
    // container separately.
    ContainerLaunchContext container = ContainerLaunchContext.newInstance(localResources, environment, null,
            serviceData, containerCredentialsBuffer, applicationACLs);
    return container;
}

---

From source file:org.apache.tez.dag.app.rm.container.TestAMContainer.java

License:Apache License

@Test(timeout = 5000)
public void testLocalResourceAddition() {
    WrappedContainer wc = new WrappedContainer();

    String rsrc1 = "rsrc1";
    String rsrc2 = "rsrc2";
    String rsrc3 = "rsrc3";

    Map<String, LocalResource> initialResources = Maps.newHashMap();
    initialResources.put(rsrc1, createLocalResource(rsrc1));

    wc.launchContainer(initialResources, new Credentials());
    wc.containerLaunched();//from w  ww .j a v  a 2 s .  c  o m
    wc.assignTaskAttempt(wc.taskAttemptID);
    ArgumentCaptor<AMContainerTask> argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    AMContainerTask task1 = argumentCaptor.getAllValues().get(0);
    assertEquals(0, task1.getAdditionalResources().size());
    wc.taskAttemptSucceeded(wc.taskAttemptID);

    // Add some resources to the next task.
    Map<String, LocalResource> additionalResources = Maps.newHashMap();
    additionalResources.put(rsrc2, createLocalResource(rsrc2));
    additionalResources.put(rsrc3, createLocalResource(rsrc3));

    TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
    wc.assignTaskAttempt(taID2, additionalResources, new Credentials());
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    AMContainerTask task2 = argumentCaptor.getAllValues().get(1);
    Map<String, LocalResource> pullTaskAdditionalResources = task2.getAdditionalResources();
    assertEquals(2, pullTaskAdditionalResources.size());
    pullTaskAdditionalResources.remove(rsrc2);
    pullTaskAdditionalResources.remove(rsrc3);
    assertEquals(0, pullTaskAdditionalResources.size());
    wc.taskAttemptSucceeded(taID2);

    // Verify Resources registered for this container.
    Map<String, LocalResource> containerLRs = new HashMap<String, LocalResource>(
            wc.amContainer.containerLocalResources);
    assertEquals(3, containerLRs.size());
    containerLRs.remove(rsrc1);
    containerLRs.remove(rsrc2);
    containerLRs.remove(rsrc3);
    assertEquals(0, containerLRs.size());

    // Try launching another task with the same reosurces as Task2. Verify the
    // task is not asked to re-localize again.
    TezTaskAttemptID taID3 = TezTaskAttemptID.getInstance(wc.taskID, 3);
    wc.assignTaskAttempt(taID3, new HashMap<String, LocalResource>(), new Credentials());
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    AMContainerTask task3 = argumentCaptor.getAllValues().get(2);
    assertEquals(0, task3.getAdditionalResources().size());
    wc.taskAttemptSucceeded(taID3);

    // Verify references are cleared after a container completes.
    wc.containerCompleted();
    assertNull(wc.amContainer.containerLocalResources);
    assertNull(wc.amContainer.additionalLocalResources);
}

---

From source file:org.apache.tez.dag.app.rm.container.TestAMContainer.java

License:Apache License

@SuppressWarnings("unchecked")
@Test(timeout = 5000)/*  w w  w  .  j a v  a2  s  . co m*/
public void testCredentialsTransfer() {
    WrappedContainerMultipleDAGs wc = new WrappedContainerMultipleDAGs();

    TezDAGID dagID2 = TezDAGID.getInstance("800", 500, 2);
    TezDAGID dagID3 = TezDAGID.getInstance("800", 500, 3);
    TezVertexID vertexID2 = TezVertexID.getInstance(dagID2, 1);
    TezVertexID vertexID3 = TezVertexID.getInstance(dagID3, 1);
    TezTaskID taskID2 = TezTaskID.getInstance(vertexID2, 1);
    TezTaskID taskID3 = TezTaskID.getInstance(vertexID3, 1);

    TezTaskAttemptID attempt11 = TezTaskAttemptID.getInstance(wc.taskID, 200);
    TezTaskAttemptID attempt12 = TezTaskAttemptID.getInstance(wc.taskID, 300);
    TezTaskAttemptID attempt21 = TezTaskAttemptID.getInstance(taskID2, 200);
    TezTaskAttemptID attempt22 = TezTaskAttemptID.getInstance(taskID2, 300);
    TezTaskAttemptID attempt31 = TezTaskAttemptID.getInstance(taskID3, 200);
    TezTaskAttemptID attempt32 = TezTaskAttemptID.getInstance(taskID3, 300);

    Map<String, LocalResource> LRs = new HashMap<String, LocalResource>();
    AMContainerTask fetchedTask = null;
    ArgumentCaptor<AMContainerTask> argumentCaptor = null;

    Token<TokenIdentifier> amGenToken = mock(Token.class);
    Token<TokenIdentifier> token1 = mock(Token.class);
    Token<TokenIdentifier> token3 = mock(Token.class);

    Credentials containerCredentials = new Credentials();
    TokenCache.setSessionToken(amGenToken, containerCredentials);

    Text token1Name = new Text("tokenDag1");
    Text token3Name = new Text("tokenDag3");

    Credentials dag1Credentials = new Credentials();
    dag1Credentials.addToken(new Text(token1Name), token1);
    Credentials dag3Credentials = new Credentials();
    dag3Credentials.addToken(new Text(token3Name), token3);

    wc.launchContainer(new HashMap<String, LocalResource>(), containerCredentials);
    wc.containerLaunched();
    wc.assignTaskAttempt(attempt11, LRs, dag1Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(1)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    fetchedTask = argumentCaptor.getAllValues().get(0);
    assertTrue(fetchedTask.haveCredentialsChanged());
    assertNotNull(fetchedTask.getCredentials());
    assertNotNull(fetchedTask.getCredentials().getToken(token1Name));
    wc.taskAttemptSucceeded(attempt11);

    wc.assignTaskAttempt(attempt12, LRs, dag1Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(2)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    fetchedTask = argumentCaptor.getAllValues().get(1);
    assertFalse(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt12);

    // Move to running a second DAG, with no credentials.
    wc.setNewDAGID(dagID2);
    wc.assignTaskAttempt(attempt21, LRs, null);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(3)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    fetchedTask = argumentCaptor.getAllValues().get(2);
    assertTrue(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt21);

    wc.assignTaskAttempt(attempt22, LRs, null);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(4)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    fetchedTask = argumentCaptor.getAllValues().get(3);
    assertFalse(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt22);

    // Move to running a third DAG, with Credentials this time
    wc.setNewDAGID(dagID3);
    wc.assignTaskAttempt(attempt31, LRs, dag3Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(5)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    fetchedTask = argumentCaptor.getAllValues().get(4);
    assertTrue(fetchedTask.haveCredentialsChanged());
    assertNotNull(fetchedTask.getCredentials());
    assertNotNull(fetchedTask.getCredentials().getToken(token3Name));
    assertNull(fetchedTask.getCredentials().getToken(token1Name));
    wc.taskAttemptSucceeded(attempt31);

    wc.assignTaskAttempt(attempt32, LRs, dag1Credentials);
    argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class);
    verify(wc.tal, times(6)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID));
    fetchedTask = argumentCaptor.getAllValues().get(5);
    assertFalse(fetchedTask.haveCredentialsChanged());
    assertNull(fetchedTask.getCredentials());
    wc.taskAttemptSucceeded(attempt32);
}

---

From source file:org.apache.tez.dag.app.rm.TestContainerReuse.java

License:Apache License

private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID, TaskAttempt ta,
        Resource capability, String[] hosts, String[] racks, Priority priority,
        Map<String, LocalResource> localResources, String jvmOpts) {
    return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority,
            new ContainerContext(localResources, new Credentials(), new HashMap<String, String>(), jvmOpts));
}

---

From source file:org.apache.tez.dag.app.rm.TestTaskSchedulerManager.java

License:Apache License

@Test(timeout = 5000)
public void testSimpleAllocate() throws Exception {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);//from  ww w. j  av a 2  s  . c om
    schedulerHandler.start();

    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
    TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
    when(mockAttemptId.getId()).thenReturn(0);
    when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
    Resource resource = Resource.newInstance(1024, 1);
    ContainerContext containerContext = new ContainerContext(new HashMap<String, LocalResource>(),
            new Credentials(), new HashMap<String, String>(), "");
    int priority = 10;
    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);

    ContainerId mockCId = mock(ContainerId.class);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(mockCId);

    AMContainer mockAMContainer = mock(AMContainer.class);
    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
    when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);

    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);

    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null,
            mockTaskAttempt, locHint, priority, containerContext, 0, 0, 0);
    schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
    assertEquals(1, mockEventHandler.events.size());
    assertTrue(mockEventHandler.events.get(0) instanceof AMContainerEventAssignTA);
    AMContainerEventAssignTA assignEvent = (AMContainerEventAssignTA) mockEventHandler.events.get(0);
    assertEquals(priority, assignEvent.getPriority());
    assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
}

---

From source file:org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl.java

License:Apache License

private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec)
        throws IOException {
    SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
    builder.setContainerIdString(containerId.toString());
    builder.setAmHost(getAddress().getHostName());
    builder.setAmPort(getAddress().getPort());
    Credentials taskCredentials = new Credentials();
    // Credentials can change across DAGs. Ideally construct only once per DAG.
    taskCredentials.addAll(getContext().getAMCredentials());

    ByteBuffer credentialsBinary = credentialMap.get(taskSpec.getDAGName());
    if (credentialsBinary == null) {
        credentialsBinary = serializeCredentials(getContext().getAMCredentials());
        credentialMap.putIfAbsent(taskSpec.getDAGName(), credentialsBinary.duplicate());
    } else {/*from  w ww .jav a 2 s  . c  o  m*/
        credentialsBinary = credentialsBinary.duplicate();
    }
    builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
    builder.setTaskSpec(ProtoConverters.convertTaskSpecToProto(taskSpec));
    return builder.build();
}

---

From source file:org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl.java

License:Apache License

private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
    Credentials containerCredentials = new Credentials();
    containerCredentials.addAll(credentials);
    DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
    containerCredentials.writeTokenStorageToStream(containerTokens_dob);
    ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
            containerTokens_dob.getLength());
    return containerCredentialsBuffer;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• »