List of usage examples for org.apache.hadoop.security.token Token Token
public Token(T id, SecretManager<T> mgr)
From source file:com.cloudera.impala.security.DelegationTokenSecretManager.java
License:Apache License
public synchronized DelegationTokenManager.DelegationToken getDelegationToken(String owner, String renewer, String realUser) throws IOException { if (realUser == null) realUser = owner;// ww w .jav a 2 s . c o m DelegationTokenIdentifier ident = new DelegationTokenIdentifier(new Text(owner), new Text(renewer), new Text(realUser)); Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>(ident, this); return new DelegationTokenManager.DelegationToken(encodeIdentifier(ident.serialize()), encodePassword(t.getPassword()), t.encodeToUrlString().getBytes()); }
From source file:com.cloudera.impala.security.DelegationTokenSecretManager.java
License:Apache License
public synchronized String getDelegationToken(String renewer) throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); Text owner = new Text(ugi.getUserName()); Text realUser = null;// w w w .j a v a 2 s. co m if (ugi.getRealUser() != null) { realUser = new Text(ugi.getRealUser().getUserName()); } DelegationTokenIdentifier ident = new DelegationTokenIdentifier(owner, new Text(renewer), realUser); Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>(ident, this); LOGGER.info("Generated delegation token. Identifer=" + ident); return t.encodeToUrlString(); }
From source file:com.datatorrent.stram.LaunchContainerRunnable.java
License:Apache License
public static ByteBuffer getTokens(StramDelegationTokenManager delegationTokenManager, InetSocketAddress heartbeatAddress) throws IOException { if (UserGroupInformation.isSecurityEnabled()) { UserGroupInformation ugi = UserGroupInformation.getLoginUser(); StramDelegationTokenIdentifier identifier = new StramDelegationTokenIdentifier( new Text(ugi.getUserName()), new Text(""), new Text("")); String service = heartbeatAddress.getAddress().getHostAddress() + ":" + heartbeatAddress.getPort(); Token<StramDelegationTokenIdentifier> stramToken = new Token<StramDelegationTokenIdentifier>(identifier, delegationTokenManager); stramToken.setService(new Text(service)); return getTokens(ugi, stramToken); }/* w w w . j a v a 2 s .c o m*/ return null; }
From source file:com.datatorrent.stram.security.StramWSFilter.java
License:Apache License
private String createClientToken(String username, String service) throws IOException { StramDelegationTokenIdentifier tokenIdentifier = new StramDelegationTokenIdentifier(new Text(username), new Text(loginUser), new Text()); //tokenIdentifier.setSequenceNumber(sequenceNumber.getAndAdd(1)); //byte[] password = tokenManager.addIdentifier(tokenIdentifier); //Token<StramDelegationTokenIdentifier> token = new Token<StramDelegationTokenIdentifier>(tokenIdentifier.getBytes(), password, tokenIdentifier.getKind(), new Text(service)); Token<StramDelegationTokenIdentifier> token = new Token<StramDelegationTokenIdentifier>(tokenIdentifier, tokenManager);//from w w w. j a v a 2s . c o m token.setService(new Text(service)); return token.encodeToUrlString(); }
From source file:com.datatorrent.stram.StreamingAppMasterService.java
License:Apache License
private Token<StramDelegationTokenIdentifier> allocateDelegationToken(String username, InetSocketAddress address) { StramDelegationTokenIdentifier identifier = new StramDelegationTokenIdentifier(new Text(username), new Text(""), new Text("")); String service = address.getAddress().getHostAddress() + ":" + address.getPort(); Token<StramDelegationTokenIdentifier> stramToken = new Token<StramDelegationTokenIdentifier>(identifier, delegationTokenManager);/* w w w .ja va 2 s . co m*/ stramToken.setService(new Text(service)); return stramToken; }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
static void createSessionToken(String tokenIdentifier, JobTokenSecretManager jobTokenSecretManager, Credentials credentials) {//from ww w. j a v a 2 s . c o m JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(tokenIdentifier)); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jobTokenSecretManager); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, credentials); }
From source file:org.apache.tez.dag.app.TestDAGAppMaster.java
License:Apache License
@SuppressWarnings("deprecation") private void testDagCredentials(boolean doMerge) throws IOException { TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge); conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true); conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString()); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); // create some sample AM credentials Credentials amCreds = new Credentials(); JobTokenSecretManager jtsm = new JobTokenSecretManager(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString())); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, amCreds); TestTokenSecretManager ttsm = new TestTokenSecretManager(); Text tokenAlias1 = new Text("alias1"); Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken1")), ttsm); amCreds.addToken(tokenAlias1, amToken1); Text tokenAlias2 = new Text("alias2"); Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken2")), ttsm); amCreds.addToken(tokenAlias2, amToken2); FileSystem fs = FileSystem.getLocal(conf); FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream); sessionJarsPBOutStream.close();/* ww w . j ava 2s .co m*/ DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), 1, amCreds, "someuser", null); am.init(conf); am.start(); // create some sample DAG credentials Credentials dagCreds = new Credentials(); Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken1")), ttsm); dagCreds.addToken(tokenAlias2, dagToken1); Text tokenAlias3 = new Text("alias3"); Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken2")), ttsm); dagCreds.addToken(tokenAlias3, dagToken2); TezDAGID dagId = TezDAGID.getInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag") .setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build(); DAGImpl dag = am.createDAG(dagPlan, dagId); Credentials fetchedDagCreds = dag.getCredentials(); am.stop(); Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1); if (doMerge) { assertNotNull("AM creds missing from DAG creds", fetchedToken1); compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1)); } else { assertNull("AM creds leaked to DAG creds", fetchedToken1); } compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2)); compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3)); }
From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java
License:Apache License
@Test(timeout = 5000) public void testPortRange_NotSpecified() throws IOException, TezException { Configuration conf = new Configuration(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text("fakeIdentifier")); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager()); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, credentials); UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf); taskAttemptListener = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(userPayload))); // no exception happen, should started properly taskAttemptListener.init(conf);/*from w w w. j a v a 2 s . c om*/ taskAttemptListener.start(); }
From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java
License:Apache License
private boolean testPortRange(int port) { boolean succeedToAllocate = true; try {/*from www .ja va 2 s . com*/ Configuration conf = new Configuration(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text("fakeIdentifier")); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager()); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, credentials); conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port); UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf); taskAttemptListener = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(userPayload))); taskAttemptListener.init(conf); taskAttemptListener.start(); int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort(); assertEquals(port, resultedPort); } catch (Exception e) { succeedToAllocate = false; } finally { if (taskAttemptListener != null) { try { taskAttemptListener.close(); } catch (IOException e) { e.printStackTrace(); fail("fail to stop TaskAttemptListener"); } } } return succeedToAllocate; }
From source file:org.apache.tez.mapreduce.processor.reduce.TestReduceProcessor.java
License:Apache License
@Test(timeout = 5000) public void testReduceProcessor() throws Exception { final String dagName = "mrdag0"; String mapVertexName = MultiStageMRConfigUtil.getInitialMapVertexName(); String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName(); JobConf jobConf = new JobConf(defaultConf); setUpJobConf(jobConf);/*from www. ja v a 2s. c o m*/ MRHelpers.translateMRConfToTez(jobConf); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); Path mapInput = new Path(workDir, "map0"); MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(MRRuntimeProtos.MRInputUserPayloadProto .newBuilder().setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)) .build().toByteArray()))), 1); OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); // Run a map TestUmbilical testUmbilical = new TestUmbilical(); LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0, mapInput, testUmbilical, dagName, mapVertexName, Collections.singletonList(mapInputSpec), Collections.singletonList(mapOutputSpec)); mapTask.initialize(); mapTask.run(); mapTask.close(); // One VME, One DME Assert.assertEquals(2, testUmbilical.getEvents().size()); Assert.assertEquals(EventType.VERTEX_MANAGER_EVENT, testUmbilical.getEvents().get(0).getEventType()); Assert.assertEquals(EventType.COMPOSITE_DATA_MOVEMENT_EVENT, testUmbilical.getEvents().get(1).getEventType()); CompositeDataMovementEvent cdmEvent = (CompositeDataMovementEvent) testUmbilical.getEvents().get(1) .getEvent(); Assert.assertEquals(1, cdmEvent.getCount()); DataMovementEvent dme = cdmEvent.getEvents().iterator().next(); dme.setTargetIndex(0); LOG.info("Starting reduce..."); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(dagName)); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(identifier, jobTokenSecretManager); shuffleToken.setService(identifier.getJobId()); jobConf.setOutputFormat(SequenceFileOutputFormat.class); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true); FileOutputFormat.setOutputPath(jobConf, new Path(workDir, "output")); ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create(ReduceProcessor.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)); InputSpec reduceInputSpec = new InputSpec(mapVertexName, InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex", OutputDescriptor .create(MROutputLegacy.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); // Now run a reduce TaskSpec taskSpec = new TaskSpec(TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0), dagName, reduceVertexName, -1, reduceProcessorDesc, Collections.singletonList(reduceInputSpec), Collections.singletonList(reduceOutputSpec), null); Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>(); serviceConsumerMetadata.put(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ShuffleUtils.convertJobTokenToBytes(shuffleToken)); Map<String, String> serviceProviderEnvMap = new HashMap<String, String>(); ByteBuffer shufflePortBb = ByteBuffer.allocate(4).putInt(0, 8000); AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, shufflePortBb, serviceProviderEnvMap); LogicalIOProcessorRuntimeTask task = new LogicalIOProcessorRuntimeTask(taskSpec, 0, jobConf, new String[] { workDir.toString() }, new TestUmbilical(), serviceConsumerMetadata, serviceProviderEnvMap, HashMultimap.<String, String>create(), null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory()); List<Event> destEvents = new LinkedList<Event>(); destEvents.add(dme); task.initialize(); OrderedGroupedInputLegacy sortedOut = (OrderedGroupedInputLegacy) task.getInputs().values().iterator() .next(); sortedOut.handleEvents(destEvents); task.run(); task.close(); // MRTask mrTask = (MRTask)t.getProcessor(); // TODO NEWTEZ Verify the partitioner has not been created // Likely not applicable anymore. // Assert.assertNull(mrTask.getPartitioner()); // Only a task commit happens, hence the data is still in the temporary directory. Path reduceOutputDir = new Path(new Path(workDir, "output"), "_temporary/0/" + IDConverter.toMRTaskIdForOutput(TezTestUtils.getMockTaskId(0, 1, 0))); Path reduceOutputFile = new Path(reduceOutputDir, "part-v001-o000-00000"); SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, jobConf); LongWritable key = new LongWritable(); Text value = new Text(); long prev = Long.MIN_VALUE; while (reader.next(key, value)) { if (prev != Long.MIN_VALUE) { Assert.assertTrue(prev < key.get()); prev = key.get(); } } reader.close(); }