List of usage examples for org.apache.hadoop.security Credentials Credentials
public Credentials()
From source file:org.apache.tez.dag.app.TestDAGAppMaster.java
License:Apache License
@SuppressWarnings("deprecation") private void testDagCredentials(boolean doMerge) throws IOException { TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge); conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true); conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString()); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); // create some sample AM credentials Credentials amCreds = new Credentials(); JobTokenSecretManager jtsm = new JobTokenSecretManager(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString())); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, amCreds); TestTokenSecretManager ttsm = new TestTokenSecretManager(); Text tokenAlias1 = new Text("alias1"); Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken1")), ttsm); amCreds.addToken(tokenAlias1, amToken1); Text tokenAlias2 = new Text("alias2"); Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken2")), ttsm); amCreds.addToken(tokenAlias2, amToken2); FileSystem fs = FileSystem.getLocal(conf); FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream); sessionJarsPBOutStream.close();/*from www .j a va 2s. c o m*/ DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), 1, amCreds, "someuser", null); am.init(conf); am.start(); // create some sample DAG credentials Credentials dagCreds = new Credentials(); Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken1")), ttsm); dagCreds.addToken(tokenAlias2, dagToken1); Text tokenAlias3 = new Text("alias3"); Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken2")), ttsm); dagCreds.addToken(tokenAlias3, dagToken2); TezDAGID dagId = TezDAGID.getInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag") .setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build(); DAGImpl dag = am.createDAG(dagPlan, dagId); Credentials fetchedDagCreds = dag.getCredentials(); am.stop(); Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1); if (doMerge) { assertNotNull("AM creds missing from DAG creds", fetchedToken1); compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1)); } else { assertNull("AM creds leaked to DAG creds", fetchedToken1); } compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2)); compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3)); }
From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java
License:Apache License
@Before public void setUp() throws TezException { appId = ApplicationId.newInstance(1000, 1); appAttemptId = ApplicationAttemptId.newInstance(appId, 1); dag = mock(DAG.class); TezDAGID dagID = TezDAGID.getInstance(appId, 1); vertexID = TezVertexID.getInstance(dagID, 1); taskID = TezTaskID.getInstance(vertexID, 1); taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); credentials = new Credentials(); amContainerMap = mock(AMContainerMap.class); Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>(); eventHandler = mock(EventHandler.class); MockClock clock = new MockClock(); appContext = mock(AppContext.class); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(dag).when(appContext).getCurrentDAG(); doReturn(appAcls).when(appContext).getApplicationACLs(); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(clock).when(appContext).getClock(); doReturn(appAttemptId).when(appContext).getApplicationAttemptId(); doReturn(credentials).when(appContext).getAppCredentials(); NodeId nodeId = NodeId.newInstance("localhost", 0); AMContainer amContainer = mock(AMContainer.class); Container container = mock(Container.class); doReturn(nodeId).when(container).getNodeId(); doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class)); doReturn(container).when(amContainer).getContainer(); Configuration conf = new TezConfiguration(); UserPayload defaultPayload;// w ww. ja v a 2s .co m try { defaultPayload = TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new TezUncheckedException(e); } taskAttemptListener = new TaskCommunicatorManagerInterfaceImplForTest(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(defaultPayload))); taskSpec = mock(TaskSpec.class); doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID(); amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0); containerTask = null; }
From source file:org.apache.tez.mapreduce.examples.FilterLinesByWord.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Credentials credentials = new Credentials(); boolean generateSplitsInClient = false; SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser(); try {/*from www .j ava 2 s.c o m*/ generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false); otherArgs = splitCmdLineParser.getRemainingArgs(); } catch (ParseException e1) { System.err.println("Invalid options"); printUsage(); return 2; } if (otherArgs.length != 3) { printUsage(); return 2; } String inputPath = otherArgs[0]; String outputPath = otherArgs[1]; String filterWord = otherArgs[2]; FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(outputPath))) { System.err.println("Output directory : " + outputPath + " already exists"); return 2; } TezConfiguration tezConf = new TezConfiguration(conf); fs.getWorkingDirectory(); Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString()); TezClientUtils.ensureStagingDirExists(tezConf, stagingDir); String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class); if (jarPath == null) { throw new TezUncheckedException( "Could not find any jar containing" + FilterLinesByWord.class.getName() + " in the classpath"); } Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar")); fs.copyFromLocalFile(new Path(jarPath), remoteJarPath); FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath); TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, conf); Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>(); LocalResource dagJarLocalRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(), remoteJarStatus.getModificationTime()); commonLocalResources.put("dag_job.jar", dagJarLocalRsrc); TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, credentials); tezSession.start(); // Why do I need to start the TezSession. Configuration stage1Conf = new JobConf(conf); stage1Conf.set(FILTER_PARAM_NAME, filterWord); Configuration stage2Conf = new JobConf(conf); stage2Conf.set(FileOutputFormat.OUTDIR, outputPath); stage2Conf.setBoolean("mapred.mapper.new-api", false); UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); // Setup stage1 Vertex Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor .create(FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload)) .addTaskLocalFiles(commonLocalResources); DataSourceDescriptor dsd; if (generateSplitsInClient) { // TODO TEZ-1406. Dont' use MRInputLegacy stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath); stage1Conf.setBoolean("mapred.mapper.new-api", false); dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true); } else { dsd = MRInputLegacy.createConfigBuilder(stage1Conf, TextInputFormat.class, inputPath).groupSplits(false) .build(); } stage1Vertex.addDataSource("MRInput", dsd); // Setup stage2 Vertex Vertex stage2Vertex = Vertex.create("stage2", ProcessorDescriptor.create(FilterByWordOutputProcessor.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)), 1); stage2Vertex.addTaskLocalFiles(commonLocalResources); // Configure the Output for stage2 OutputDescriptor od = OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)); OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(MROutputCommitter.class.getName()); stage2Vertex.addDataSink("MROutput", DataSinkDescriptor.create(od, ocd, null)); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig .newBuilder(Text.class.getName(), TextLongPair.class.getName()).setFromConfiguration(tezConf) .build(); DAG dag = DAG.create("FilterLinesByWord"); Edge edge = Edge.create(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty()); dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge); LOG.info("Submitting DAG to Tez Session"); DAGClient dagClient = tezSession.submitDAG(dag); LOG.info("Submitted DAG to Tez Session"); DAGStatus dagStatus = null; String[] vNames = { "stage1", "stage2" }; try { while (true) { dagStatus = dagClient.getDAGStatus(null); if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { break; } try { Thread.sleep(500); } catch (InterruptedException e) { // continue; } } while (dagStatus.getState() == DAGStatus.State.RUNNING) { try { ExampleDriver.printDAGStatus(dagClient, vNames); try { Thread.sleep(1000); } catch (InterruptedException e) { // continue; } dagStatus = dagClient.getDAGStatus(null); } catch (TezException e) { LOG.fatal("Failed to get application progress. Exiting"); return -1; } } dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS)); } finally { fs.delete(stagingDir, true); tezSession.stop(); } ExampleDriver.printDAGStatus(dagClient, vNames, true, true); LOG.info("Application completed. " + "FinalState=" + dagStatus.getState()); return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1; }
From source file:org.apache.tez.service.impl.ContainerRunnerImpl.java
License:Apache License
/** * Submit a container which is ready for running. * The regular pull mechanism will be used to fetch work from the AM * @param request//from w w w . j ava 2 s . c om * @throws TezException */ @Override public void queueContainer(RunContainerRequestProto request) throws TezException { LOG.info("Queuing container for execution: " + request); Map<String, String> env = new HashMap<String, String>(); env.putAll(localEnv); env.put(ApplicationConstants.Environment.USER.name(), request.getUser()); String[] localDirs = new String[localDirsBase.length]; // Setup up local dirs to be application specific, and create them. for (int i = 0; i < localDirsBase.length; i++) { localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(), request.getUser()); try { localFs.mkdirs(new Path(localDirs[i])); } catch (IOException e) { throw new TezException(e); } } LOG.info("Dirs for {} are {}", request.getContainerIdString(), Arrays.toString(localDirs)); // Setup workingDir. This is otherwise setup as Environment.PWD // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream) String workingDir = localDirs[0]; Credentials credentials = new Credentials(); DataInputBuffer dib = new DataInputBuffer(); byte[] tokenBytes = request.getCredentialsBinary().toByteArray(); dib.reset(tokenBytes, tokenBytes.length); try { credentials.readTokenStorageStream(dib); } catch (IOException e) { throw new TezException(e); } Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes. LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString()); ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser()); ContainerRunnerCallable callable = new ContainerRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor); ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable); Futures.addCallback(future, new ContainerRunnerCallback(request, callable)); }
From source file:org.apache.tez.service.impl.ContainerRunnerImpl.java
License:Apache License
/** * Submit an entire work unit - containerId + TaskSpec. * This is intended for a task push from the AM * * @param request/*from www .jav a 2s.c o m*/ * @throws org.apache.tez.dag.api.TezException */ @Override public void submitWork(SubmitWorkRequestProto request) throws TezException { LOG.info("Queuing work for execution: " + request); checkAndThrowExceptionForTests(request); Map<String, String> env = new HashMap<String, String>(); env.putAll(localEnv); env.put(ApplicationConstants.Environment.USER.name(), request.getUser()); String[] localDirs = new String[localDirsBase.length]; // Setup up local dirs to be application specific, and create them. for (int i = 0; i < localDirsBase.length; i++) { localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(), request.getUser()); try { localFs.mkdirs(new Path(localDirs[i])); } catch (IOException e) { throw new TezException(e); } } if (LOG.isDebugEnabled()) { LOG.debug("Dirs are: " + Arrays.toString(localDirs)); } // Setup workingDir. This is otherwise setup as Environment.PWD // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream) String workingDir = localDirs[0]; Credentials credentials = new Credentials(); DataInputBuffer dib = new DataInputBuffer(); byte[] tokenBytes = request.getCredentialsBinary().toByteArray(); dib.reset(tokenBytes, tokenBytes.length); try { credentials.readTokenStorageStream(dib); } catch (IOException e) { throw new TezException(e); } Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes. LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString()); ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser()); TaskRunnerCallable callable = new TaskRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor); ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable); Futures.addCallback(future, new TaskRunnerCallback(request, callable)); }
From source file:org.apache.twill.internal.AbstractTwillService.java
License:Apache License
/** * Attempts to handle secure store update. * * @param message The message received//from www .ja v a2 s. c om * @return {@code true} if the message requests for secure store update, {@code false} otherwise. */ protected final boolean handleSecureStoreUpdate(Message message) { if (!SystemMessages.SECURE_STORE_UPDATED.equals(message)) { return false; } // If not in secure mode, simply ignore the message. if (!UserGroupInformation.isSecurityEnabled()) { return true; } try { Credentials credentials = new Credentials(); Location location = getSecureStoreLocation(); DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream())); try { credentials.readTokenStorageStream(input); } finally { input.close(); } UserGroupInformation.getCurrentUser().addCredentials(credentials); this.credentials = credentials; LOG.info("Secure store updated from {}.", location.toURI()); } catch (Throwable t) { LOG.error("Failed to update secure store.", t); } return true; }
From source file:org.apache.twill.internal.appmaster.ApplicationMasterService.java
License:Apache License
private Credentials createCredentials() { Credentials credentials = new Credentials(); if (!UserGroupInformation.isSecurityEnabled()) { return credentials; }/*from w ww . j a va2s. co m*/ try { credentials.addAll(UserGroupInformation.getCurrentUser().getCredentials()); // Remove the AM->RM tokens Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRM_TOKEN_KIND_NAME)) { iter.remove(); } } } catch (IOException e) { LOG.warn("Failed to get current user. No credentials will be provided to containers.", e); } return credentials; }
From source file:org.apache.twill.internal.container.TwillContainerMain.java
License:Apache License
private static void loadSecureStore() throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return;//w ww. j a va 2 s .c o m } File file = new File(Constants.Files.CREDENTIALS); if (file.exists()) { Credentials credentials = new Credentials(); try (DataInputStream input = new DataInputStream(new FileInputStream(file))) { credentials.readTokenStorageStream(input); } UserGroupInformation.getCurrentUser().addCredentials(credentials); LOG.info("Secure store updated from {}", file); } }
From source file:org.apache.twill.internal.yarn.AbstractYarnTwillService.java
License:Apache License
/** * Attempts to handle secure store update. * * @param message The message received/*from w w w. j a v a 2 s . co m*/ * @return {@code true} if the message requests for secure store update, {@code false} otherwise. */ protected final boolean handleSecureStoreUpdate(Message message) { if (!SystemMessages.SECURE_STORE_UPDATED.equals(message)) { return false; } // If not in secure mode, simply ignore the message. if (!UserGroupInformation.isSecurityEnabled()) { return true; } try { Credentials credentials = new Credentials(); Location location = getSecureStoreLocation(); try (DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream()))) { credentials.readTokenStorageStream(input); } UserGroupInformation.getCurrentUser().addCredentials(credentials); // CDAP-5844 Workaround for HDFS-9276, to update HDFS delegation token for long running application in HA mode cloneHaNnCredentials(location, UserGroupInformation.getCurrentUser()); this.credentials = credentials; LOG.info("Secure store updated from {}.", location); } catch (Throwable t) { LOG.error("Failed to update secure store.", t); } return true; }
From source file:org.apache.twill.internal.yarn.YarnUtils.java
License:Apache License
/** * Decodes {@link Credentials} from the given buffer. * If the buffer is null or empty, it returns an empty Credentials. *//*w w w . j a va 2s.c o m*/ public static Credentials decodeCredentials(ByteBuffer buffer) throws IOException { Credentials credentials = new Credentials(); if (buffer != null && buffer.hasRemaining()) { DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(buffer); credentials.readTokenStorageStream(in); } return credentials; }