List of usage examples for org.apache.hadoop.security Credentials Credentials
public Credentials()
From source file:org.apache.tez.client.TestTezClientUtils.java
License:Apache License
@Test(timeout = 5000) public void testAMLoggingOptsSimple() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN"); ApplicationId appId = ApplicationId.newInstance(1000, 1); DAG dag = DAG.create("testdag"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) .setTaskLaunchCmdOpts("initialLaunchOpts")); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), new Credentials()); ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), new Credentials(), false, new TezApiVersionInfo(), mock(HistoryACLPolicyManager.class)); List<String> expectedCommands = new LinkedList<String>(); expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"); expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands(); assertEquals(1, commands.size());/*from www . ja v a 2 s. c om*/ for (String expectedCmd : expectedCommands) { assertTrue(commands.get(0).contains(expectedCmd)); } Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment(); String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS); assertNull(logEnv); }
From source file:org.apache.tez.client.TestTezClientUtils.java
License:Apache License
@Test(timeout = 5000) public void testAMLoggingOptsPerLogger() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG"); ApplicationId appId = ApplicationId.newInstance(1000, 1); DAG dag = DAG.create("testdag"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) .setTaskLaunchCmdOpts("initialLaunchOpts")); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), new Credentials()); ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), new Credentials(), false, new TezApiVersionInfo(), mock(HistoryACLPolicyManager.class)); List<String> expectedCommands = new LinkedList<String>(); expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"); expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands(); assertEquals(1, commands.size());/*w w w .j a v a2 s . c om*/ for (String expectedCmd : expectedCommands) { assertTrue(commands.get(0).contains(expectedCmd)); } Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment(); String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS); assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv); }
From source file:org.apache.tez.client.TestTezClientUtils.java
License:Apache License
public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, Configuration conf) throws Exception { Path topLevelDir = null;/* www .j av a 2 s. c o m*/ try { FsPermission publicDirPerms = new FsPermission((short) 0755); // rwxr-xr-x FsPermission privateDirPerms = new FsPermission((short) 0754); // rwxr-xr-- FsPermission publicFilePerms = new FsPermission((short) 0554); // r-xr-xr-- FsPermission privateFilePerms = new FsPermission((short) 0550); // r-xr-x--- String fsURI = remoteFs.getUri().toString(); topLevelDir = new Path(fsURI, "/testLRVisibility"); Assert.assertTrue(remoteFs.mkdirs(topLevelDir, publicDirPerms)); Path publicSubDir = new Path(topLevelDir, "public_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(publicSubDir, publicDirPerms)); Path privateSubDir = new Path(topLevelDir, "private_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(privateSubDir, privateDirPerms)); Path publicFile = new Path(publicSubDir, "public_file"); Assert.assertTrue(remoteFs.createNewFile(publicFile)); remoteFs.setPermission(publicFile, publicFilePerms); Path privateFile = new Path(publicSubDir, "private_file"); Assert.assertTrue(remoteFs.createNewFile(privateFile)); remoteFs.setPermission(privateFile, privateFilePerms); Path publicFileInPrivateSubdir = new Path(privateSubDir, "public_file_in_private_subdir"); Assert.assertTrue(remoteFs.createNewFile(publicFileInPrivateSubdir)); remoteFs.setPermission(publicFileInPrivateSubdir, publicFilePerms); TezConfiguration tezConf = new TezConfiguration(conf); String tmpTezLibUris = String.format("%s,%s,%s,%s", topLevelDir, publicSubDir, privateSubDir, conf.get(TezConfiguration.TEZ_LIB_URIS, "")); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tmpTezLibUris); Map<String, LocalResource> lrMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(publicFile.getName(), LocalResourceVisibility.PUBLIC, lrMap.get(publicFile.getName()).getVisibility()); Assert.assertEquals(privateFile.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(privateFile.getName()).getVisibility()); Assert.assertEquals(publicFileInPrivateSubdir.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(publicFileInPrivateSubdir.getName()).getVisibility()); // test tar.gz tezConf = new TezConfiguration(conf); Path tarFile = new Path(topLevelDir, "foo.tar.gz"); Assert.assertTrue(remoteFs.createNewFile(tarFile)); //public remoteFs.setPermission(tarFile, publicFilePerms); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tarFile.toString()); lrMap.clear(); Assert.assertTrue(TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap)); Assert.assertEquals(LocalResourceVisibility.PUBLIC, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); //private remoteFs.setPermission(tarFile, privateFilePerms); lrMap.clear(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(LocalResourceVisibility.PRIVATE, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); } finally { if (topLevelDir != null) { remoteFs.delete(topLevelDir, true); } } }
From source file:org.apache.tez.client.TestTezClientUtils.java
License:Apache License
@Test(timeout = 5000) public void validateSetTezAuxLocalResourcesWithFilesAndFolders() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); List<String> resources = new ArrayList<String>(); StringBuilder auxUriStr = new StringBuilder(); // Create 2 files Path topDir = new Path(TEST_ROOT_DIR, "validateauxwithfiles"); if (localFs.exists(topDir)) { localFs.delete(topDir, true);/* www .j a va 2s. co m*/ } localFs.mkdirs(topDir); resources.add(createFile(localFs, topDir, "f1.txt").toString()); auxUriStr.append(localFs.makeQualified(topDir).toString()).append(","); Path dir2 = new Path(topDir, "dir2"); localFs.mkdirs(dir2); Path nestedDir = new Path(dir2, "nestedDir"); localFs.mkdirs(nestedDir); createFile(localFs, nestedDir, "nested-f.txt"); resources.add(createFile(localFs, dir2, "dir2-f.txt").toString()); auxUriStr.append(localFs.makeQualified(dir2).toString()).append(","); Path dir3 = new Path(topDir, "dir3"); localFs.mkdirs(dir3); auxUriStr.append(localFs.makeQualified(dir3).toString()).append(","); TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); conf.set(TezConfiguration.TEZ_AUX_URIS, auxUriStr.toString()); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); Set<String> resourceNames = localizedMap.keySet(); Assert.assertEquals(2, resourceNames.size()); Assert.assertTrue(resourceNames.contains("f1.txt")); Assert.assertTrue(resourceNames.contains("dir2-f.txt")); }
From source file:org.apache.tez.client.TezClient.java
License:Apache License
@Private // To be used only by YarnRunner DAGClient submitDAGApplication(ApplicationId appId, DAG dag) throws TezException, IOException { LOG.info("Submitting DAG application with id: " + appId); try {/*from www .j a v a2 s . co m*/ // Use the AMCredentials object in client mode, since this won't be re-used. // Ensures we don't fetch credentially unnecessarily if the user has already provided them. Credentials credentials = amConfig.getCredentials(); if (credentials == null) { credentials = new Credentials(); } TezClientUtils.processTezLocalCredentialsFile(credentials, amConfig.getTezConfiguration()); // Add session token for shuffle TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); // Add credentials for tez-local resources. Map<String, LocalResource> tezJarResources = getTezJarResources(credentials); ApplicationSubmissionContext appContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, dag.getName(), amConfig, tezJarResources, credentials, usingTezArchiveDeploy, apiVersionInfo, historyACLPolicyManager); LOG.info("Submitting DAG to YARN" + ", applicationId=" + appId + ", dagName=" + dag.getName()); frameworkClient.submitApplication(appContext); ApplicationReport appReport = frameworkClient.getApplicationReport(appId); LOG.info("The url to track the Tez AM: " + appReport.getTrackingUrl()); lastSubmittedAppId = appId; } catch (YarnException e) { throw new TezException(e); } return getDAGClient(appId, amConfig.getTezConfiguration(), frameworkClient); }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Obtains tokens for the DAG based on the list of URIs setup in the DAG. The * fetched credentials are populated back into the DAG and can be retrieved * via dag.getCredentials//from w w w. j ava 2s. co m * * @param dag * the dag for which credentials need to be setup * @param sessionCredentials * session credentials which have already been obtained, and will be * required for the DAG * @param conf * @throws IOException */ @Private static Credentials setupDAGCredentials(DAG dag, Credentials sessionCredentials, Configuration conf) throws IOException { Preconditions.checkNotNull(sessionCredentials); TezCommonUtils.logCredentials(LOG, sessionCredentials, "session"); Credentials dagCredentials = new Credentials(); // All session creds are required for the DAG. dagCredentials.mergeAll(sessionCredentials); // Add additional credentials based on any URIs that the user may have specified. // Obtain Credentials for any paths that the user may have configured. addFileSystemCredentialsFromURIs(dag.getURIsForCredentials(), dagCredentials, conf); // Obtain Credentials for the local resources configured on the DAG try { Set<Path> lrPaths = new HashSet<Path>(); for (Vertex v : dag.getVertices()) { for (LocalResource lr : v.getTaskLocalFiles().values()) { lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource())); } List<DataSourceDescriptor> dataSources = v.getDataSources(); for (DataSourceDescriptor dataSource : dataSources) { addFileSystemCredentialsFromURIs(dataSource.getURIsForCredentials(), dagCredentials, conf); } List<DataSinkDescriptor> dataSinks = v.getDataSinks(); for (DataSinkDescriptor dataSink : dataSinks) { addFileSystemCredentialsFromURIs(dataSink.getURIsForCredentials(), dagCredentials, conf); } } for (LocalResource lr : dag.getTaskLocalFiles().values()) { lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource())); } Path[] paths = lrPaths.toArray(new Path[lrPaths.size()]); TokenCache.obtainTokensForFileSystems(dagCredentials, paths, conf); } catch (URISyntaxException e) { throw new IOException(e); } return dagCredentials; }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Create an ApplicationSubmissionContext to launch a Tez AM * @param appId Application Id/*w w w. j av a2 s . com*/ * @param dag DAG to be submitted * @param amName Name for the application * @param amConfig AM Configuration * @param tezJarResources Resources to be used by the AM * @param sessionCreds the credential object which will be populated with session specific * @param historyACLPolicyManager * @return an ApplicationSubmissionContext to launch a Tez AM * @throws IOException * @throws YarnException */ @Private @VisibleForTesting public static ApplicationSubmissionContext createApplicationSubmissionContext(ApplicationId appId, DAG dag, String amName, AMConfiguration amConfig, Map<String, LocalResource> tezJarResources, Credentials sessionCreds, boolean tezLrsAsArchive, TezApiVersionInfo apiVersionInfo, HistoryACLPolicyManager historyACLPolicyManager) throws IOException, YarnException { Preconditions.checkNotNull(sessionCreds); TezConfiguration conf = amConfig.getTezConfiguration(); FileSystem fs = TezClientUtils.ensureStagingDirExists(conf, TezCommonUtils.getTezBaseStagingPath(conf)); String strAppId = appId.toString(); Path tezSysStagingPath = TezCommonUtils.createTezSystemStagingPath(conf, strAppId); Path binaryConfPath = TezCommonUtils.getTezConfStagingPath(tezSysStagingPath); binaryConfPath = fs.makeQualified(binaryConfPath); // Setup resource requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT)); capability.setVirtualCores(amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT)); if (LOG.isDebugEnabled()) { LOG.debug("AppMaster capability = " + capability); } // Setup required Credentials for the AM launch. DAG specific credentials // are handled separately. ByteBuffer securityTokens = null; // Setup security tokens Credentials amLaunchCredentials = new Credentials(); if (amConfig.getCredentials() != null) { amLaunchCredentials.addAll(amConfig.getCredentials()); } // Add Staging dir creds to the list of session credentials. TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { binaryConfPath }, conf); // Add session specific credentials to the AM credentials. amLaunchCredentials.mergeAll(sessionCreds); DataOutputBuffer dob = new DataOutputBuffer(); amLaunchCredentials.writeTokenStorageToStream(dob); securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Setup the command to run the AM List<String> vargs = new ArrayList<String>(8); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability); vargs.add(amOpts); String amLogLevelString = amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_LOG_LEVEL, TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT); String[] amLogParams = parseLogParams(amLogLevelString); String amLogLevel = amLogParams[0]; maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs); // FIX sun bug mentioned in TEZ-327 vargs.add("-Dsun.nio.ch.bugLevel=''"); vargs.add(TezConstants.TEZ_APPLICATION_MASTER_CLASS); if (dag == null) { vargs.add("--" + TezConstants.TEZ_SESSION_MODE_CLI_OPTION); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDOUT); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR); Vector<String> vargsFinal = new Vector<String>(8); // Final command StringBuilder mergedCommand = new StringBuilder(); for (CharSequence str : vargs) { mergedCommand.append(str).append(" "); } vargsFinal.add(mergedCommand.toString()); if (LOG.isDebugEnabled()) { LOG.debug("Command to launch container for ApplicationMaster is : " + mergedCommand); } Map<String, String> environment = new TreeMap<String, String>(); TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV, TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); addVersionInfoToEnv(environment, apiVersionInfo); addLogParamsToEnv(environment, amLogParams); Map<String, LocalResource> amLocalResources = new TreeMap<String, LocalResource>(); // Not fetching credentials for AMLocalResources. Expect this to be provided via AMCredentials. if (amConfig.getAMLocalResources() != null) { amLocalResources.putAll(amConfig.getAMLocalResources()); } amLocalResources.putAll(tezJarResources); // Setup Session ACLs and update conf as needed Map<String, String> aclConfigs = null; if (historyACLPolicyManager != null) { if (dag == null) { aclConfigs = historyACLPolicyManager.setupSessionACLs(amConfig.getTezConfiguration(), appId); } else { // Non-session mode // As only a single DAG is support, we should combine AM and DAG ACLs under the same // acl management layer aclConfigs = historyACLPolicyManager.setupNonSessionACLs(amConfig.getTezConfiguration(), appId, dag.getDagAccessControls()); } } // emit conf as PB file ConfigurationProto finalConfProto = createFinalConfProtoForApp(amConfig.getTezConfiguration(), aclConfigs); FSDataOutputStream amConfPBOutBinaryStream = null; try { amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath); finalConfProto.writeTo(amConfPBOutBinaryStream); } finally { if (amConfPBOutBinaryStream != null) { amConfPBOutBinaryStream.close(); } } LocalResource binaryConfLRsrc = TezClientUtils.createLocalResource(fs, binaryConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); amConfig.setBinaryConfLR(binaryConfLRsrc); amLocalResources.put(TezConstants.TEZ_PB_BINARY_CONF_NAME, binaryConfLRsrc); // Create Session Jars definition to be sent to AM as a local resource Path sessionJarsPath = TezCommonUtils.getTezAMJarStagingPath(tezSysStagingPath); FSDataOutputStream sessionJarsPBOutStream = null; try { sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, sessionJarsPath); // Write out the initial list of resources which will be available in the AM DAGProtos.PlanLocalResourcesProto amResourceProto; if (amLocalResources != null && !amLocalResources.isEmpty()) { amResourceProto = DagTypeConverters.convertFromLocalResources(amLocalResources); } else { amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance(); } amResourceProto.writeDelimitedTo(sessionJarsPBOutStream); } finally { if (sessionJarsPBOutStream != null) { sessionJarsPBOutStream.close(); } } LocalResource sessionJarsPBLRsrc = TezClientUtils.createLocalResource(fs, sessionJarsPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); amLocalResources.put(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, sessionJarsPBLRsrc); String user = UserGroupInformation.getCurrentUser().getShortUserName(); ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration()); Map<ApplicationAccessType, String> acls = aclManager.toYARNACls(); if (dag != null) { DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, sessionCreds); // emit protobuf DAG file style Path binaryPath = TezCommonUtils.getTezBinPlanStagingPath(tezSysStagingPath); if (LOG.isDebugEnabled()) { LOG.debug("Stage directory information for AppId :" + appId + " tezSysStagingPath :" + tezSysStagingPath + " binaryConfPath :" + binaryConfPath + " sessionJarsPath :" + sessionJarsPath + " binaryPlanPath :" + binaryPath); } FSDataOutputStream dagPBOutBinaryStream = null; try { //binary output dagPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryPath); dagPB.writeTo(dagPBOutBinaryStream); } finally { if (dagPBOutBinaryStream != null) { dagPBOutBinaryStream.close(); } } amLocalResources.put(TezConstants.TEZ_PB_PLAN_BINARY_NAME, TezClientUtils.createLocalResource(fs, binaryPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) { Path textPath = localizeDagPlanAsText(dagPB, fs, amConfig, strAppId, tezSysStagingPath); amLocalResources.put(TezConstants.TEZ_PB_PLAN_TEXT_NAME, TezClientUtils.createLocalResource(fs, textPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); } } // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(amLocalResources, environment, vargsFinal, null, securityTokens, acls); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE); appContext.setApplicationId(appId); appContext.setResource(capability); if (amConfig.getQueueName() != null) { appContext.setQueue(amConfig.getQueueName()); } appContext.setApplicationName(amName); appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION, TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT)); appContext.setAMContainerSpec(amContainer); appContext.setMaxAppAttempts(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT)); return appContext; }
From source file:org.apache.tez.common.ContainerTask.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { shouldDie = in.readBoolean();/*from w ww . j a v a 2s . c o m*/ boolean taskComing = in.readBoolean(); if (taskComing) { taskSpec = new TaskSpec(); taskSpec.readFields(in); } int numAdditionalResources = in.readInt(); additionalResources = Maps.newHashMap(); if (numAdditionalResources != -1) { for (int i = 0; i < numAdditionalResources; i++) { String resourceName = in.readUTF(); TezLocalResource localResource = new TezLocalResource(); localResource.readFields(in); additionalResources.put(resourceName, localResource); } } credentialsChanged = in.readBoolean(); if (credentialsChanged) { boolean hasCredentials = in.readBoolean(); if (hasCredentials) { credentials = new Credentials(); credentials.readFields(in); } } }
From source file:org.apache.tez.common.security.TestTokenCache.java
License:Apache License
@Test(timeout = 5000) @SuppressWarnings("deprecation") public void testBinaryCredentials() throws Exception { String binaryTokenFile = null; try {//from w w w . j a v a 2 s. c o m Path TEST_ROOT_DIR = new Path("target"); binaryTokenFile = FileSystem.getLocal(conf).makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri() .getPath(); MockFileSystem fs1 = createFileSystemForServiceName("service1"); MockFileSystem fs2 = createFileSystemForServiceName("service2"); MockFileSystem fs3 = createFileSystemForServiceName("service3"); // get the tokens for fs1 & fs2 and write out to binary creds file Credentials creds = new Credentials(); Token<?> token1 = fs1.getDelegationToken(renewer); Token<?> token2 = fs2.getDelegationToken(renewer); creds.addToken(token1.getService(), token1); creds.addToken(token2.getService(), token2); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); Credentials newCreds = new Credentials(); TokenCache.mergeBinaryTokens(newCreds, conf, binaryTokenFile); Assert.assertTrue(newCreds.getAllTokens().size() > 0); checkTokens(creds, newCreds); } finally { if (binaryTokenFile != null) { try { FileSystem.getLocal(conf).delete(new Path(binaryTokenFile)); } catch (IOException e) { // Ignore } } } }
From source file:org.apache.tez.common.TezCommonUtils.java
License:Apache License
public static Credentials parseCredentialsBytes(byte[] credentialsBytes) throws IOException { Credentials credentials = new Credentials(); DataInputBuffer dib = new DataInputBuffer(); try {//from ww w. j a v a2 s .c o m byte[] tokenBytes = credentialsBytes; dib.reset(tokenBytes, tokenBytes.length); credentials.readTokenStorageStream(dib); return credentials; } finally { dib.close(); } }