List of usage examples for org.apache.hadoop.security UserGroupInformation setConfiguration
@InterfaceAudience.Public @InterfaceStability.Evolving public static void setConfiguration(Configuration conf)
From source file:org.apache.tez.mapreduce.examples.IntersectDataGen.java
License:Apache License
private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession) throws IOException, TezException, InterruptedException { LOG.info("Running IntersectDataGen"); UserGroupInformation.setConfiguration(tezConf); String outDir1 = args[0];//ww w .j a v a 2 s . c om long outDir1Size = Long.parseLong(args[1]); String outDir2 = args[2]; long outDir2Size = Long.parseLong(args[3]); String expectedOutputDir = args[4]; int numTasks = Integer.parseInt(args[5]); Path largeOutPath = null; Path smallOutPath = null; long largeOutSize = 0; long smallOutSize = 0; if (outDir1Size >= outDir2Size) { largeOutPath = new Path(outDir1); largeOutSize = outDir1Size; smallOutPath = new Path(outDir2); smallOutSize = outDir2Size; } else { largeOutPath = new Path(outDir2); largeOutSize = outDir2Size; smallOutPath = new Path(outDir1); smallOutSize = outDir1Size; } Path expectedOutputPath = new Path(expectedOutputDir); // Verify output path existence FileSystem fs = FileSystem.get(tezConf); int res = 0; res = checkOutputDirectory(fs, largeOutPath) + checkOutputDirectory(fs, smallOutPath) + checkOutputDirectory(fs, expectedOutputPath); if (res != 0) { return 3; } if (numTasks <= 0) { System.err.println("NumTasks must be > 0"); return 4; } DAG dag = createDag(tezConf, largeOutPath, smallOutPath, expectedOutputPath, numTasks, largeOutSize, smallOutSize); setupURIsForCredentials(dag, largeOutPath, smallOutPath, expectedOutputPath); tezSession.waitTillReady(); DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null); if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics()); return -1; } return 0; }
From source file:org.apache.tez.mapreduce.examples.IntersectExample.java
License:Apache License
private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession) throws IOException, TezException, InterruptedException { LOG.info("Running IntersectExample"); UserGroupInformation.setConfiguration(tezConf); String streamInputDir = args[0]; String hashInputDir = args[1]; int numPartitions = Integer.parseInt(args[2]); String outputDir = args[3];/*www . j a va 2 s .c o m*/ Path streamInputPath = new Path(streamInputDir); Path hashInputPath = new Path(hashInputDir); Path outputPath = new Path(outputDir); // Verify output path existence FileSystem fs = FileSystem.get(tezConf); if (fs.exists(outputPath)) { System.err.println("Output directory: " + outputDir + " already exists"); return 3; } if (numPartitions <= 0) { System.err.println("NumPartitions must be > 0"); return 4; } DAG dag = createDag(tezConf, streamInputPath, hashInputPath, outputPath, numPartitions); setupURIsForCredentials(dag, streamInputPath, hashInputPath, outputPath); tezSession.waitTillReady(); DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null); if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics()); return -1; } return 0; }
From source file:org.apache.tez.mapreduce.examples.IntersectValidate.java
License:Apache License
private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession) throws IOException, TezException, InterruptedException { LOG.info("Running IntersectValidate"); UserGroupInformation.setConfiguration(tezConf); String lhsDir = args[0];/*from w w w.ja v a 2 s .c o m*/ String rhsDir = args[1]; int numPartitions = 1; if (args.length == 3) { numPartitions = Integer.parseInt(args[2]); } if (numPartitions <= 0) { System.err.println("NumPartitions must be > 0"); return 4; } Path lhsPath = new Path(lhsDir); Path rhsPath = new Path(rhsDir); DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions); setupURIsForCredentials(dag, lhsPath, rhsPath); tezSession.waitTillReady(); DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null); if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics()); return -1; } else { dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS)); TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME, MISSING_KEY_COUNTER_NAME); if (counter == null) { LOG.info("Unable to determing equality"); return -2; } else { if (counter.getValue() != 0) { LOG.info("Validate failed. The two sides are not equivalent"); return -3; } else { LOG.info("Vlidation successful. The two sides are equivalent"); return 0; } } } }
From source file:org.apache.tez.mapreduce.examples.TestOrderedWordCount.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); boolean generateSplitsInClient; SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser(); try {/* w ww .j a v a 2 s .c o m*/ generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false); otherArgs = splitCmdLineParser.getRemainingArgs(); } catch (ParseException e1) { System.err.println("Invalid options"); printUsage(); return 2; } boolean useTezSession = conf.getBoolean("USE_TEZ_SESSION", true); long interJobSleepTimeout = conf.getInt("INTER_JOB_SLEEP_INTERVAL", 0) * 1000; boolean retainStagingDir = conf.getBoolean("RETAIN_STAGING_DIR", false); boolean useMRSettings = conf.getBoolean("USE_MR_CONFIGS", true); // TODO needs to use auto reduce parallelism int intermediateNumReduceTasks = conf.getInt("IREDUCE_NUM_TASKS", 2); if (((otherArgs.length % 2) != 0) || (!useTezSession && otherArgs.length != 2)) { printUsage(); return 2; } List<String> inputPaths = new ArrayList<String>(); List<String> outputPaths = new ArrayList<String>(); for (int i = 0; i < otherArgs.length; i += 2) { inputPaths.add(otherArgs[i]); outputPaths.add(otherArgs[i + 1]); } UserGroupInformation.setConfiguration(conf); TezConfiguration tezConf = new TezConfiguration(conf); TestOrderedWordCount instance = new TestOrderedWordCount(); FileSystem fs = FileSystem.get(conf); String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); FileSystem pathFs = stagingDir.getFileSystem(tezConf); pathFs.mkdirs(new Path(stagingDirStr)); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = pathFs.makeQualified(new Path(stagingDirStr)); TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] { stagingDir }, conf); TezClientUtils.ensureStagingDirExists(tezConf, stagingDir); // No need to add jar containing this class as assumed to be part of // the tez jars. // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir // is the same filesystem as the one used for Input/Output. if (useTezSession) { LOG.info("Creating Tez Session"); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); } else { tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); } TezClient tezSession = TezClient.create("OrderedWordCountSession", tezConf, null, instance.credentials); tezSession.start(); DAGStatus dagStatus = null; DAGClient dagClient = null; String[] vNames = { "initialmap", "intermediate_reducer", "finalreduce" }; Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS); try { for (int dagIndex = 1; dagIndex <= inputPaths.size(); ++dagIndex) { if (dagIndex != 1 && interJobSleepTimeout > 0) { try { LOG.info("Sleeping between jobs, sleepInterval=" + (interJobSleepTimeout / 1000)); Thread.sleep(interJobSleepTimeout); } catch (InterruptedException e) { LOG.info("Main thread interrupted. Breaking out of job loop"); break; } } String inputPath = inputPaths.get(dagIndex - 1); String outputPath = outputPaths.get(dagIndex - 1); if (fs.exists(new Path(outputPath))) { throw new FileAlreadyExistsException("Output directory " + outputPath + " already exists"); } LOG.info("Running OrderedWordCount DAG" + ", dagIndex=" + dagIndex + ", inputPath=" + inputPath + ", outputPath=" + outputPath); Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>(); DAG dag = instance.createDAG(fs, conf, localResources, stagingDir, dagIndex, inputPath, outputPath, generateSplitsInClient, useMRSettings, intermediateNumReduceTasks); boolean doPreWarm = dagIndex == 1 && useTezSession && conf.getBoolean("PRE_WARM_SESSION", true); int preWarmNumContainers = 0; if (doPreWarm) { preWarmNumContainers = conf.getInt("PRE_WARM_NUM_CONTAINERS", 0); if (preWarmNumContainers <= 0) { doPreWarm = false; } } if (doPreWarm) { LOG.info("Pre-warming Session"); PreWarmVertex preWarmVertex = PreWarmVertex.create("PreWarm", preWarmNumContainers, dag.getVertex("initialmap").getTaskResource()); preWarmVertex.addTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles()); preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment()); preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts()); tezSession.preWarm(preWarmVertex); } if (useTezSession) { LOG.info("Waiting for TezSession to get into ready state"); waitForTezSessionReady(tezSession); LOG.info("Submitting DAG to Tez Session, dagIndex=" + dagIndex); dagClient = tezSession.submitDAG(dag); LOG.info("Submitted DAG to Tez Session, dagIndex=" + dagIndex); } else { LOG.info("Submitting DAG as a new Tez Application"); dagClient = tezSession.submitDAG(dag); } while (true) { dagStatus = dagClient.getDAGStatus(statusGetOpts); if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { break; } try { Thread.sleep(500); } catch (InterruptedException e) { // continue; } } while (dagStatus.getState() != DAGStatus.State.SUCCEEDED && dagStatus.getState() != DAGStatus.State.FAILED && dagStatus.getState() != DAGStatus.State.KILLED && dagStatus.getState() != DAGStatus.State.ERROR) { if (dagStatus.getState() == DAGStatus.State.RUNNING) { ExampleDriver.printDAGStatus(dagClient, vNames); } try { try { Thread.sleep(1000); } catch (InterruptedException e) { // continue; } dagStatus = dagClient.getDAGStatus(statusGetOpts); } catch (TezException e) { LOG.fatal("Failed to get application progress. Exiting"); return -1; } } ExampleDriver.printDAGStatus(dagClient, vNames, true, true); LOG.info("DAG " + dagIndex + " completed. " + "FinalState=" + dagStatus.getState()); if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { LOG.info("DAG " + dagIndex + " diagnostics: " + dagStatus.getDiagnostics()); } } } catch (Exception e) { LOG.error("Error occurred when submitting/running DAGs", e); throw e; } finally { if (!retainStagingDir) { pathFs.delete(stagingDir, true); } LOG.info("Shutting down session"); tezSession.stop(); } if (!useTezSession) { ExampleDriver.printDAGStatus(dagClient, vNames); LOG.info("Application completed. " + "FinalState=" + dagStatus.getState()); } return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1; }
From source file:org.apache.tez.mapreduce.examples.TezExampleBase.java
License:Apache License
private int _execute(String[] otherArgs, TezConfiguration tezConf, TezClient tezClient) throws IOException, TezException, InterruptedException { int result = _validateArgs(otherArgs); if (result != 0) { return result; }// w ww . j av a 2 s . c o m if (tezConf == null) { tezConf = new TezConfiguration(getConf()); } UserGroupInformation.setConfiguration(tezConf); boolean ownTezClient = false; if (tezClient == null) { ownTezClient = true; tezClientInternal = createTezClient(tezConf); } try { return runJob(otherArgs, tezConf, tezClientInternal); } finally { if (ownTezClient && tezClientInternal != null) { tezClientInternal.stop(); } } }
From source file:org.apache.tez.mapreduce.examples.UnionExample.java
License:Apache License
public boolean run(String inputPath, String outputPath, Configuration conf) throws Exception { System.out.println("Running UnionExample"); // conf and UGI TezConfiguration tezConf;/* www . ja v a2 s . com*/ if (conf != null) { tezConf = new TezConfiguration(conf); } else { tezConf = new TezConfiguration(); } UserGroupInformation.setConfiguration(tezConf); String user = UserGroupInformation.getCurrentUser().getShortUserName(); // staging dir FileSystem fs = FileSystem.get(tezConf); String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR + user + Path.SEPARATOR + ".staging" + Path.SEPARATOR + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = fs.makeQualified(stagingDir); // No need to add jar containing this class as assumed to be part of // the tez jars. // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir // is the same filesystem as the one used for Input/Output. TezClient tezSession = TezClient.create("UnionExampleSession", tezConf); tezSession.start(); DAGClient dagClient = null; try { if (fs.exists(new Path(outputPath))) { throw new FileAlreadyExistsException("Output directory " + outputPath + " already exists"); } Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>(); DAG dag = createDAG(fs, tezConf, localResources, stagingDir, inputPath, outputPath); tezSession.waitTillReady(); dagClient = tezSession.submitDAG(dag); // monitoring DAGStatus dagStatus = dagClient .waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS)); if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics()); return false; } return true; } finally { fs.delete(stagingDir, true); tezSession.stop(); } }
From source file:org.apache.tez.runtime.task.TezChild.java
License:Apache License
public static void main(String[] args) throws IOException, InterruptedException, TezException { final Configuration defaultConf = new Configuration(); Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); LOG.info("TezChild starting"); assert args.length == 5; String host = args[0];//from w w w .ja v a 2 s. c o m int port = Integer.parseInt(args[1]); final String containerIdentifier = args[2]; final String tokenIdentifier = args[3]; final int attemptNumber = Integer.parseInt(args[4]); final String[] localDirs = TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())); final String pid = System.getenv().get("JVM_PID"); LOG.info("PID, containerIdentifier: " + pid + ", " + containerIdentifier); if (LOG.isDebugEnabled()) { LOG.debug("Info from cmd line: AM-host: " + host + " AM-port: " + port + " containerIdentifier: " + containerIdentifier + " appAttemptNumber: " + attemptNumber + " tokenIdentifier: " + tokenIdentifier); } // Security framework already loaded the tokens into current ugi TezUtilsInternal.addUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()), defaultConf); UserGroupInformation.setConfiguration(defaultConf); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); TezChild tezChild = newTezChild(defaultConf, host, port, containerIdentifier, tokenIdentifier, attemptNumber, localDirs, System.getenv(Environment.PWD.name()), System.getenv(), pid, new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())), credentials, Runtime.getRuntime().maxMemory(), System.getenv(ApplicationConstants.Environment.USER.toString())); tezChild.run(); }
From source file:org.apache.zeppelin.impala.security.JDBCSecurityImpl.java
License:Apache License
/*** * @param properties// w w w. j a v a 2 s . c om */ public static void createSecureConfiguration(Properties properties) { AuthenticationMethod authType = getAuthtype(properties); switch (authType) { case KERBEROS: Configuration conf = new org.apache.hadoop.conf.Configuration(); conf.set("hadoop.security.authentication", KERBEROS.toString()); UserGroupInformation.setConfiguration(conf); try { UserGroupInformation.loginUserFromKeytab(properties.getProperty("zeppelin.jdbc.principal"), properties.getProperty("zeppelin.jdbc.keytab.location")); } catch (IOException e) { LOGGER.error("Failed to get either keytab location or principal name in the " + "interpreter", e); } } }
From source file:org.apache.zeppelin.jdbc.security.JDBCSecurityImpl.java
License:Apache License
/*** * @param properties/*ww w . j a va 2s. co m*/ */ public static void createSecureConfiguration(Properties properties, AuthenticationMethod authType) { switch (authType) { case KERBEROS: Configuration conf = new org.apache.hadoop.conf.Configuration(); conf.set("hadoop.security.authentication", KERBEROS.toString()); UserGroupInformation.setConfiguration(conf); try { // Check TGT before calling login // Ref: https://github.com/apache/hadoop/blob/release-3.0.1-RC1/hadoop-common-project/ // hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java#L1232 if (!UserGroupInformation.isSecurityEnabled() || UserGroupInformation.getCurrentUser().getAuthenticationMethod() != KERBEROS || !UserGroupInformation.isLoginKeytabBased()) { UserGroupInformation.loginUserFromKeytab(properties.getProperty("zeppelin.jdbc.principal"), properties.getProperty("zeppelin.jdbc.keytab.location")); } else { LOGGER.info( "The user has already logged in using Keytab and principal, " + "no action required"); } } catch (IOException e) { LOGGER.error("Failed to get either keytab location or principal name in the " + "interpreter", e); } } }
From source file:org.elasticsearch.hadoop.yarn.am.EsCluster.java
License:Apache License
public void start() { running = true;/*from w ww . j a v a2 s. c o m*/ nmRpc.start(); UserGroupInformation.setConfiguration(cfg); log.info(String.format("Allocating Elasticsearch cluster with %d nodes", appConfig.containersToAllocate())); // register requests Resource capability = YarnCompat.resource(cfg, appConfig.containerMem(), appConfig.containerVCores()); Priority prio = Priority.newInstance(appConfig.amPriority()); for (int i = 0; i < appConfig.containersToAllocate(); i++) { // TODO: Add allocation (host/rack rules) - and disable location constraints ContainerRequest req = new ContainerRequest(capability, null, null, prio); amRpc.addContainerRequest(req); } // update status every 5 sec final long heartBeatRate = TimeUnit.SECONDS.toMillis(5); // start the allocation loop // when a new container is allocated, launch it right away int responseId = 0; try { do { AllocateResponse alloc = amRpc.allocate(responseId++); List<Container> currentlyAllocated = alloc.getAllocatedContainers(); for (Container container : currentlyAllocated) { launchContainer(container); allocatedContainers.add(container.getId()); } if (currentlyAllocated.size() > 0) { int needed = appConfig.containersToAllocate() - allocatedContainers.size(); if (needed > 0) { log.info(String.format("%s containers allocated, %s remaining", allocatedContainers.size(), needed)); } else { log.info(String.format("Fully allocated %s containers", allocatedContainers.size())); } } List<ContainerStatus> completed = alloc.getCompletedContainersStatuses(); for (ContainerStatus status : completed) { if (!completedContainers.contains(status.getContainerId())) { ContainerId containerId = status.getContainerId(); completedContainers.add(containerId); boolean containerSuccesful = false; switch (status.getExitStatus()) { case ContainerExitStatus.SUCCESS: log.info(String.format("Container %s finished succesfully...", containerId)); containerSuccesful = true; break; case ContainerExitStatus.ABORTED: log.warn(String.format("Container %s aborted...", containerId)); break; case ContainerExitStatus.DISKS_FAILED: log.warn(String.format("Container %s ran out of disk...", containerId)); break; case ContainerExitStatus.PREEMPTED: log.warn(String.format("Container %s preempted...", containerId)); break; default: log.warn(String.format("Container %s exited with an invalid/unknown exit code...", containerId)); } if (!containerSuccesful) { log.warn("Cluster has not completed succesfully..."); clusterHasFailed = true; running = false; } } } if (completedContainers.size() == appConfig.containersToAllocate()) { running = false; } if (running) { try { Thread.sleep(heartBeatRate); } catch (Exception ex) { throw new EsYarnNmException("Cluster interrupted"); } } } while (running); } finally { log.info("Cluster has completed running..."); try { Thread.sleep(TimeUnit.SECONDS.toMillis(15)); } catch (InterruptedException e) { throw new RuntimeException(e); } close(); } }