List of usage examples for org.apache.hadoop.mapreduce MRJobConfig USER_NAME
String USER_NAME
To view the source code for org.apache.hadoop.mapreduce MRJobConfig USER_NAME.
Click Source Link
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * Application entry point//from w w w .ja v a 2 s. co m * * @param args * command-line args (set by GuaguaYarnClient, if any) */ public static void main(final String[] args) { LOG.info("Starting GuaguaAppMaster. "); String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); try { UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appMasterUgi = UserGroupInformation.createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); // Now remove the AM->RM token so tasks don't have it Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } final GuaguaAppMaster appMaster = new GuaguaAppMaster(containerId, appAttemptId, conf); appMasterUgi.doAs(new PrivilegedAction<Void>() { @Override public Void run() { boolean result = false; try { result = appMaster.run(); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } if (result) { LOG.info("Guagua Application Master completed successfully. exiting"); System.exit(0); } else { LOG.info("Guagua Application Master failed. exiting"); System.exit(2); } return null; } }); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } }
From source file:ml.shifu.guagua.yarn.GuaguaYarnTask.java
License:Apache License
public static void main(String[] args) { LOG.info("args:{}", Arrays.toString(args)); if (args.length != 7) { throw new IllegalStateException(String.format( "GuaguaYarnTask could not construct a TaskAttemptID for the Guagua job from args: %s", Arrays.toString(args))); }/*from ww w .j a v a2 s .co m*/ String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); try { Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appTaskUGI = UserGroupInformation.createRemoteUser(jobUserName); appTaskUGI.addCredentials(credentials); @SuppressWarnings("rawtypes") final GuaguaYarnTask<?, ?> guaguaYarnTask = new GuaguaYarnTask(appAttemptId, containerId, Integer.parseInt(args[args.length - 3]), args[args.length - 2], args[args.length - 1], conf); appTaskUGI.doAs(new PrivilegedAction<Void>() { @Override public Void run() { guaguaYarnTask.run(); return null; } }); } catch (Throwable t) { LOG.error("GuaguaYarnTask threw a top-level exception, failing task", t); System.exit(2); } System.exit(0); }
From source file:org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.java
License:Apache License
/** {@inheritDoc} */ @SuppressWarnings("ConstantConditions") @Override// w ww. j a v a 2 s . com public void initialize(URI name, Configuration cfg) throws IOException { enterBusy(); try { if (rmtClient != null) throw new IOException("File system is already initialized: " + rmtClient); A.notNull(name, "name"); A.notNull(cfg, "cfg"); super.initialize(name, cfg); setConf(cfg); String disableCacheName = String.format("fs.%s.impl.disable.cache", name.getScheme()); cacheEnabled = !cfg.getBoolean(disableCacheName, false); mgmt = cfg.getBoolean(IGFS_MANAGEMENT, false); if (!IGFS_SCHEME.equals(name.getScheme())) throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME + "://[name]/[optional_path], actual=" + name + ']'); uri = name; uriAuthority = uri.getAuthority(); setUser(cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME)); // Override sequential reads before prefetch if needed. seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0); if (seqReadsBeforePrefetch > 0) seqReadsBeforePrefetchOverride = true; // In Ignite replication factor is controlled by data cache affinity. // We use replication factor to force the whole file to be stored on local node. dfltReplication = (short) cfg.getInt("dfs.replication", 3); // Get file colocation control flag. colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false); preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false); // Get log directory. String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR); File logDirFile = U.resolveIgnitePath(logDirCfg); String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null; rmtClient = new HadoopIgfsWrapper(uriAuthority, logDir, cfg, LOG); // Handshake. IgfsHandshakeResponse handshake = rmtClient.handshake(logDir); igfsGrpBlockSize = handshake.blockSize(); IgfsPaths paths = handshake.secondaryPaths(); // Initialize client logger. Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false); if (handshake.sampling() != null ? handshake.sampling() : logEnabled) { // Initiate client logger. if (logDir == null) throw new IOException("Failed to resolve log directory: " + logDirCfg); Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority, DFLT_IGFS_LOG_BATCH_SIZE); clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize); } else clientLog = IgfsLogger.disabledLogger(); modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes()); boolean initSecondary = paths.defaultMode() == PROXY; if (!initSecondary && paths.pathModes() != null && !paths.pathModes().isEmpty()) { for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) { IgfsMode mode = pathMode.getValue(); if (mode == PROXY) { initSecondary = true; break; } } } if (initSecondary) { Map<String, String> props = paths.properties(); String secUri = props.get(SECONDARY_FS_URI); String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH); String secUserName = props.get(SECONDARY_FS_USER_NAME); try { SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath, secUserName); secondaryFs = secProvider.createFileSystem(); secondaryUri = secProvider.uri(); } catch (IOException e) { if (!mgmt) throw new IOException("Failed to connect to the secondary file system: " + secUri, e); else LOG.warn( "Visor failed to create secondary file system (operations on paths with PROXY mode " + "will have no effect): " + e.getMessage()); } } } finally { leaveBusy(); } }
From source file:org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.java
License:Apache License
/** * @param name URI for file system./*w w w . j a v a 2 s . c om*/ * @param cfg Configuration. * @throws URISyntaxException if name has invalid syntax. * @throws IOException If initialization failed. */ public IgniteHadoopFileSystem(URI name, Configuration cfg) throws URISyntaxException, IOException { super(HadoopIgfsEndpoint.normalize(name), IGFS_SCHEME, false, -1); uri = name; try { initialize(name, cfg); } catch (IOException e) { // Close client if exception occurred. if (rmtClient != null) rmtClient.close(false); throw e; } workingDir = new IgfsPath("/user/" + cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME)); }
From source file:org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem.java
License:Apache License
/** {@inheritDoc} */ @Override/* w w w .java 2 s.c o m*/ public void initialize(URI name, Configuration cfg) throws IOException { enterBusy(); try { if (rmtClient != null) throw new IOException("File system is already initialized: " + rmtClient); A.notNull(name, "name"); A.notNull(cfg, "cfg"); super.initialize(name, cfg); setConf(cfg); String disableCacheName = String.format("fs.%s.impl.disable.cache", name.getScheme()); cacheEnabled = !cfg.getBoolean(disableCacheName, false); mgmt = cfg.getBoolean(IGFS_MANAGEMENT, false); if (!IGFS_SCHEME.equals(name.getScheme())) throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME + "://[name]/[optional_path], actual=" + name + ']'); uri = name; uriAuthority = uri.getAuthority(); setUser(cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME)); // Override sequential reads before prefetch if needed. seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0); if (seqReadsBeforePrefetch > 0) seqReadsBeforePrefetchOverride = true; // In GG replication factor is controlled by data cache affinity. // We use replication factor to force the whole file to be stored on local node. dfltReplication = (short) cfg.getInt("dfs.replication", 3); // Get file colocation control flag. colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false); preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false); // Get log directory. String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR); File logDirFile = U.resolveIgnitePath(logDirCfg); String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null; rmtClient = new IgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG); // Handshake. IgfsHandshakeResponse handshake = rmtClient.handshake(logDir); igfsGrpBlockSize = handshake.blockSize(); IgfsPaths paths = handshake.secondaryPaths(); // Initialize client logger. Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false); if (handshake.sampling() != null ? handshake.sampling() : logEnabled) { // Initiate client logger. if (logDir == null) throw new IOException("Failed to resolve log directory: " + logDirCfg); Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority, DFLT_IGFS_LOG_BATCH_SIZE); clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize); } else clientLog = IgfsLogger.disabledLogger(); modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes()); boolean initSecondary = paths.defaultMode() == PROXY; if (paths.pathModes() != null && !paths.pathModes().isEmpty()) { for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) { IgfsMode mode = pathMode.getValue(); initSecondary |= mode == PROXY; } } if (initSecondary) { Map<String, String> props = paths.properties(); String secUri = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_URI); String secConfPath = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH); if (secConfPath == null) throw new IOException("Failed to connect to the secondary file system because configuration " + "path is not provided."); if (secUri == null) throw new IOException( "Failed to connect to the secondary file system because URI is not " + "provided."); try { secondaryUri = new URI(secUri); URL secondaryCfgUrl = U.resolveIgniteUrl(secConfPath); Configuration conf = new Configuration(); if (secondaryCfgUrl != null) conf.addResource(secondaryCfgUrl); String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme()); conf.setBoolean(prop, true); secondaryFs = FileSystem.get(secondaryUri, conf); } catch (URISyntaxException ignore) { if (!mgmt) throw new IOException("Failed to resolve secondary file system URI: " + secUri); else LOG.warn( "Visor failed to create secondary file system (operations on paths with PROXY mode " + "will have no effect)."); } catch (IOException e) { if (!mgmt) throw new IOException("Failed to connect to the secondary file system: " + secUri, e); else LOG.warn( "Visor failed to create secondary file system (operations on paths with PROXY mode " + "will have no effect): " + e.getMessage()); } } } finally { leaveBusy(); } }
From source file:org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.java
License:Apache License
/** * @param name URI for file system./*from w w w . j a va 2s .c o m*/ * @param cfg Configuration. * @throws URISyntaxException if name has invalid syntax. * @throws IOException If initialization failed. */ public IgfsHadoopFileSystem(URI name, Configuration cfg) throws URISyntaxException, IOException { super(IgfsHadoopEndpoint.normalize(name), IGFS_SCHEME, false, -1); uri = name; try { initialize(name, cfg); } catch (IOException e) { // Close client if exception occurred. if (rmtClient != null) rmtClient.close(false); throw e; } workingDir = new IgfsPath("/user/" + cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME)); }
From source file:org.apache.ignite.internal.processors.hadoop.fs.GridHadoopDistributedFileSystem.java
License:Apache License
/** {@inheritDoc} */ @Override//from w w w .j a v a 2 s . c om public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setUser(conf.get(MRJobConfig.USER_NAME, DFLT_USER_NAME)); }
From source file:org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemCacheUtils.java
License:Apache License
/** * Gets non-null user name as per the Hadoop viewpoint. * @param cfg the Hadoop job configuration, may be null. * @return the user name, never null./*from ww w. j a v a2 s.c o m*/ */ private static String getMrHadoopUser(Configuration cfg) throws IOException { String user = cfg.get(MRJobConfig.USER_NAME); if (user == null) user = IgniteHadoopFileSystem.getFsHadoopUser(); return user; }
From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopFileSystemsTest.java
License:Apache License
/** * Test the file system with specified URI for the multi-thread working directory support. * * @param uri Base URI of the file system (scheme and authority). * @throws Exception If fails./*from w w w. jav a 2s .c o m*/ */ private void testFileSystem(final URI uri) throws Exception { final Configuration cfg = new Configuration(); setupFileSystems(cfg); cfg.set(GridHadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString()); final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT); final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT); final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT); final CountDownLatch finishPhase = new CountDownLatch(THREAD_COUNT); final Path[] newUserInitWorkDir = new Path[THREAD_COUNT]; final Path[] newWorkDir = new Path[THREAD_COUNT]; final Path[] newAbsWorkDir = new Path[THREAD_COUNT]; final Path[] newInstanceWorkDir = new Path[THREAD_COUNT]; final AtomicInteger threadNum = new AtomicInteger(0); GridTestUtils.runMultiThreadedAsync(new Runnable() { @Override public void run() { try { int curThreadNum = threadNum.getAndIncrement(); FileSystem fs = FileSystem.get(uri, cfg); GridHadoopFileSystemsUtils.setUser(fs, "user" + curThreadNum); if ("file".equals(uri.getScheme())) FileSystem.get(uri, cfg).setWorkingDirectory(new Path("file:///user/user" + curThreadNum)); changeUserPhase.countDown(); changeUserPhase.await(); newUserInitWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory(); FileSystem.get(uri, cfg).setWorkingDirectory(new Path("folder" + curThreadNum)); changeDirPhase.countDown(); changeDirPhase.await(); newWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory(); FileSystem.get(uri, cfg).setWorkingDirectory(new Path("/folder" + curThreadNum)); changeAbsDirPhase.countDown(); changeAbsDirPhase.await(); newAbsWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory(); newInstanceWorkDir[curThreadNum] = FileSystem.newInstance(uri, cfg).getWorkingDirectory(); finishPhase.countDown(); } catch (InterruptedException | IOException e) { error("Failed to execute test thread.", e); fail(); } } }, THREAD_COUNT, "filesystems-test"); finishPhase.await(); for (int i = 0; i < THREAD_COUNT; i++) { cfg.set(MRJobConfig.USER_NAME, "user" + i); Path workDir = new Path(new Path(uri), "user/user" + i); cfg.set(GridHadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, workDir.toString()); assertEquals(workDir, FileSystem.newInstance(uri, cfg).getWorkingDirectory()); assertEquals(workDir, newUserInitWorkDir[i]); assertEquals(new Path(new Path(uri), "user/user" + i + "/folder" + i), newWorkDir[i]); assertEquals(new Path("/folder" + i), newAbsWorkDir[i]); assertEquals(new Path(new Path(uri), "user/" + System.getProperty("user.name")), newInstanceWorkDir[i]); } System.out.println(System.getProperty("user.dir")); }
From source file:org.apache.ignite.internal.processors.hadoop.HadoopFileSystemsTest.java
License:Apache License
/** * Test the file system with specified URI for the multi-thread working directory support. * * @param uri Base URI of the file system (scheme and authority). * @throws Exception If fails./*from www . jav a 2 s .c om*/ */ private void testFileSystem(final URI uri) throws Exception { final Configuration cfg = new Configuration(); setupFileSystems(cfg); cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString()); final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT); final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT); final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT); final CountDownLatch finishPhase = new CountDownLatch(THREAD_COUNT); final Path[] newUserInitWorkDir = new Path[THREAD_COUNT]; final Path[] newWorkDir = new Path[THREAD_COUNT]; final Path[] newAbsWorkDir = new Path[THREAD_COUNT]; final Path[] newInstanceWorkDir = new Path[THREAD_COUNT]; final AtomicInteger threadNum = new AtomicInteger(0); GridTestUtils.runMultiThreadedAsync(new Runnable() { @Override public void run() { try { int curThreadNum = threadNum.getAndIncrement(); FileSystem fs = FileSystem.get(uri, cfg); HadoopFileSystemsUtils.setUser(fs, "user" + curThreadNum); if ("file".equals(uri.getScheme())) FileSystem.get(uri, cfg).setWorkingDirectory(new Path("file:///user/user" + curThreadNum)); changeUserPhase.countDown(); changeUserPhase.await(); newUserInitWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory(); FileSystem.get(uri, cfg).setWorkingDirectory(new Path("folder" + curThreadNum)); changeDirPhase.countDown(); changeDirPhase.await(); newWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory(); FileSystem.get(uri, cfg).setWorkingDirectory(new Path("/folder" + curThreadNum)); changeAbsDirPhase.countDown(); changeAbsDirPhase.await(); newAbsWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory(); newInstanceWorkDir[curThreadNum] = FileSystem.newInstance(uri, cfg).getWorkingDirectory(); finishPhase.countDown(); } catch (InterruptedException | IOException e) { error("Failed to execute test thread.", e); fail(); } } }, THREAD_COUNT, "filesystems-test"); finishPhase.await(); for (int i = 0; i < THREAD_COUNT; i++) { cfg.set(MRJobConfig.USER_NAME, "user" + i); Path workDir = new Path(new Path(uri), "user/user" + i); cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, workDir.toString()); assertEquals(workDir, FileSystem.newInstance(uri, cfg).getWorkingDirectory()); assertEquals(workDir, newUserInitWorkDir[i]); assertEquals(new Path(new Path(uri), "user/user" + i + "/folder" + i), newWorkDir[i]); assertEquals(new Path("/folder" + i), newAbsWorkDir[i]); assertEquals(new Path(new Path(uri), "user/" + System.getProperty("user.name")), newInstanceWorkDir[i]); } System.out.println(System.getProperty("user.dir")); }