Example usage for org.apache.hadoop.mapreduce MRJobConfig USER

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce MRJobConfig USER_NAME.

Prototype

String USER_NAME

To view the source code for org.apache.hadoop.mapreduce MRJobConfig USER_NAME.

Click Source Link

Usage

From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopFileSystemCounterWriterDelegateImpl.java

License:Apache License

/** {@inheritDoc} */
public void write(HadoopJobEx job, HadoopCounters cntrs) throws IgniteCheckedException {
    Configuration hadoopCfg = HadoopUtils.safeCreateConfiguration();

    final HadoopJobInfo jobInfo = job.info();

    final HadoopJobId jobId = job.id();

    for (Map.Entry<String, String> e : ((HadoopDefaultJobInfo) jobInfo).properties().entrySet())
        hadoopCfg.set(e.getKey(), e.getValue());

    String user = jobInfo.user();

    user = IgfsUtils.fixUserName(user);/*from   ww  w.  j a  v  a  2 s.c o  m*/

    String dir = jobInfo.property(IgniteHadoopFileSystemCounterWriter.COUNTER_WRITER_DIR_PROPERTY);

    if (dir == null)
        dir = DEFAULT_COUNTER_WRITER_DIR;

    Path jobStatPath = new Path(new Path(dir.replace(USER_MACRO, user)), jobId.toString());

    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);

    try {
        hadoopCfg.set(MRJobConfig.USER_NAME, user);

        FileSystem fs = ((HadoopV2Job) job).fileSystem(jobStatPath.toUri(), hadoopCfg);

        fs.mkdirs(jobStatPath);

        try (PrintStream out = new PrintStream(fs.create(
                new Path(jobStatPath, IgniteHadoopFileSystemCounterWriter.PERFORMANCE_COUNTER_FILE_NAME)))) {
            for (T2<String, Long> evt : perfCntr.evts()) {
                out.print(evt.get1());
                out.print(':');
                out.println(evt.get2().toString());
            }

            out.flush();
        }
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.HadoopFileSystemsTest.java

License:Apache License

/**
 * Test the file system with specified URI for the multi-thread working directory support.
 *
 * @param uri Base URI of the file system (scheme and authority).
 * @throws Exception If fails.//from  w ww  .j a  va2  s .c om
 */
private void testFileSystem(final URI uri) throws Exception {
    final Configuration cfg = new Configuration();

    setupFileSystems(cfg);

    cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP,
            new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString());

    FileSystem fs = FileSystem.get(uri, cfg);

    assertTrue(fs instanceof HadoopLocalFileSystemV1);

    final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT);
    final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT);
    final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT);
    final CountDownLatch finishPhase = new CountDownLatch(THREAD_COUNT);

    final Path[] newUserInitWorkDir = new Path[THREAD_COUNT];
    final Path[] newWorkDir = new Path[THREAD_COUNT];
    final Path[] newAbsWorkDir = new Path[THREAD_COUNT];
    final Path[] newInstanceWorkDir = new Path[THREAD_COUNT];

    final AtomicInteger threadNum = new AtomicInteger(0);

    GridTestUtils.runMultiThreadedAsync(new Runnable() {
        @Override
        public void run() {
            try {
                int curThreadNum = threadNum.getAndIncrement();

                if ("file".equals(uri.getScheme()))
                    FileSystem.get(uri, cfg).setWorkingDirectory(new Path("file:///user/user" + curThreadNum));

                changeUserPhase.countDown();
                changeUserPhase.await();

                newUserInitWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();

                FileSystem.get(uri, cfg).setWorkingDirectory(new Path("folder" + curThreadNum));

                changeDirPhase.countDown();
                changeDirPhase.await();

                newWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();

                FileSystem.get(uri, cfg).setWorkingDirectory(new Path("/folder" + curThreadNum));

                changeAbsDirPhase.countDown();
                changeAbsDirPhase.await();

                newAbsWorkDir[curThreadNum] = FileSystem.get(uri, cfg).getWorkingDirectory();

                newInstanceWorkDir[curThreadNum] = FileSystem.newInstance(uri, cfg).getWorkingDirectory();

                finishPhase.countDown();
            } catch (InterruptedException | IOException e) {
                error("Failed to execute test thread.", e);

                fail();
            }
        }
    }, THREAD_COUNT, "filesystems-test");

    finishPhase.await();

    for (int i = 0; i < THREAD_COUNT; i++) {
        cfg.set(MRJobConfig.USER_NAME, "user" + i);

        Path workDir = new Path(new Path(uri), "user/user" + i);

        cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, workDir.toString());

        assertEquals(workDir, FileSystem.newInstance(uri, cfg).getWorkingDirectory());

        assertEquals(workDir, newUserInitWorkDir[i]);

        assertEquals(new Path(new Path(uri), "user/user" + i + "/folder" + i), newWorkDir[i]);

        assertEquals(new Path("/folder" + i), newAbsWorkDir[i]);

        assertEquals(new Path(new Path(uri), "user/" + System.getProperty("user.name")), newInstanceWorkDir[i]);
    }

    System.out.println(System.getProperty("user.dir"));
}

From source file:org.apache.oozie.util.TestConfigUtils.java

License:Apache License

public void testCheckAndSetNonConflictingUserNamesNoChange() throws Exception {
    checkAndSetNonConflictingNoChange(OozieClient.USER_NAME);
    checkAndSetNonConflictingNoChange(MRJobConfig.USER_NAME);
}

From source file:org.apache.oozie.util.TestConfigUtils.java

License:Apache License

public void testCheckAndSetConflictingUserNameSets() throws Exception {
    checkAndSetConflictingSets(OozieClient.USER_NAME);
    checkAndSetConflictingSets(MRJobConfig.USER_NAME);
}

From source file:org.apache.oozie.util.TestConfigUtils.java

License:Apache License

protected void checkAndSetConflictingSets(final String key) throws Exception {
    final Configuration base = new Configuration();
    base.set(key, getTestUser());//from  www.ja v a 2s. c  o  m

    ConfigurationService.setBoolean("oozie.configuration.check-and-set." + OozieClient.USER_NAME, true);
    ConfigurationService.setBoolean("oozie.configuration.check-and-set." + MRJobConfig.USER_NAME, true);

    ConfigUtils.checkAndSetDisallowedProperties(base, getTestUser2(), new Exception(), false);

    assertEquals("user.name should be preserved as no write will be performed", getTestUser(), base.get(key));

    ConfigUtils.checkAndSetDisallowedProperties(base, getTestUser2(), new Exception(), true);

    assertEquals("user.name should be set as one write operation will be performed", getTestUser2(),
            base.get(key));
}

From source file:org.apache.oozie.util.TestConfigUtils.java

License:Apache License

public void testCheckAndSetConflictingUserNameThrows() {
    checkAndSetConflictingThrows(OozieClient.USER_NAME);
    checkAndSetConflictingThrows(MRJobConfig.USER_NAME);
}

From source file:org.apache.oozie.util.TestConfigUtils.java

License:Apache License

public void testCheckAndSetConflictingUserNamesSetsAndThrows() {
    final Configuration base = new Configuration();
    base.set(OozieClient.USER_NAME, getTestUser());
    base.set(MRJobConfig.USER_NAME, getTestUser());

    ConfigurationService.setBoolean("oozie.configuration.check-and-set." + OozieClient.USER_NAME, true);
    ConfigurationService.setBoolean("oozie.configuration.check-and-set." + MRJobConfig.USER_NAME, false);

    try {/*from  w  ww.  j a  v a 2 s. com*/
        ConfigUtils.checkAndSetDisallowedProperties(base, getTestUser2(), new Exception("test message"), false);
        fail("Exception should have been thrown");
    } catch (final Exception e) {
        assertTrue("message mismatch", e.getMessage().contains("test message"));
    }

    assertEquals("user.name should be preserved as no write will be performed", getTestUser(),
            base.get(OozieClient.USER_NAME));

    try {
        ConfigUtils.checkAndSetDisallowedProperties(base, getTestUser2(), new Exception("test message"), true);
    } catch (final Exception e) {
        fail("Exception should not have been thrown");
    }

    assertEquals("user.name should be set as one write operation will be performed", getTestUser2(),
            base.get(OozieClient.USER_NAME));

    assertEquals("mapreduce.job.user.name should be set implicitly by Configuration#set(user.name)",
            getTestUser2(), base.get(MRJobConfig.USER_NAME));
}

From source file:org.gridgain.grid.ggfs.hadoop.v1.GridGgfsHadoopFileSystem.java

License:Open Source License

/** {@inheritDoc} */
@Override// ww  w.  ja  v a2  s  . co  m
public void initialize(URI name, Configuration cfg) throws IOException {
    enterBusy();

    try {
        if (rmtClient != null)
            throw new IOException("File system is already initialized: " + rmtClient);

        A.notNull(name, "name");
        A.notNull(cfg, "cfg");

        super.initialize(name, cfg);

        setConf(cfg);

        String disableCacheName = String.format("fs.%s.impl.disable.cache", name.getScheme());

        cacheEnabled = !cfg.getBoolean(disableCacheName, false);

        mgmt = cfg.getBoolean(GGFS_MANAGEMENT, false);

        if (!GGFS_SCHEME.equals(name.getScheme()))
            throw new IOException("Illegal file system URI [expected=" + GGFS_SCHEME
                    + "://[name]/[optional_path], actual=" + name + ']');

        uri = name;

        uriAuthority = uri.getAuthority();

        setUser(cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));

        // Override sequential reads before prefetch if needed.
        seqReadsBeforePrefetch = parameter(cfg, PARAM_GGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);

        if (seqReadsBeforePrefetch > 0)
            seqReadsBeforePrefetchOverride = true;

        // In GG replication factor is controlled by data cache affinity.
        // We use replication factor to force the whole file to be stored on local node.
        dfltReplication = (short) cfg.getInt("dfs.replication", 3);

        // Get file colocation control flag.
        colocateFileWrites = parameter(cfg, PARAM_GGFS_COLOCATED_WRITES, uriAuthority, false);
        preferLocFileWrites = cfg.getBoolean(PARAM_GGFS_PREFER_LOCAL_WRITES, false);

        // Get log directory.
        String logDirCfg = parameter(cfg, PARAM_GGFS_LOG_DIR, uriAuthority, DFLT_GGFS_LOG_DIR);

        File logDirFile = U.resolveGridGainPath(logDirCfg);

        String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;

        rmtClient = new GridGgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);

        // Handshake.
        GridGgfsHandshakeResponse handshake = rmtClient.handshake(logDir);

        ggfsGrpBlockSize = handshake.blockSize();

        GridGgfsPaths paths = handshake.secondaryPaths();

        // Initialize client logger.
        Boolean logEnabled = parameter(cfg, PARAM_GGFS_LOG_ENABLED, uriAuthority, false);

        if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
            // Initiate client logger.
            if (logDir == null)
                throw new IOException("Failed to resolve log directory: " + logDirCfg);

            Integer batchSize = parameter(cfg, PARAM_GGFS_LOG_BATCH_SIZE, uriAuthority,
                    DFLT_GGFS_LOG_BATCH_SIZE);

            clientLog = GridGgfsLogger.logger(uriAuthority, handshake.ggfsName(), logDir, batchSize);
        } else
            clientLog = GridGgfsLogger.disabledLogger();

        modeRslvr = new GridGgfsModeResolver(paths.defaultMode(), paths.pathModes());

        boolean initSecondary = paths.defaultMode() == PROXY;

        if (paths.pathModes() != null && !paths.pathModes().isEmpty()) {
            for (T2<GridGgfsPath, GridGgfsMode> pathMode : paths.pathModes()) {
                GridGgfsMode mode = pathMode.getValue();

                initSecondary |= mode == PROXY;
            }
        }

        if (initSecondary) {
            Map<String, String> props = paths.properties();

            String secUri = props.get(GridGgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
            String secConfPath = props.get(GridGgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            try {
                secondaryUri = new URI(secUri);

                URL secondaryCfgUrl = U.resolveGridGainUrl(secConfPath);

                Configuration conf = new Configuration();

                if (secondaryCfgUrl != null)
                    conf.addResource(secondaryCfgUrl);

                String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());

                conf.setBoolean(prop, true);

                secondaryFs = FileSystem.get(secondaryUri, conf);
            } catch (URISyntaxException ignore) {
                if (!mgmt)
                    throw new IOException("Failed to resolve secondary file system URI: " + secUri);
                else
                    LOG.warn(
                            "Visor failed to create secondary file system (operations on paths with PROXY mode "
                                    + "will have no effect).");
            } catch (IOException e) {
                if (!mgmt)
                    throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
                else
                    LOG.warn(
                            "Visor failed to create secondary file system (operations on paths with PROXY mode "
                                    + "will have no effect): " + e.getMessage());
            }
        }
    } finally {
        leaveBusy();
    }
}

From source file:org.gridgain.grid.ggfs.hadoop.v2.GridGgfsHadoopFileSystem.java

License:Open Source License

/**
 * @param name URI for file system.//  w w  w  . j a  v  a2  s.  c  o m
 * @param cfg Configuration.
 * @throws URISyntaxException if name has invalid syntax.
 * @throws IOException If initialization failed.
 */
public GridGgfsHadoopFileSystem(URI name, Configuration cfg) throws URISyntaxException, IOException {
    super(GridGgfsHadoopEndpoint.normalize(name), GGFS_SCHEME, false, -1);

    uri = name;

    try {
        initialize(name, cfg);
    } catch (IOException e) {
        // Close client if exception occurred.
        if (rmtClient != null)
            rmtClient.close(false);

        throw e;
    }

    workingDir = new GridGgfsPath("/user/" + cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));
}

Example usage for org.apache.hadoop.mapreduce MRJobConfig USER_NAME

Introduction

Prototype

Usage