Example usage for org.apache.hadoop.mapred JobConf getUser

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf getUser.

Prototype

public String getUser() 

Source Link

Document

Get the reported username for this job.

Usage

From source file:com.mellanox.hadoop.mapred.UdaPluginTT.java

License:Apache License

static IndexRecordBridge getPathIndex(String jobId, String mapId, int reduce) {
    String userName = null;/*from  www. j  a  v  a2 s .c  om*/
    String runAsUserName = null;
    IndexRecordBridge data = null;

    try {
        JobConf jobConf = udaShuffleProvider.getJobConfFromSuperClass(JobID.forName(jobId));
        userName = jobConf.getUser();
        runAsUserName = taskTracker.getTaskController().getRunAsUser(jobConf);

        String intermediateOutputDir = UdaShuffleProviderPlugin.getIntermediateOutputDirFromSuperClass(userName,
                jobId, mapId);

        String indexKey = intermediateOutputDir + "/file.out.index";
        Path indexFileName = fileIndexCache.get(indexKey);
        if (indexFileName == null) {
            indexFileName = localDirAllocator.getLocalPathToRead(indexKey, mjobConf);
            fileIndexCache.put(indexKey, indexFileName);
        }
        // Map-output file
        String fileKey = intermediateOutputDir + "/file.out";
        Path mapOutputFileName = fileCache.get(fileKey);
        if (mapOutputFileName == null) {
            mapOutputFileName = localDirAllocator.getLocalPathToRead(fileKey, mjobConf);
            fileCache.put(fileKey, mapOutputFileName);
        }

        //  Read the index file to get the information about where
        //  the map-output for the given reducer is available. 

        data = indexCache.getIndexInformationBridge(mapId, reduce, indexFileName, runAsUserName);
        data.pathMOF = mapOutputFileName.toString();

    } catch (IOException e) {
        LOG.error("exception caught" + e.toString()); //to check how C behaves in case there is an exception
    }
    return data;

}

---

From source file:it.crs4.pydoop.pipes.Application.java

License:Apache License

/**
 * Start the child process to handle the task for us.
 * @param conf the task's configuration/*from   w w w. j a  va2  s.  c  o m*/
 * @param recordReader the fake record reader to update progress with
 * @param output the collector to send output to
 * @param reporter the reporter for the task
 * @param outputKeyClass the class of the output keys
 * @param outputValueClass the class of the output values
 * @throws IOException
 * @throws InterruptedException
 */
Application(JobConf conf, RecordReader<FloatWritable, NullWritable> recordReader,
        OutputCollector<K2, V2> output, Reporter reporter, Class<? extends K2> outputKeyClass,
        Class<? extends V2> outputValueClass) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    // add TMPDIR environment variable with the value of java.io.tmpdir
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put(Submitter.PORT, Integer.toString(serverSocket.getLocalPort()));

    TaskAttemptID taskid = TaskAttemptID.forName(conf.get(MRJobConfig.TASK_ATTEMPT_ID));

    // get the task's working directory
    String workDir = LocalJobRunner.getLocalTaskDir(conf.getUser(), taskid.getJobID().toString(),
            taskid.getTaskID().toString(), false);

    //Add token to the environment if security is enabled
    Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(conf.getCredentials());
    // This password is used as shared secret key between this application and
    // child pipes process
    byte[] password = jobToken.getPassword();

    String localPasswordFile = new File(workDir, "jobTokenPassword").getAbsolutePath();
    writePasswordToLocalFile(localPasswordFile, password, conf);
    env.put("hadoop.pipes.shared.secret.location", localPasswordFile);

    List<String> cmd = new ArrayList<String>();
    String interpretor = conf.get(Submitter.INTERPRETOR);
    if (interpretor != null) {
        cmd.add(interpretor);
    }
    String executable = DistributedCache.getLocalCacheFiles(conf)[0].toString();
    if (!(new File(executable).canExecute())) {
        // LinuxTaskController sets +x permissions on all distcache files already.
        // In case of DefaultTaskController, set permissions here.
        FileUtil.chmod(executable, "u+x");
    }
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    // we are starting map/reduce task of the pipes job. this is not a cleanup
    // attempt. 
    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
    cmd = TaskLog.captureOutAndError(null, cmd, stdout, stderr, logLength, false);

    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();

    String challenge = getSecurityChallenge();
    String digestToSend = createDigest(password, challenge);
    String digestExpected = createDigest(password, digestToSend);

    handler = new OutputHandler<K2, V2>(output, reporter, recordReader, digestExpected);
    K2 outputKey = (K2) ReflectionUtils.newInstance(outputKeyClass, conf);
    V2 outputValue = (V2) ReflectionUtils.newInstance(outputValueClass, conf);
    downlink = new BinaryProtocol<K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue, conf);

    downlink.authenticate(digestToSend, challenge);
    waitForAuthentication();
    LOG.debug("Authentication succeeded");
    downlink.start();
    downlink.setJobConf(conf);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.java

License:Apache License

/**
 * Creates JobInfo from hadoop configuration.
 *
 * @param cfg Hadoop configuration.//from   w ww  . j  av a 2s.c om
 * @return Job info.
 * @throws IgniteCheckedException If failed.
 */
public static GridHadoopDefaultJobInfo createJobInfo(Configuration cfg) throws IgniteCheckedException {
    JobConf jobConf = new JobConf(cfg);

    boolean hasCombiner = jobConf.get("mapred.combiner.class") != null
            || jobConf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null;

    int numReduces = jobConf.getNumReduceTasks();

    jobConf.setBooleanIfUnset("mapred.mapper.new-api", jobConf.get(OLD_MAP_CLASS_ATTR) == null);

    if (jobConf.getUseNewMapper()) {
        String mode = "new map API";

        ensureNotSet(jobConf, "mapred.input.format.class", mode);
        ensureNotSet(jobConf, OLD_MAP_CLASS_ATTR, mode);

        if (numReduces != 0)
            ensureNotSet(jobConf, "mapred.partitioner.class", mode);
        else
            ensureNotSet(jobConf, "mapred.output.format.class", mode);
    } else {
        String mode = "map compatibility";

        ensureNotSet(jobConf, MRJobConfig.INPUT_FORMAT_CLASS_ATTR, mode);
        ensureNotSet(jobConf, MRJobConfig.MAP_CLASS_ATTR, mode);

        if (numReduces != 0)
            ensureNotSet(jobConf, MRJobConfig.PARTITIONER_CLASS_ATTR, mode);
        else
            ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
    }

    if (numReduces != 0) {
        jobConf.setBooleanIfUnset("mapred.reducer.new-api", jobConf.get(OLD_REDUCE_CLASS_ATTR) == null);

        if (jobConf.getUseNewReducer()) {
            String mode = "new reduce API";

            ensureNotSet(jobConf, "mapred.output.format.class", mode);
            ensureNotSet(jobConf, OLD_REDUCE_CLASS_ATTR, mode);
        } else {
            String mode = "reduce compatibility";

            ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
            ensureNotSet(jobConf, MRJobConfig.REDUCE_CLASS_ATTR, mode);
        }
    }

    Map<String, String> props = new HashMap<>();

    for (Map.Entry<String, String> entry : jobConf)
        props.put(entry.getKey(), entry.getValue());

    return new GridHadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces,
            props);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.HadoopUtils.java

License:Apache License

/**
 * Creates JobInfo from hadoop configuration.
 *
 * @param cfg Hadoop configuration.//from   w ww. j a  v  a2  s  .  c om
 * @return Job info.
 * @throws IgniteCheckedException If failed.
 */
public static HadoopDefaultJobInfo createJobInfo(Configuration cfg) throws IgniteCheckedException {
    JobConf jobConf = new JobConf(cfg);

    boolean hasCombiner = jobConf.get("mapred.combiner.class") != null
            || jobConf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null;

    int numReduces = jobConf.getNumReduceTasks();

    jobConf.setBooleanIfUnset("mapred.mapper.new-api", jobConf.get(OLD_MAP_CLASS_ATTR) == null);

    if (jobConf.getUseNewMapper()) {
        String mode = "new map API";

        ensureNotSet(jobConf, "mapred.input.format.class", mode);
        ensureNotSet(jobConf, OLD_MAP_CLASS_ATTR, mode);

        if (numReduces != 0)
            ensureNotSet(jobConf, "mapred.partitioner.class", mode);
        else
            ensureNotSet(jobConf, "mapred.output.format.class", mode);
    } else {
        String mode = "map compatibility";

        ensureNotSet(jobConf, MRJobConfig.INPUT_FORMAT_CLASS_ATTR, mode);
        ensureNotSet(jobConf, MRJobConfig.MAP_CLASS_ATTR, mode);

        if (numReduces != 0)
            ensureNotSet(jobConf, MRJobConfig.PARTITIONER_CLASS_ATTR, mode);
        else
            ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
    }

    if (numReduces != 0) {
        jobConf.setBooleanIfUnset("mapred.reducer.new-api", jobConf.get(OLD_REDUCE_CLASS_ATTR) == null);

        if (jobConf.getUseNewReducer()) {
            String mode = "new reduce API";

            ensureNotSet(jobConf, "mapred.output.format.class", mode);
            ensureNotSet(jobConf, OLD_REDUCE_CLASS_ATTR, mode);
        } else {
            String mode = "reduce compatibility";

            ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
            ensureNotSet(jobConf, MRJobConfig.REDUCE_CLASS_ATTR, mode);
        }
    }

    Map<String, String> props = new HashMap<>();

    for (Map.Entry<String, String> entry : jobConf)
        props.put(entry.getKey(), entry.getValue());

    return new HadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces, props);
}

---

From source file:org.apache.oozie.example.SampleOozieActionConfigurator.java

License:Apache License

@Override
public void configure(JobConf actionConf) throws OozieActionConfiguratorException {
    if (actionConf.getUser() == null) {
        throw new OozieActionConfiguratorException("No user set");
    }/* ww w  .  ja v  a2s.  co m*/
    if (actionConf.get("examples.root") == null) {
        throw new OozieActionConfiguratorException("examples.root not set");
    }
    if (actionConf.get("output.dir.name") == null) {
        throw new OozieActionConfiguratorException("output.dir.name not set");
    }

    actionConf.setMapperClass(SampleMapper.class);
    actionConf.setReducerClass(SampleReducer.class);
    actionConf.setNumMapTasks(1);
    FileInputFormat.setInputPaths(actionConf, new Path(
            "/user/" + actionConf.getUser() + "/" + actionConf.get("examples.root") + "/input-data/text"));
    FileOutputFormat.setOutputPath(actionConf, new Path("/user/" + actionConf.getUser() + "/"
            + actionConf.get("examples.root") + "/output-data/" + actionConf.get("output.dir.name")));
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper.java

License:Apache License

public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
    lastConf = job;/* ww w .  j ava2s . c om*/
    DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
    store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
    wrapped.init(store);

    // Mimic org.apache.hadoop.mapred.FileSplit if feasible...
    String[] locations = wrapped.getLocations();
    if (locations.length > 0) {
        job.set("map.input.file", locations[0]);
        job.setLong("map.input.start", wrapped.getStart());
        job.setLong("map.input.length", wrapped.getLength());
    }

    return new RecordReader<Text, Tuple>() {

        public void close() throws IOException {
            wrapped.close();
        }

        public Text createKey() {
            return new Text();
        }

        public Tuple createValue() {
            return new Tuple();
        }

        public long getPos() throws IOException {
            return wrapped.getPos();
        }

        public float getProgress() throws IOException {
            return wrapped.getProgress();
        }

        public boolean next(Text key, Tuple value) throws IOException {
            return wrapped.next(value);
        }
    };
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper.java

License:Apache License

public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
    lastConf = job;/*from  w w  w  .ja  va2 s  .  co m*/
    DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
    // if the execution is against Mapred DFS, set
    // working dir to /user/<userid>
    if (pigContext.getExecType() == ExecType.MAPREDUCE)
        store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
    wrapped.init(store);

    job.set("map.target.ops", ObjectSerializer.serialize(targetOps));
    // Mimic org.apache.hadoop.mapred.FileSplit if feasible...
    String[] locations = wrapped.getLocations();
    if (locations.length > 0) {
        job.set("map.input.file", locations[0]);
        job.setLong("map.input.start", wrapped.getStart());
        job.setLong("map.input.length", wrapped.getLength());
    }

    return new RecordReader<Text, Tuple>() {

        TupleFactory tupFac = TupleFactory.getInstance();

        public void close() throws IOException {
            wrapped.close();
        }

        public Text createKey() {
            return null; // we never use the key!
        }

        public Tuple createValue() {
            return tupFac.newTuple();
        }

        public long getPos() throws IOException {
            return wrapped.getPos();
        }

        public float getProgress() throws IOException {
            return wrapped.getProgress();
        }

        public boolean next(Text key, Tuple value) throws IOException {
            return wrapped.next(value);
        }
    };
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.GridHadoopUtils.java

License:Open Source License

/**
 * Creates JobInfo from hadoop configuration.
 *
 * @param cfg Hadoop configuration./* w  w w . j  a v a2s .  com*/
 * @return Job info.
 * @throws GridException If failed.
 */
public static GridHadoopDefaultJobInfo createJobInfo(Configuration cfg) throws GridException {
    JobConf jobConf = new JobConf(cfg);

    boolean hasCombiner = jobConf.get("mapred.combiner.class") != null
            || jobConf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null;

    int numReduces = jobConf.getNumReduceTasks();

    jobConf.setBooleanIfUnset("mapred.mapper.new-api", jobConf.get(OLD_MAP_CLASS_ATTR) == null);

    if (jobConf.getUseNewMapper()) {
        String mode = "new map API";

        ensureNotSet(jobConf, "mapred.input.format.class", mode);
        ensureNotSet(jobConf, OLD_MAP_CLASS_ATTR, mode);

        if (numReduces != 0)
            ensureNotSet(jobConf, "mapred.partitioner.class", mode);
        else
            ensureNotSet(jobConf, "mapred.output.format.class", mode);
    } else {
        String mode = "map compatibility";

        ensureNotSet(jobConf, MRJobConfig.INPUT_FORMAT_CLASS_ATTR, mode);
        ensureNotSet(jobConf, MRJobConfig.MAP_CLASS_ATTR, mode);

        if (numReduces != 0)
            ensureNotSet(jobConf, MRJobConfig.PARTITIONER_CLASS_ATTR, mode);
        else
            ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
    }

    if (numReduces != 0) {
        jobConf.setBooleanIfUnset("mapred.reducer.new-api", jobConf.get(OLD_REDUCE_CLASS_ATTR) == null);

        if (jobConf.getUseNewReducer()) {
            String mode = "new reduce API";

            ensureNotSet(jobConf, "mapred.output.format.class", mode);
            ensureNotSet(jobConf, OLD_REDUCE_CLASS_ATTR, mode);
        } else {
            String mode = "reduce compatibility";

            ensureNotSet(jobConf, MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, mode);
            ensureNotSet(jobConf, MRJobConfig.REDUCE_CLASS_ATTR, mode);
        }
    }

    Map<String, String> props = new HashMap<>();

    for (Map.Entry<String, String> entry : jobConf)
        props.put(entry.getKey(), entry.getValue());

    return new GridHadoopDefaultJobInfo(jobConf.getJobName(), jobConf.getUser(), hasCombiner, numReduces,
            props);
}

---