Example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser.

Prototype

@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation getCurrentUser() throws IOException 

Source Link

Document

Return the current user, including any doAs in the current stack.

Usage

From source file:org.apache.tez.mapreduce.output.MROutput.java

License:Apache License

@Override
public List<Event> initialize() throws IOException, InterruptedException {
    LOG.info("Initializing Simple Output");
    getContext().requestInitialMemory(0l, null); //mandatory call
    taskNumberFormat.setMinimumIntegerDigits(5);
    taskNumberFormat.setGroupingUsed(false);
    nonTaskNumberFormat.setMinimumIntegerDigits(3);
    nonTaskNumberFormat.setGroupingUsed(false);
    Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
    this.jobConf = new JobConf(conf);
    // Add tokens to the jobConf - in case they are accessed within the RW / OF
    jobConf.getCredentials().mergeAll(UserGroupInformation.getCurrentUser().getCredentials());
    this.useNewApi = this.jobConf.getUseNewMapper();
    this.isMapperOutput = jobConf.getBoolean(MRConfig.IS_MAP_PROCESSOR, false);
    jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, getContext().getDAGAttemptNumber());
    TaskAttemptID taskAttemptId = org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl
            .createMockTaskAttemptID(getContext().getApplicationId().getClusterTimestamp(),
                    getContext().getTaskVertexIndex(), getContext().getApplicationId().getId(),
                    getContext().getTaskIndex(), getContext().getTaskAttemptNumber(), isMapperOutput);
    jobConf.set(JobContext.TASK_ATTEMPT_ID, taskAttemptId.toString());
    jobConf.set(JobContext.TASK_ID, taskAttemptId.getTaskID().toString());
    jobConf.setBoolean(JobContext.TASK_ISMAP, isMapperOutput);
    jobConf.setInt(JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId());
    jobConf.set(JobContext.ID, taskAttemptId.getJobID().toString());

    if (useNewApi) {
        // set the output part name to have a unique prefix
        if (jobConf.get("mapreduce.output.basename") == null) {
            jobConf.set("mapreduce.output.basename", getOutputFileNamePrefix());
        }//from w  w w .  j ava  2  s.co m
    }

    outputRecordCounter = getContext().getCounters().findCounter(TaskCounter.OUTPUT_RECORDS);

    if (useNewApi) {
        newApiTaskAttemptContext = createTaskAttemptContext(taskAttemptId);
        try {
            newOutputFormat = org.apache.hadoop.util.ReflectionUtils
                    .newInstance(newApiTaskAttemptContext.getOutputFormatClass(), jobConf);
        } catch (ClassNotFoundException cnfe) {
            throw new IOException(cnfe);
        }

        try {
            newRecordWriter = newOutputFormat.getRecordWriter(newApiTaskAttemptContext);
        } catch (InterruptedException e) {
            throw new IOException("Interrupted while creating record writer", e);
        }
    } else {
        oldApiTaskAttemptContext = new org.apache.tez.mapreduce.hadoop.mapred.TaskAttemptContextImpl(jobConf,
                taskAttemptId, new MRTaskReporter(getContext()));
        oldOutputFormat = jobConf.getOutputFormat();

        FileSystem fs = FileSystem.get(jobConf);
        String finalName = getOutputName();

        oldRecordWriter = oldOutputFormat.getRecordWriter(fs, jobConf, finalName,
                new MRReporter(getContext().getCounters()));
    }
    initCommitter(jobConf, useNewApi);

    LOG.info("Initialized Simple Output" + ", using_new_api: " + useNewApi);
    return null;
}

---

From source file:org.apache.tez.mapreduce.processor.MRTask.java

License:Apache License

private void configureMRTask() throws IOException, InterruptedException {

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    jobConf.setCredentials(credentials);
    // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
    // the Tez parameter.
    // TODO This could be fetched from the env if YARN is setting it for all
    // Containers.
    // Set it in conf, so as to be able to be used the the OutputCommitter.

    // Not needed. This is probably being set via the source/consumer meta
    Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
    if (jobToken != null) {
        // Will MR ever run without a job token.
        SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken.getPassword());
        this.jobTokenSecret = sk;
    } else {//from   ww  w .  j  a v a2s.  c  om
        LOG.warn("No job token set");
    }

    configureLocalDirs();

    // Set up the DistributedCache related configs
    setupDistributedCacheConfig(jobConf);
}

---

From source file:org.apache.tez.mapreduce.task.MRRuntimeTask.java

License:Apache License

private static void configureMRTask(JobConf job, MRTask task) throws IOException, InterruptedException {

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    job.setCredentials(credentials);/*from ww w. j  av  a  2 s  .c om*/
    // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
    // the Tez parameter.
    // TODO This could be fetched from the env if YARN is setting it for all
    // Containers.
    // Set it in conf, so as to be able to be used the the OutputCommitter.
    job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, job.getInt(TezJobConfig.APPLICATION_ATTEMPT_ID, -1));

    job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS, YarnOutputFiles.class, MapOutputFile.class); // MR

    Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
    if (jobToken != null) {
        // Will MR ever run without a job token.
        SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken.getPassword());
        task.setJobTokenSecret(sk);
    } else {
        LOG.warn("No job token set");
    }

    job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
    job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
    if (job.get(TezJobConfig.DAG_CREDENTIALS_BINARY) != null) {
        job.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, job.get(TezJobConfig.DAG_CREDENTIALS_BINARY));
    }

    // setup the child's attempt directories
    // Do the task-type specific localization
    task.localizeConfiguration(job);

    // Set up the DistributedCache related configs
    setupDistributedCacheConfig(job);

    task.setConf(job);
}

---

From source file:org.apache.tez.mapreduce.YARNRunner.java

License:Apache License

@Override
public JobStatus getJobStatus(JobID jobID) throws IOException, InterruptedException {
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    String jobFile = MRApps.getJobFile(conf, user, jobID);
    DAGStatus dagStatus;/*  ww w .j ava  2s.c o  m*/
    try {
        if (dagClient == null) {
            dagClient = tezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId());
        }
        dagStatus = dagClient.getDAGStatus();
        return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
    } catch (TezException e) {
        throw new IOException(e);
    }
}

---

From source file:org.apache.tez.runtime.library.common.shuffle.impl.Shuffle.java

License:Apache License

public Shuffle(TezInputContext inputContext, Configuration conf, int numInputs) throws IOException {
    this.inputContext = inputContext;
    this.conf = conf;
    this.metrics = new ShuffleClientMetrics(inputContext.getDAGName(), inputContext.getTaskVertexName(),
            inputContext.getTaskIndex(), this.conf, UserGroupInformation.getCurrentUser().getShortUserName());

    this.numInputs = numInputs;

    this.jobTokenSecret = ShuffleUtils.getJobTokenSecretFromTokenBytes(
            inputContext.getServiceConsumerMetaData(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID));

    if (ConfigUtils.isIntermediateInputCompressed(conf)) {
        Class<? extends CompressionCodec> codecClass = ConfigUtils.getIntermediateInputCompressorClass(conf,
                DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, conf);
    } else {//from   w  ww  . j a va 2s  .com
        codec = null;
    }
    this.ifileReadAhead = conf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD,
            TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT);
    if (this.ifileReadAhead) {
        this.ifileReadAheadLength = conf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES,
                TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT);
    } else {
        this.ifileReadAheadLength = 0;
    }

    Combiner combiner = TezRuntimeUtils.instantiateCombiner(conf, inputContext);

    FileSystem localFS = FileSystem.getLocal(this.conf);
    LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);

    // TODO TEZ Get rid of Map / Reduce references.
    TezCounter shuffledMapsCounter = inputContext.getCounters().findCounter(TaskCounter.SHUFFLED_MAPS);
    TezCounter reduceShuffleBytes = inputContext.getCounters().findCounter(TaskCounter.REDUCE_SHUFFLE_BYTES);
    TezCounter failedShuffleCounter = inputContext.getCounters().findCounter(TaskCounter.FAILED_SHUFFLE);
    TezCounter spilledRecordsCounter = inputContext.getCounters().findCounter(TaskCounter.SPILLED_RECORDS);
    TezCounter reduceCombineInputCounter = inputContext.getCounters()
            .findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
    TezCounter mergedMapOutputsCounter = inputContext.getCounters().findCounter(TaskCounter.MERGED_MAP_OUTPUTS);

    LOG.info("Shuffle assigned with " + numInputs + " inputs" + ", codec: "
            + (codec == null ? "None" : codec.getClass().getName()) + "ifileReadAhead: " + ifileReadAhead);

    scheduler = new ShuffleScheduler(this.inputContext, this.conf, this.numInputs, this, shuffledMapsCounter,
            reduceShuffleBytes, failedShuffleCounter);
    eventHandler = new ShuffleInputEventHandler(inputContext, scheduler);
    merger = new MergeManager(this.conf, localFS, localDirAllocator, inputContext, combiner,
            spilledRecordsCounter, reduceCombineInputCounter, mergedMapOutputsCounter, this);
}

---

From source file:org.apache.tez.runtime.library.common.shuffle.orderedgrouped.Shuffle.java

License:Apache License

public Shuffle(InputContext inputContext, Configuration conf, int numInputs, long initialMemoryAvailable)
        throws IOException {
    this.inputContext = inputContext;
    this.conf = conf;
    this.httpConnectionParams = ShuffleUtils.constructHttpShuffleConnectionParams(conf);
    this.metrics = new ShuffleClientMetrics(inputContext.getDAGName(), inputContext.getTaskVertexName(),
            inputContext.getTaskIndex(), this.conf, UserGroupInformation.getCurrentUser().getShortUserName());

    this.srcNameTrimmed = TezUtilsInternal.cleanVertexName(inputContext.getSourceVertexName());

    this.jobTokenSecret = ShuffleUtils.getJobTokenSecretFromTokenBytes(
            inputContext.getServiceConsumerMetaData(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID));
    this.jobTokenSecretMgr = new JobTokenSecretManager(jobTokenSecret);

    if (ConfigUtils.isIntermediateInputCompressed(conf)) {
        Class<? extends CompressionCodec> codecClass = ConfigUtils.getIntermediateInputCompressorClass(conf,
                DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, conf);
    } else {/*  ww w.  ja va2  s  .co m*/
        codec = null;
    }
    this.ifileReadAhead = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
            TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT);
    if (this.ifileReadAhead) {
        this.ifileReadAheadLength = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES,
                TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT);
    } else {
        this.ifileReadAheadLength = 0;
    }

    Combiner combiner = TezRuntimeUtils.instantiateCombiner(conf, inputContext);

    FileSystem localFS = FileSystem.getLocal(this.conf);
    LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);

    // TODO TEZ Get rid of Map / Reduce references.
    TezCounter shuffledInputsCounter = inputContext.getCounters().findCounter(TaskCounter.NUM_SHUFFLED_INPUTS);
    TezCounter reduceShuffleBytes = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES);
    TezCounter reduceDataSizeDecompressed = inputContext.getCounters()
            .findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED);
    TezCounter failedShuffleCounter = inputContext.getCounters()
            .findCounter(TaskCounter.NUM_FAILED_SHUFFLE_INPUTS);
    TezCounter spilledRecordsCounter = inputContext.getCounters().findCounter(TaskCounter.SPILLED_RECORDS);
    TezCounter reduceCombineInputCounter = inputContext.getCounters()
            .findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
    TezCounter mergedMapOutputsCounter = inputContext.getCounters().findCounter(TaskCounter.MERGED_MAP_OUTPUTS);
    TezCounter bytesShuffedToDisk = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_TO_DISK);
    TezCounter bytesShuffedToDiskDirect = inputContext.getCounters()
            .findCounter(TaskCounter.SHUFFLE_BYTES_DISK_DIRECT);
    TezCounter bytesShuffedToMem = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_TO_MEM);

    LOG.info("Shuffle assigned with " + numInputs + " inputs" + ", codec: "
            + (codec == null ? "None" : codec.getClass().getName()) + "ifileReadAhead: " + ifileReadAhead);

    boolean sslShuffle = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL,
            TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL_DEFAULT);
    startTime = System.currentTimeMillis();
    scheduler = new ShuffleScheduler(this.inputContext, this.conf, numInputs, this, shuffledInputsCounter,
            reduceShuffleBytes, reduceDataSizeDecompressed, failedShuffleCounter, bytesShuffedToDisk,
            bytesShuffedToDiskDirect, bytesShuffedToMem, startTime);
    this.mergePhaseTime = inputContext.getCounters().findCounter(TaskCounter.MERGE_PHASE_TIME);
    this.shufflePhaseTime = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_PHASE_TIME);

    merger = new MergeManager(this.conf, localFS, localDirAllocator, inputContext, combiner,
            spilledRecordsCounter, reduceCombineInputCounter, mergedMapOutputsCounter, this,
            initialMemoryAvailable, codec, ifileReadAhead, ifileReadAheadLength);

    eventHandler = new ShuffleInputEventHandlerOrderedGrouped(inputContext, scheduler, sslShuffle);

    ExecutorService rawExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true)
            .setNameFormat("ShuffleAndMergeRunner [" + srcNameTrimmed + "]").build());

    int configuredNumFetchers = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES,
            TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT);
    numFetchers = Math.min(configuredNumFetchers, numInputs);
    LOG.info("Num fetchers being started: " + numFetchers);
    fetchers = Lists.newArrayListWithCapacity(numFetchers);
    localDiskFetchEnabled = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH,
            TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT);

    executor = MoreExecutors.listeningDecorator(rawExecutor);
    runShuffleCallable = new RunShuffleCallable();
}

---

From source file:org.apache.tez.runtime.task.TestTaskExecution.java

License:Apache License

private TezTaskRunner createTaskRunner(ApplicationId appId, TezTaskUmbilicalForTest umbilical,
        TaskReporter taskReporter, ListeningExecutorService executor, String processorClass,
        byte[] processorConf) throws IOException {
    TezConfiguration tezConf = new TezConfiguration(defaultConf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    Path testDir = new Path(workDir, UUID.randomUUID().toString());
    String[] localDirs = new String[] { testDir.toString() };

    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
            .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
            new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);

    TezTaskRunner taskRunner = new TezTaskRunner(tezConf, ugi, localDirs, taskSpec, umbilical, 1,
            new HashMap<String, ByteBuffer>(), new HashMap<String, String>(),
            HashMultimap.<String, String>create(), taskReporter, executor, null, "",
            new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory());
    return taskRunner;
}

---

From source file:org.apache.tez.runtime.task.TestTaskExecution2.java

License:Apache License

private TezTaskRunner2 createTaskRunner(ApplicationId appId,
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical, TaskReporter taskReporter,
        ListeningExecutorService executor, String processorClass, byte[] processorConf, boolean testRunner,
        boolean updateSysCounters) throws IOException {
    TezConfiguration tezConf = new TezConfiguration(defaultConf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    Path testDir = new Path(workDir, UUID.randomUUID().toString());
    String[] localDirs = new String[] { testDir.toString() };

    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
            .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
            new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);

    TezTaskRunner2 taskRunner;/* w w w .ja va  2 s .co  m*/
    if (testRunner) {
        taskRunner = new TezTaskRunner2ForTest(tezConf, ugi, localDirs, taskSpec, 1,
                new HashMap<String, ByteBuffer>(), new HashMap<String, String>(),
                HashMultimap.<String, String>create(), taskReporter, executor, null, "",
                new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters);
    } else {
        taskRunner = new TezTaskRunner2(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(),
                new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor,
                null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(),
                updateSysCounters, new DefaultHadoopShim());
    }

    return taskRunner;
}

---

From source file:org.apache.tez.runtime.task.TezChild.java

License:Apache License

public static void main(String[] args) throws IOException, InterruptedException, TezException {

    final Configuration defaultConf = new Configuration();

    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    LOG.info("TezChild starting");

    assert args.length == 5;
    String host = args[0];//from   w ww .  j  a va  2s . c om
    int port = Integer.parseInt(args[1]);
    final String containerIdentifier = args[2];
    final String tokenIdentifier = args[3];
    final int attemptNumber = Integer.parseInt(args[4]);
    final String[] localDirs = TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name()));
    final String pid = System.getenv().get("JVM_PID");
    LOG.info("PID, containerIdentifier:  " + pid + ", " + containerIdentifier);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Info from cmd line: AM-host: " + host + " AM-port: " + port + " containerIdentifier: "
                + containerIdentifier + " appAttemptNumber: " + attemptNumber + " tokenIdentifier: "
                + tokenIdentifier);
    }

    // Security framework already loaded the tokens into current ugi
    TezUtilsInternal.addUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()), defaultConf);
    UserGroupInformation.setConfiguration(defaultConf);
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    TezChild tezChild = newTezChild(defaultConf, host, port, containerIdentifier, tokenIdentifier,
            attemptNumber, localDirs, System.getenv(Environment.PWD.name()), System.getenv(), pid,
            new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())), credentials,
            Runtime.getRuntime().maxMemory(), System.getenv(ApplicationConstants.Environment.USER.toString()));
    tezChild.run();
}

---

From source file:org.apache.twill.filesystem.LocalLocationTest.java

License:Apache License

@Override
protected UserGroupInformation createTestUGI() throws IOException {
    // In local location, UGI is not supported, hence using the current user as the testing ugi.
    return UserGroupInformation.getCurrentUser();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• »