Example usage for org.apache.hadoop.mapreduce TaskInputOutputContext getOutputValueClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskInputOutputContext getOutputValueClass.

Prototype

public Class<?> getOutputValueClass();

Source Link

Document

Get the value class for job outputs.

Usage

From source file:it.crs4.pydoop.mapreduce.pipes.Application.java

License:Apache License

/**
 * Start the child process to handle the task for us.
 * @throws IOException//from  w w  w .  ja va 2  s . c  o  m
 * @throws InterruptedException
 */
Application(TaskInputOutputContext<K1, V1, K2, V2> context, DummyRecordReader input)
        throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    // add TMPDIR environment variable with the value of java.io.tmpdir
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put(Submitter.PORT, Integer.toString(serverSocket.getLocalPort()));

    //Add token to the environment if security is enabled
    Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(context.getCredentials());
    // This password is used as shared secret key between this application and
    // child pipes process
    byte[] password = jobToken.getPassword();
    String localPasswordFile = new File(".") + Path.SEPARATOR + "jobTokenPassword";
    writePasswordToLocalFile(localPasswordFile, password, conf);
    // FIXME why is this not Submitter.SECRET_LOCATION ?
    env.put("hadoop.pipes.shared.secret.location", localPasswordFile);

    List<String> cmd = new ArrayList<String>();
    String interpretor = conf.get(Submitter.INTERPRETOR);
    if (interpretor != null) {
        cmd.add(interpretor);
    }
    String executable = context.getLocalCacheFiles()[0].toString();
    if (!(new File(executable).canExecute())) {
        // LinuxTaskController sets +x permissions on all distcache files already.
        // In case of DefaultTaskController, set permissions here.
        FileUtil.chmod(executable, "u+x");
    }
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    // we are starting map/reduce task of the pipes job. this is not a cleanup
    // attempt. 
    TaskAttemptID taskid = context.getTaskAttemptID();

    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
    cmd = TaskLog.captureOutAndError(null, cmd, stdout, stderr, logLength, false);
    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();

    String challenge = getSecurityChallenge();
    String digestToSend = createDigest(password, challenge);
    String digestExpected = createDigest(password, digestToSend);

    handler = new OutputHandler<K2, V2>(context, input, digestExpected);
    K2 outputKey = (K2) ReflectionUtils.newInstance(context.getOutputKeyClass(), conf);
    V2 outputValue = (V2) ReflectionUtils.newInstance(context.getOutputValueClass(), conf);
    downlink = new BinaryProtocol<K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue, conf);

    downlink.authenticate(digestToSend, challenge);
    waitForAuthentication();
    LOG.debug("Authentication succeeded");
    downlink.start();
    downlink.setJobConf(conf);
}

---