Example usage for org.apache.hadoop.mapreduce TaskAttemptID forName

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptID forName

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptID
  6. forName

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptID forName.

Prototype

public static TaskAttemptID forName(String str) throws IllegalArgumentException

Source Link

Document

Construct a TaskAttemptID object from given string

Usage

From source file:com.tuplejump.calliope.hadoop.cql3.CqlInputFormat.java

License:Apache License

public RecordReader<Long, Row> getRecordReader(org.apache.hadoop.mapred.InputSplit split, JobConf jobConf,
        final Reporter reporter) throws IOException {
    TaskAttemptContext tac = HadoopCompat.newMapContext(jobConf,
            TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)), null, null, null, new ReporterWrapper(reporter),
            null);//from w w  w .jav a  2 s .c o  m

    CqlRecordReader recordReader = new CqlRecordReader();
    recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac);
    return recordReader;
}

From source file:com.tuplejump.calliope.hadoop.cql3.CqlPagingInputFormat.java

License:Apache License

public RecordReader<Map<String, ByteBuffer>, Map<String, ByteBuffer>> getRecordReader(InputSplit split,
        JobConf jobConf, final Reporter reporter) throws IOException {
    TaskAttemptContext tac = HadoopCompat.newMapContext(jobConf,
            TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)), null, null, null, new ReporterWrapper(reporter),
            null);/*from   w  ww  .  j  a v a  2s  .co  m*/

    CqlPagingRecordReader recordReader = new CqlPagingRecordReader();
    recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac);
    return recordReader;
}

From source file:eu.stratosphere.hadoopcompatibility.mapreduce.HadoopOutputFormat.java

License:Apache License

/**
 * create the temporary output file for hadoop RecordWriter.
 * @param taskNumber The number of the parallel instance.
 * @param numTasks The number of parallel tasks.
 * @throws IOException/*  w w w. j  a v  a  2 s.c o m*/
 */
@Override
public void open(int taskNumber, int numTasks) throws IOException {
    if (Integer.toString(taskNumber + 1).length() > 6) {
        throw new IOException("Task id too large.");
    }

    // for hadoop 2.2
    this.configuration.set("mapreduce.output.basename", "tmp");

    TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_"
            + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0")
            + Integer.toString(taskNumber + 1) + "_0");

    try {
        this.context = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    this.configuration.set("mapred.task.id", taskAttemptID.toString());
    // for hadoop 2.2
    this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString());

    this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")),
            context);

    try {
        this.fileOutputCommitter.setupJob(HadoopUtils.instantiateJobContext(this.configuration, new JobID()));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    // compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
    this.configuration.set("mapreduce.task.output.dir", this.fileOutputCommitter.getWorkPath().toString());

    try {
        this.recordWriter = this.mapreduceOutputFormat.getRecordWriter(this.context);
    } catch (InterruptedException e) {
        throw new IOException("Could not create RecordWriter.", e);
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.TaskLogAppender.java

License:Apache License

@Override
public void activateOptions() {
    synchronized (this) {
        setOptionsFromSystemProperties();

        if (maxEvents > 0) {
            tail = new LinkedList<LoggingEvent>();
        }/*from  ww  w .j  a  va  2 s  . c o  m*/
        setFile(TaskLog.getTaskLogFile(TaskAttemptID.forName(taskId), isCleanup, TaskLog.LogName.SYSLOG)
                .toString());
        setAppend(true);
        super.activateOptions();
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.TestPipeApplication.java

License:Apache License

/**
 * clean previous std error and outs//from  ww  w.j  ava  2s .  co  m
 */

private void initStdOut(Configuration configuration) {
    TaskAttemptID taskId = TaskAttemptID.forName(configuration.get(MRJobConfig.TASK_ATTEMPT_ID));
    File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT);
    File stdErr = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDERR);
    // prepare folder
    if (!stdOut.getParentFile().exists()) {
        stdOut.getParentFile().mkdirs();
    } else { // clean logs
        stdOut.deleteOnExit();
        stdErr.deleteOnExit();
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.TestPipeApplication.java

License:Apache License

private String readStdOut(Configuration conf) throws Exception {
    TaskAttemptID taskId = TaskAttemptID.forName(conf.get(MRJobConfig.TASK_ATTEMPT_ID));
    File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT);

    return readFile(stdOut);

}

From source file:org.apache.avro.mapreduce.TestAvroKeyOutputFormat.java

License:Apache License

/**
 * Tests that the record writer is constructed and returned correctly from the output format.
 *///from  ww  w.jav a2  s. co m
private void testGetRecordWriter(Configuration conf, CodecFactory expectedCodec) throws IOException {
    // Configure a mock task attempt context.
    Job job = new Job(conf);
    job.getConfiguration().set("mapred.output.dir", mTempDir.getRoot().getPath());
    Schema writerSchema = Schema.create(Schema.Type.INT);
    AvroJob.setOutputKeySchema(job, writerSchema);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration()).andReturn(job.getConfiguration()).anyTimes();
    expect(context.getTaskAttemptID()).andReturn(TaskAttemptID.forName("attempt_200707121733_0001_m_000000_0"))
            .anyTimes();
    expect(context.getNumReduceTasks()).andReturn(1);

    // Create a mock record writer.
    @SuppressWarnings("unchecked")
    RecordWriter<AvroKey<Integer>, NullWritable> expectedRecordWriter = createMock(RecordWriter.class);
    AvroKeyOutputFormat.RecordWriterFactory recordWriterFactory = createMock(
            AvroKeyOutputFormat.RecordWriterFactory.class);

    // Expect the record writer factory to be called with appropriate parameters.
    Capture<CodecFactory> capturedCodecFactory = new Capture<CodecFactory>();
    expect(recordWriterFactory.create(eq(writerSchema), anyObject(GenericData.class),
            capture(capturedCodecFactory), // Capture for comparison later.
            anyObject(OutputStream.class))).andReturn(expectedRecordWriter);

    replay(context);
    replay(expectedRecordWriter);
    replay(recordWriterFactory);

    AvroKeyOutputFormat<Integer> outputFormat = new AvroKeyOutputFormat<Integer>(recordWriterFactory);
    RecordWriter<AvroKey<Integer>, NullWritable> recordWriter = outputFormat.getRecordWriter(context);
    // Make sure the expected codec was used.
    assertTrue(capturedCodecFactory.hasCaptured());
    assertEquals(expectedCodec.toString(), capturedCodecFactory.getValue().toString());

    verify(context);
    verify(expectedRecordWriter);
    verify(recordWriterFactory);

    assertNotNull(recordWriter);
    assertTrue(expectedRecordWriter == recordWriter);
}

From source file:org.apache.blur.mapreduce.lib.BlurOutputCommitter.java

License:Apache License

private void commitOrAbortJob(JobContext jobContext, Path shardPath, boolean commit) throws IOException {
    LOG.info("CommitOrAbort [{0}] path [{1}]", commit, shardPath);
    FileSystem fileSystem = shardPath.getFileSystem(jobContext.getConfiguration());
    FileStatus[] listStatus = fileSystem.listStatus(shardPath, new PathFilter() {
        @Override/*  ww w. j av a2s. com*/
        public boolean accept(Path path) {
            LOG.info("Checking path [{0}]", path);
            if (path.getName().endsWith(".task_complete")) {
                return true;
            }
            return false;
        }
    });
    for (FileStatus fileStatus : listStatus) {
        Path path = fileStatus.getPath();
        LOG.info("Trying to commitOrAbort [{0}]", path);
        String name = path.getName();
        boolean taskComplete = name.endsWith(".task_complete");
        if (fileStatus.isDir()) {
            String taskAttemptName = getTaskAttemptName(name);
            if (taskAttemptName == null) {
                LOG.info("Dir name [{0}] not task attempt", name);
                continue;
            }
            TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptName);
            if (taskAttemptID.getJobID().equals(jobContext.getJobID())) {
                if (commit) {
                    if (taskComplete) {
                        fileSystem.rename(path, new Path(shardPath, taskAttemptName + ".commit"));
                        LOG.info("Committing [{0}] in path [{1}]", taskAttemptID, path);
                    } else {
                        fileSystem.delete(path, true);
                        LOG.info("Deleting tmp dir [{0}] in path [{1}]", taskAttemptID, path);
                    }
                } else {
                    fileSystem.delete(path, true);
                    LOG.info("Deleting aborted job dir [{0}] in path [{1}]", taskAttemptID, path);
                }
            } else {
                LOG.warn("TaskAttempt JobID [{0}] does not match JobContext JobId [{1}]",
                        taskAttemptID.getJobID(), jobContext.getJobID());
            }
        }
    }
}

From source file:org.apache.cassandra.hadoop.ColumnFamilyInputFormat.java

License:Apache License

public org.apache.hadoop.mapred.RecordReader<ByteBuffer, SortedMap<ByteBuffer, ColumnFamilyRecordReader.Column>> getRecordReader(
        org.apache.hadoop.mapred.InputSplit split, JobConf jobConf, final Reporter reporter)
        throws IOException {
    TaskAttemptContext tac = HadoopCompat.newMapContext(jobConf,
            TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)), null, null, null, new ReporterWrapper(reporter),
            null);/*ww  w  . ja  v a  2 s .c om*/

    ColumnFamilyRecordReader recordReader = new ColumnFamilyRecordReader(
            jobConf.getInt(CASSANDRA_HADOOP_MAX_KEY_SIZE, CASSANDRA_HADOOP_MAX_KEY_SIZE_DEFAULT));
    recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac);
    return recordReader;
}

From source file:org.apache.cassandra.hadoop.cql3.CqlInputFormat.java

License:Apache License

public RecordReader<Long, Row> getRecordReader(InputSplit split, JobConf jobConf, final Reporter reporter)
        throws IOException {
    TaskAttemptContext tac = new TaskAttemptContext(jobConf,
            TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID))) {
        @Override/*from   www. j  a  v a 2  s. c o m*/
        public void progress() {
            reporter.progress();
        }
    };

    CqlRecordReader recordReader = new CqlRecordReader();
    recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac);
    return recordReader;
}