Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getNumReduceTasks

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getNumReduceTasks

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptContext
  6. getNumReduceTasks

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getNumReduceTasks.

Prototype

public int getNumReduceTasks();

Source Link

Document

Get configured the number of reduce tasks for this job.

Usage

From source file:com.bigfishgames.biginsights.upsight.mapreduce.MyAvroKeyOutputFormat.java

License:Apache License

/** {@inheritDoc} */
@Override/*from w ww .j  av  a 2  s  . c  o m*/
@SuppressWarnings("unchecked")
public RecordWriter<AvroKey<T>, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    // Get the writer schema.
    Schema writerSchema = AvroJob.getOutputKeySchema(conf);
    boolean isMapOnly = context.getNumReduceTasks() == 0;
    if (isMapOnly) {
        Schema mapOutputSchema = AvroJob.getMapOutputKeySchema(conf);
        if (mapOutputSchema != null) {
            writerSchema = mapOutputSchema;
        }
    }
    if (null == writerSchema) {
        throw new IOException(
                "AvroKeyOutputFormat requires an output schema. Use AvroJob.setOutputKeySchema().");
    }

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return mRecordWriterFactory.create(writerSchema, dataModel, getCompressionCodec(context),
            getAvroFileOutputStream(context), getSyncInterval(context));
}

From source file:org.apache.avro.mapreduce.AvroKeyOutputFormat.java

License:Apache License

/** {@inheritDoc} */
@Override// w  w w  .  ja  va 2s . c om
@SuppressWarnings("unchecked")
public RecordWriter<AvroKey<T>, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    // Get the writer schema.
    Schema writerSchema = AvroJob.getOutputKeySchema(conf);
    boolean isMapOnly = context.getNumReduceTasks() == 0;
    if (isMapOnly) {
        Schema mapOutputSchema = AvroJob.getMapOutputKeySchema(conf);
        if (mapOutputSchema != null) {
            writerSchema = mapOutputSchema;
        }
    }
    if (null == writerSchema) {
        throw new IOException(
                "AvroKeyOutputFormat requires an output schema. Use AvroJob.setOutputKeySchema().");
    }

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return mRecordWriterFactory.create(writerSchema, dataModel, getCompressionCodec(context),
            getAvroFileOutputStream(context));
}

From source file:org.apache.avro.mapreduce.lib.output.AvroOutputFormat.java

License:Apache License

@Override
public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(TaskAttemptContext job)
        throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    boolean isMapOnly = job.getNumReduceTasks() == 0;
    Schema schema = isMapOnly ? AvroJob.getMapOutputSchema(conf) : AvroJob.getOutputSchema(conf);

    final DataFileWriter<T> writer = new DataFileWriter<T>(new ReflectDatumWriter<T>());

    configureDataFileWriter(writer, job);
    Path path = getDefaultWorkFile(job, EXT);
    writer.create(schema, path.getFileSystem(job.getConfiguration()).create(path));

    return new RecordWriter<AvroWrapper<T>, NullWritable>() {
        @Override//  w w w  . ja  v a2s . c  om
        public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException {
            writer.append(wrapper.datum());
        }

        @Override
        public void close(TaskAttemptContext context) throws IOException {
            writer.close();
        }
    };
}

From source file:org.apache.avro.mapreduce.TestAvroKeyOutputFormat.java

License:Apache License

/**
 * Tests that the record writer is constructed and returned correctly from the output format.
 *//*w w  w  . java2  s  .  co  m*/
private void testGetRecordWriter(Configuration conf, CodecFactory expectedCodec) throws IOException {
    // Configure a mock task attempt context.
    Job job = new Job(conf);
    job.getConfiguration().set("mapred.output.dir", mTempDir.getRoot().getPath());
    Schema writerSchema = Schema.create(Schema.Type.INT);
    AvroJob.setOutputKeySchema(job, writerSchema);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);
    expect(context.getConfiguration()).andReturn(job.getConfiguration()).anyTimes();
    expect(context.getTaskAttemptID()).andReturn(TaskAttemptID.forName("attempt_200707121733_0001_m_000000_0"))
            .anyTimes();
    expect(context.getNumReduceTasks()).andReturn(1);

    // Create a mock record writer.
    @SuppressWarnings("unchecked")
    RecordWriter<AvroKey<Integer>, NullWritable> expectedRecordWriter = createMock(RecordWriter.class);
    AvroKeyOutputFormat.RecordWriterFactory recordWriterFactory = createMock(
            AvroKeyOutputFormat.RecordWriterFactory.class);

    // Expect the record writer factory to be called with appropriate parameters.
    Capture<CodecFactory> capturedCodecFactory = new Capture<CodecFactory>();
    expect(recordWriterFactory.create(eq(writerSchema), anyObject(GenericData.class),
            capture(capturedCodecFactory), // Capture for comparison later.
            anyObject(OutputStream.class))).andReturn(expectedRecordWriter);

    replay(context);
    replay(expectedRecordWriter);
    replay(recordWriterFactory);

    AvroKeyOutputFormat<Integer> outputFormat = new AvroKeyOutputFormat<Integer>(recordWriterFactory);
    RecordWriter<AvroKey<Integer>, NullWritable> recordWriter = outputFormat.getRecordWriter(context);
    // Make sure the expected codec was used.
    assertTrue(capturedCodecFactory.hasCaptured());
    assertEquals(expectedCodec.toString(), capturedCodecFactory.getValue().toString());

    verify(context);
    verify(expectedRecordWriter);
    verify(recordWriterFactory);

    assertNotNull(recordWriter);
    assertTrue(expectedRecordWriter == recordWriter);
}

From source file:org.apache.blur.mapreduce.lib.BlurOutputCommitter.java

License:Apache License

@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
    int numReduceTasks = context.getNumReduceTasks();
    TaskAttemptID taskAttemptID = context.getTaskAttemptID();
    return taskAttemptID.isMap() && numReduceTasks != 0 ? false : true;
}

From source file:org.apache.crunch.io.avro.trevni.TrevniRecordWriter.java

License:Apache License

/** {@inheritDoc} */
protected Schema initSchema(TaskAttemptContext context) {
    boolean isMapOnly = context.getNumReduceTasks() == 0;
    return isMapOnly ? AvroJob.getMapOutputKeySchema(context.getConfiguration())
            : AvroJob.getOutputKeySchema(context.getConfiguration());
}

From source file:org.apache.sqoop.mapreduce.AvroOutputFormat.java

License:Apache License

@Override
public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {

    boolean isMapOnly = context.getNumReduceTasks() == 0;
    Schema schema = isMapOnly ? AvroJob.getMapOutputSchema(context.getConfiguration())
            : AvroJob.getOutputSchema(context.getConfiguration());

    final DataFileWriter<T> WRITER = new DataFileWriter<T>(new ReflectDatumWriter<T>());

    configureDataFileWriter(WRITER, context);

    Path path = getDefaultWorkFile(context, EXT);
    WRITER.create(schema, path.getFileSystem(context.getConfiguration()).create(path));

    return new RecordWriter<AvroWrapper<T>, NullWritable>() {
        @Override/*  w ww  .j  a  v a  2 s  . c  o  m*/
        public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException {
            WRITER.append(wrapper.datum());
        }

        @Override
        public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
            WRITER.close();
        }
    };
}

From source file:org.apache.trevni.avro.mapreduce.AvroTrevniKeyRecordWriter.java

License:Apache License

/** {@inheritDoc} */
@Override//  w w w.  ja  v  a  2s  .c  o  m
protected Schema initSchema(TaskAttemptContext context) {
    boolean isMapOnly = context.getNumReduceTasks() == 0;
    return isMapOnly ? AvroJob.getMapOutputKeySchema(context.getConfiguration())
            : AvroJob.getOutputKeySchema(context.getConfiguration());
}

From source file:org.schedoscope.export.ftp.outputformat.FtpUploadOutputCommitter.java

License:Apache License

/**
 * The constructor to initialize the output committer.
 *
 * @param outputPath The HDFS location of the src file to upload.
 * @param context    The TaskAttemptContext,
 * @throws IOException Is thrown if an error occurs.
 *///from w  w w  . j a v a 2  s  .  com
public FtpUploadOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {

    super(outputPath, context);

    Configuration conf = context.getConfiguration();

    this.outputPath = outputPath;
    this.endpoint = conf.get(FtpUploadOutputFormat.FTP_EXPORT_ENDPOINT);
    this.filePrefix = conf.get(FtpUploadOutputFormat.FTP_EXPORT_FILE_PREFIX);

    this.user = conf.get(FtpUploadOutputFormat.FTP_EXPORT_USER);
    this.pass = conf.get(FtpUploadOutputFormat.FTP_EXPORT_PASS);
    this.keyContent = conf.get(FtpUploadOutputFormat.FTP_EXPORT_KEY_FILE_CONTENT);
    this.passiveMode = conf.getBoolean(FtpUploadOutputFormat.FTP_EXPORT_PASSIVE_MODE, true);
    this.userIsRoot = conf.getBoolean(FtpUploadOutputFormat.FTP_EXPORT_USER_IS_ROOT, true);
    this.cleanHdfsDir = conf.getBoolean(FtpUploadOutputFormat.FTP_EXPORT_CLEAN_HDFS_DIR, true);
    this.numReducer = context.getNumReduceTasks();

    try {

        String protocol = new URI(endpoint).getScheme();

        if (!protocol.equals("ftp") && !protocol.equals("sftp")) {
            throw new IllegalArgumentException("protocol not supported, must be either 'ftp' or 'sftp'");
        }

    } catch (URISyntaxException e) {
        throw new IllegalArgumentException(e);
    }
}