Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.cloudera.science.avro2parquet.CombinedAvroKeyInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
@Override/*from  w  ww.j  a va  2  s  . c o m*/
public RecordReader<AvroKey<T>, NullWritable> createRecordReader(InputSplit inputSplit,
        TaskAttemptContext context) throws IOException {
    Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration());
    if (null == readerSchema) {
        LOG.warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired.");
        LOG.info("Using a reader schema equal to the writer schema.");
    }

    Object c = CombinedAvroKeyRecordReader.class;
    return new CombineFileRecordReader<AvroKey<T>, NullWritable>((CombineFileSplit) inputSplit, context,
            (Class<? extends RecordReader<AvroKey<T>, NullWritable>>) c);
}

From source file:com.cloudera.sqoop.mapreduce.AvroOutputFormat.java

License:Apache License

@Override
public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {

    Schema schema = AvroJob.getMapOutputSchema(context.getConfiguration());

    final DataFileWriter<T> WRITER = new DataFileWriter<T>(new GenericDatumWriter<T>());

    Path path = getDefaultWorkFile(context, org.apache.avro.mapred.AvroOutputFormat.EXT);
    WRITER.create(schema, path.getFileSystem(context.getConfiguration()).create(path));

    return new RecordWriter<AvroWrapper<T>, NullWritable>() {
        @Override/*from   w w w  .  jav a 2 s .com*/
        public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException {
            WRITER.append(wrapper.datum());
        }

        @Override
        public void close(TaskAttemptContext context) throws IOException, InterruptedException {
            WRITER.close();
        }
    };
}

From source file:com.cloudera.sqoop.mapreduce.db.DBInputFormat.java

License:Apache License

@Override
/** {@inheritDoc} */
public RecordReader<LongWritable, T> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException, InterruptedException {

    return createDBRecordReader((DBInputSplit) split, context.getConfiguration());
}

From source file:com.cloudera.sqoop.mapreduce.db.DBOutputFormat.java

License:Apache License

@Override
/** {@inheritDoc} */
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    DBConfiguration dbConf = new DBConfiguration(context.getConfiguration());
    String tableName = dbConf.getOutputTableName();
    String[] fieldNames = dbConf.getOutputFieldNames();

    if (fieldNames == null) {
        fieldNames = new String[dbConf.getOutputFieldCount()];
    }/*w w w.  ja v a2 s .c  o  m*/

    try {
        Connection connection = dbConf.getConnection();
        PreparedStatement statement = null;

        statement = connection.prepareStatement(constructQuery(tableName, fieldNames));
        return new DBRecordWriter(connection, statement);
    } catch (Exception ex) {
        throw new IOException(ex.getMessage());
    }
}

From source file:com.cloudera.sqoop.mapreduce.RawKeyTextOutputFormat.java

License:Apache License

public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    boolean isCompressed = getCompressOutput(context);
    Configuration conf = context.getConfiguration();
    String ext = "";
    CompressionCodec codec = null;// ww  w.j a v a2s  .c  om

    if (isCompressed) {
        // create the named codec
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(context, GzipCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, conf);

        ext = codec.getDefaultExtension();
    }

    Path file = getDefaultWorkFile(context, ext);
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    DataOutputStream ostream = fileOut;

    if (isCompressed) {
        ostream = new DataOutputStream(codec.createOutputStream(fileOut));
    }

    return new RawKeyRecordWriter<K, V>(ostream);
}

From source file:com.conductor.kafka.hadoop.KafkaRecordReader.java

License:Apache License

/**
 * {@inheritDoc}//from   w  w w .ja  v  a 2s .c om
 */
@Override
public void initialize(final InputSplit split, final TaskAttemptContext context)
        throws IOException, InterruptedException {
    if (!(split instanceof KafkaInputSplit)) {
        throw new IllegalArgumentException(
                "Expected an InputSplit of type KafkaInputSplit but got " + split.getClass());
    }

    final KafkaInputSplit inputSplit = (KafkaInputSplit) split;
    this.conf = context.getConfiguration();
    this.split = inputSplit;
    this.start = inputSplit.getStartOffset();
    this.pos = inputSplit.getStartOffset();
    this.currentOffset = inputSplit.getStartOffset();
    this.end = inputSplit.getEndOffset();
    this.fetchSize = KafkaInputFormat.getKafkaFetchSizeBytes(conf);
    this.consumer = getConsumer(inputSplit, conf);
}

From source file:com.conductor.s3.S3TextInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    final String delimiter = context.getConfiguration().get("textinputformat.record.delimiter");
    return new LineRecordReader(delimiter != null ? delimiter.getBytes() : null);
}

From source file:com.conversantmedia.mapreduce.output.BloomFilterOutputFormat.java

License:Apache License

public BloomFilterOutputFormat(TaskAttemptContext context, int size) {
    this.context = context;
    this.insertionSize = size;
    context.getConfiguration().set(BASE_OUTPUT_NAME, BLOOM_FILTER);
}

From source file:com.conversantmedia.mapreduce.output.BloomFilterOutputFormat.java

License:Apache License

public BloomFilterOutputFormat(TaskAttemptContext context, int size, String fileNamePrefix) {
    this.context = context;
    this.insertionSize = size;
    context.getConfiguration().set(BASE_OUTPUT_NAME, fileNamePrefix);
}

From source file:com.conversantmedia.mapreduce.output.BloomFilterOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    if (writer == null) {
        int size = getExpectedInsertions(job);
        checkState(size > 0, "Expected insertion insertionSize not set.");

        Configuration conf = job.getConfiguration();
        String extension = "";
        Path file = getDefaultWorkFile(job, extension);
        FileSystem fs = file.getFileSystem(conf);
        FSDataOutputStream fileOut = fs.create(file, false);

        writer = new BloomFilterRecordWriter<>(fileOut, size);
    }/* www  . j a  va 2  s. c  o m*/
    return writer;
}