Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getOutputKeyClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getOutputKeyClass.

Prototype

public Class<?> getOutputKeyClass();

Source Link

Document

Get the key class for the job output data.

Usage

From source file:org.apache.avro.mapreduce.DirectAvroKeyValueOutputFormat.java

License:Apache License

/** {@inheritDoc} */
@Override/* w  w w  . ja  v  a2  s .c  o  m*/
@SuppressWarnings("unchecked")
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();

    AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(conf);

    AvroDatumConverter<K, ?> keyConverter = converterFactory.create((Class<K>) context.getOutputKeyClass());
    AvroDatumConverter<V, ?> valueConverter = converterFactory.create((Class<V>) context.getOutputValueClass());

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter, dataModel,
            getCompressionCodec(context), getAvroFileOutputStream(context), getSyncInterval(context));
}

---

From source file:org.apache.trevni.avro.mapreduce.AvroTrevniKeyValueRecordWriter.java

License:Apache License

/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override/*from  w ww  . j ava  2 s .com*/
protected Schema initSchema(TaskAttemptContext context) {
    AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(context.getConfiguration());

    keyConverter = converterFactory.create((Class<K>) context.getOutputKeyClass());
    valueConverter = converterFactory.create((Class<V>) context.getOutputValueClass());

    // Create the generic record schema for the key/value pair.
    return AvroKeyValue.getSchema(keyConverter.getWriterSchema(), valueConverter.getWriterSchema());

}

---

From source file:org.kiji.avro.mapreduce.AvroKeyValueOutputFormat.java

License:Apache License

/** {@inheritDoc} */
@Override/*from  w  ww  .  ja  v a 2s . c  o m*/
@SuppressWarnings("unchecked")
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(context.getConfiguration());

    AvroDatumConverter<K, ?> keyConverter = converterFactory.create((Class<K>) context.getOutputKeyClass());
    AvroDatumConverter<V, ?> valueConverter = converterFactory.create((Class<V>) context.getOutputValueClass());

    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter, getCompressionCodec(context),
            getAvroFileOutputStream(context));
}

---

From source file:org.mrgeo.hdfs.output.image.HdfsMrsPyramidOutputFormat.java

License:Apache License

protected MapFile.Writer createMapFileWriter(TaskAttemptContext context, CompressionCodec codec,
        SequenceFile.CompressionType compressionType, Path file) throws IOException {
    return new MapFile.Writer(context.getConfiguration(), file,
            MapFile.Writer.keyClass(context.getOutputKeyClass().asSubclass(WritableComparable.class)),
            MapFile.Writer.valueClass(context.getOutputValueClass().asSubclass(Writable.class)),
            MapFile.Writer.compression(compressionType, codec), MapFile.Writer.progressable(context));
}

---

From source file:org.mrgeo.hdfs.output.MapFileOutputFormat.java

License:Apache License

@Override
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context)
        throws IOException {
    Configuration conf = context.getConfiguration();
    CompressionCodec codec = null;/*from  w ww  .  jav a2 s  .  c  o  m*/
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);

        // find the right codec
        Class<?> codecClass = getOutputCompressorClass(context, DefaultCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
    }

    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    // ignore the progress parameter, since MapFile is local
    final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(),
            context.getOutputKeyClass().asSubclass(WritableComparable.class),
            context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context);

    return new RecordWriter<WritableComparable<?>, Writable>() {
        @Override
        public void write(WritableComparable<?> key, Writable value) throws IOException {
            out.append(key, value);
        }

        @Override
        public void close(TaskAttemptContext contxt) throws IOException {
            out.close();
        }
    };
}

---

• «
• 1
• 2
• »