Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.awcoleman.BouncyCastleGenericCDRHadoopWithWritable.RawFileRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    path = ((FileSplit) split).getPath();
    FileSystem fs = path.getFileSystem(conf);
    FSDataInputStream fsin = fs.open(path);
    is = decompressStream(fsin);// ww  w.j av  a  2s.  com
    asnin = new ASN1InputStream(is);

    recordCounter = 0;
}

From source file:com.awcoleman.ExampleJobSummaryLogWithOutput.BinRecRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit insplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    FileSplit split = (FileSplit) insplit;

    start = split.getStart();//from  w  ww  . ja v  a2  s  .  c  om
    end = start + split.getLength();
    pos = start;

    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);
    fsin = fs.open(path);
}

From source file:com.basho.riak.hadoop.RiakRecordWriter.java

License:Apache License

RiakRecordWriter(TaskAttemptContext tac) throws RiakException {
    Configuration conf = tac.getConfiguration();
    client = ClientFactory.clusterClient(RiakConfig.getRiakLocatons(conf));
    ns = new Namespace(RiakConfig.getOutputBucket(conf));
}

From source file:com.baynote.hadoop.DelegatingRecordReader.java

License:Apache License

@SuppressWarnings("unchecked")
public DelegatingRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException, InterruptedException {
    // Find the InputFormat and then the RecordReader from the TaggedInputSplit.
    TaggedInputSplit taggedInputSplit = (TaggedInputSplit) split;
    InputFormat<K, V> inputFormat = (InputFormat<K, V>) ReflectionUtils
            .newInstance(taggedInputSplit.getInputFormatClass(), context.getConfiguration());
    originalRR = inputFormat.createRecordReader(taggedInputSplit.getInputSplit(), context);
}

From source file:com.baynote.kafka.hadoop.KafkaRecordReader.java

License:Apache License

/**
 * {@inheritDoc}//from   w w  w .j  av a  2  s . c o m
 */
@Override
public void initialize(final InputSplit split, final TaskAttemptContext context)
        throws IOException, InterruptedException {
    if (!(split instanceof KafkaInputSplit)) {
        throw new IllegalArgumentException(
                "Expected an InputSplit of type KafkaInputSplit but got " + split.getClass());
    }

    final KafkaInputSplit inputSplit = (KafkaInputSplit) split;
    this.conf = context.getConfiguration();
    this.split = inputSplit;
    this.start = inputSplit.getStartOffset();
    this.pos = inputSplit.getStartOffset();
    this.end = inputSplit.getEndOffset();
    this.fetchSize = KafkaInputFormat.getKafkaFetchSizeBytes(conf);
    this.consumer = getConsumer(inputSplit, conf);
}

From source file:com.bigdata.mapreduce.seqtotext.beta.ZipFileRecordReader.java

License:Apache License

/**
 * Initialise and open the ZIP file from the FileSystem
 *///from   w  w  w .ja v a 2s.  com
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    //      Configuration conf = new Configuration();
    //      conf.set("fs.defaultFS", "hdfs://training.musigma.com:8020/user/musigma/");
    FileSplit split = (FileSplit) inputSplit;
    System.out.println("the task attempt instance is : " + taskAttemptContext.getJobName());
    System.out.println("the task attempt instance is : " + taskAttemptContext.getWorkingDirectory().toString());
    Configuration conf = taskAttemptContext.getConfiguration();
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);
    System.out.println("file system replication : " + fs.getDefaultReplication());
    // Open the stream
    fsin = fs.open(path);
    // zip = new ZipInputStream(fsin);
    tar = new TarInputStream(fsin);
    System.out.println("tar input stream is : " + tar.toString());
}

From source file:com.bigfishgames.biginsights.upsight.mapreduce.MyAvroKeyOutputFormat.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w w w  . ja  va2 s. c  o m*/
@SuppressWarnings("unchecked")
public RecordWriter<AvroKey<T>, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    // Get the writer schema.
    Schema writerSchema = AvroJob.getOutputKeySchema(conf);
    boolean isMapOnly = context.getNumReduceTasks() == 0;
    if (isMapOnly) {
        Schema mapOutputSchema = AvroJob.getMapOutputKeySchema(conf);
        if (mapOutputSchema != null) {
            writerSchema = mapOutputSchema;
        }
    }
    if (null == writerSchema) {
        throw new IOException(
                "AvroKeyOutputFormat requires an output schema. Use AvroJob.setOutputKeySchema().");
    }

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return mRecordWriterFactory.create(writerSchema, dataModel, getCompressionCodec(context),
            getAvroFileOutputStream(context), getSyncInterval(context));
}

From source file:com.bigfishgames.biginsights.upsight.mapreduce.MyAvroOutputFormatBase.java

License:Apache License

/**
 * Gets the configured compression codec from the task context.
 *
 * @param context The task attempt context.
 * @return The compression codec to use for the output Avro container file.
 *//*from ww  w .  j a va  2  s.com*/
protected static CodecFactory getCompressionCodec(TaskAttemptContext context) {
    if (FileOutputFormat.getCompressOutput(context)) {
        // Default to deflate compression.
        int deflateLevel = context.getConfiguration().getInt(
                org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY, CodecFactory.DEFAULT_DEFLATE_LEVEL);
        int xzLevel = context.getConfiguration().getInt(org.apache.avro.mapred.AvroOutputFormat.XZ_LEVEL_KEY,
                CodecFactory.DEFAULT_XZ_LEVEL);

        String outputCodec = context.getConfiguration().get(AvroJob.CONF_OUTPUT_CODEC);

        if (outputCodec == null) {
            String compressionCodec = context.getConfiguration().get("mapred.output.compression.codec");
            String avroCodecName = HadoopCodecFactory.getAvroCodecName(compressionCodec);
            if (avroCodecName != null) {
                context.getConfiguration().set(AvroJob.CONF_OUTPUT_CODEC, avroCodecName);
                return HadoopCodecFactory.fromHadoopString(compressionCodec);
            } else {
                return CodecFactory.deflateCodec(deflateLevel);
            }
        } else if (DataFileConstants.DEFLATE_CODEC.equals(outputCodec)) {
            return CodecFactory.deflateCodec(deflateLevel);
        } else if (DataFileConstants.XZ_CODEC.equals(outputCodec)) {
            return CodecFactory.xzCodec(xzLevel);
        } else {
            return CodecFactory.fromString(outputCodec);
        }

    }

    // No compression.
    return CodecFactory.nullCodec();
}

From source file:com.bigfishgames.biginsights.upsight.mapreduce.MyAvroOutputFormatBase.java

License:Apache License

/**
 * Gets the target output stream where the Avro container file should be written.
 *
 * @param context The task attempt context.
 * @return The target output stream.//from w  w w .jav  a 2  s  . co m
 */
protected OutputStream getAvroFileOutputStream(TaskAttemptContext context) throws IOException {
    Path path = new Path(((FileOutputCommitter) getOutputCommitter(context)).getWorkPath(),
            getUniqueFile(context, context.getConfiguration().get("avro.mo.config.namedOutput", "part"),
                    org.apache.avro.mapred.AvroOutputFormat.EXT));
    return path.getFileSystem(context.getConfiguration()).create(path);
}

From source file:com.bigfishgames.biginsights.upsight.mapreduce.MyAvroOutputFormatBase.java

License:Apache License

/**
 * Gets the configured sync interval from the task context.
 *
 * @param context The task attempt context.
 * @return The sync interval to use for the output Avro container file.
 *//*from ww  w.  ja v  a2 s. c  o m*/
protected static int getSyncInterval(TaskAttemptContext context) {
    return context.getConfiguration().getInt(org.apache.avro.mapred.AvroOutputFormat.SYNC_INTERVAL_KEY,
            DataFileConstants.DEFAULT_SYNC_INTERVAL);
}