Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.blackberry.logdriver.mapreduce.avro.AvroBlockInputFormat.java

License:Apache License

/**
 * Creates a new AvroBlockRecordReader.//from w  ww. java 2 s. c om
 * 
 * Increases there default value mapreduce.job.max.split.locations to 100000,
 * if it's not already set.
 * 
 * Also sets mapred.max.split.size to the default block size for the root
 * directory ("/"), if it's not already set.
 * 
 * @param split
 *          The InputSplit.
 * @param context
 *          The TaskAttemptContext.
 * @return A new AvroBlockRecordReader.
 * @throws IOException
 *           If there is an I/O error.
 */
@SuppressWarnings("deprecation")
@Override
public RecordReader<AvroFileHeader, BytesWritable> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();

    // Ensure we have sensible defaults for how we build blocks.
    if (conf.get("mapreduce.job.max.split.locations") == null) {
        conf.setLong("mapreduce.job.max.split.locations", MAX_SPLIT_LOCATIONS);
    }
    if (conf.get("mapred.max.split.size") == null) {
        // Try to set the split size to the default block size. In case of
        // failure, we'll use this 128MB default.
        long blockSize = 128 * 1024 * 1024; // 128MB
        try {
            blockSize = FileSystem.get(conf).getDefaultBlockSize();
        } catch (IOException e) {
            LOG.error("Error getting filesystem to get get default block size (this does not bode well).");
        }
        conf.setLong("mapred.max.split.size", blockSize);
    }

    return new AvroBlockRecordReader();
}

From source file:com.blackberry.logdriver.mapreduce.avro.AvroBlockRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    this.split = (CombineFileSplit) split;
    fs = FileSystem.get(context.getConfiguration());
}

From source file:com.blackberry.logdriver.mapreduce.BinaryRecordWriter.java

License:Apache License

/**
 * Create a writer for the given BinaryOutputFormat and TaskAttemptContext.
 * /* www  .j a va 2s  .com*/
 * @param outputFormat
 * @param context
 */
public BinaryRecordWriter(BinaryOutputFormat outputFormat, TaskAttemptContext context) {
    Configuration conf = context.getConfiguration();
    String extension = conf.get("output.file.extension", "");

    try {
        Path outputPath = outputFormat.getDefaultWorkFile(context, extension);
        FileSystem fs = FileSystem.get(conf);
        LOG.info("Creating output path: {}", outputPath);
        out = fs.create(outputPath, true);
    } catch (IOException e) {
        LOG.error("Error creating output file.", e);
    }
}

From source file:com.blackberry.logdriver.mapreduce.boom.BoomIndividualRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit fileSplit = (FileSplit) split;

    LOG.info("Initializing {}:{}+{}",
            new Object[] { fileSplit.getPath(), fileSplit.getStart(), fileSplit.getLength() });

    // Check for zero length files
    if (fileSplit.getPath().getFileSystem(context.getConfiguration()).getFileStatus(fileSplit.getPath())
            .getLen() == 0) {//w  ww .j  a  va2 s  . c om
        reader = null;
        return;
    }

    GenericDatumReader<Record> datumReader = new GenericDatumReader<Record>(Schemas.getSchema("logBlock"));
    reader = new DataFileReader<Record>(new FsInput(fileSplit.getPath(), context.getConfiguration()),
            datumReader);
    datumReader.setExpected(Schemas.getSchema("logBlock"));
    datumReader.setSchema(reader.getSchema());

    long size = fileSplit.getLength();
    start = fileSplit.getStart();
    end = start + size;

    reader.sync(start);
}

From source file:com.blackberry.logdriver.mapreduce.boom.PigReBoomRecordWriter.java

License:Apache License

public PigReBoomRecordWriter(PigReBoomOutputFormat boomOutputFormat, TaskAttemptContext context)
        throws IOException {
    path = boomOutputFormat.getDefaultWorkFile(context, ".bm");
    out = path.getFileSystem(context.getConfiguration()).create(path);
    writer = new ReBoomWriter(out);
}

From source file:com.blackberry.logdriver.mapreduce.gzip.GzipLineRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    key = new LongWritable();
    value = new Text();

    Path path = ((FileSplit) split).getPath();

    LOG.info("Starting to process file {}", path);
    in = new BufferedReader(
            new InputStreamReader(new GZIPInputStream(FileSystem.get(context.getConfiguration()).open(path))));
}

From source file:com.bonc.mr_roamRecognition_hjpt.comm.FileCountTextOutputFormat.java

License:Apache License

public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = conf.get(SEPERATOR, "\t");
    CompressionCodec codec = null;/* ww w  .  j  a  va  2  s.  c  o m*/
    String extension = "";
    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    if (!isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
    } else {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)),
                keyValueSeparator);
    }
}

From source file:com.bonc.mr_roamRecognition_hjpt.comm.GetFileTextInputFormat.java

License:Apache License

@Override
public RecordReader<Text, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    String delimiter = context.getConfiguration().get("textinputformat.record.delimiter");
    byte[] recordDelimiterBytes = null;
    if (null != delimiter)
        recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
    return new PathRecordReader(recordDelimiterBytes);
}

From source file:com.bonc.mr_roamRecognition_hjpt.comm.PathRecordReader.java

License:Apache License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE);
    start = split.getStart();//from   w  ww.  j  av  a  2s.c  om
    end = start + split.getLength();
    final Path file = split.getPath();

    path = split.getPath().toString();

    // open the file and seek to the start of the split
    final FileSystem fs = file.getFileSystem(job);
    fileIn = fs.open(file);

    CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
    if (null != codec) {
        isCompressedInput = true;
        decompressor = CodecPool.getDecompressor(codec);
        if (codec instanceof SplittableCompressionCodec) {
            final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream(
                    fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK);
            in = new CompressedSplitLineReader(cIn, job, this.recordDelimiterBytes);
            start = cIn.getAdjustedStart();
            end = cIn.getAdjustedEnd();
            filePosition = cIn;
        } else {
            in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job,
                    this.recordDelimiterBytes);
            filePosition = fileIn;
        }
    } else {
        fileIn.seek(start);
        in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes);
        filePosition = fileIn;
    }
    // If this is not the first split, we always throw away first record
    // because we always (except the last split) read one extra line in
    // next() method.
    if (start != 0) {
        start += in.readLine(new Text(), 0, maxBytesToConsume(start));
    }
    this.pos = start;
}

From source file:com.buzzinate.dm.cassandra.ColumnFamilyRecordWriter.java

License:Apache License

/**
 * Upon construction, obtain the map that this writer will use to collect
 * mutations, and the ring cache for the given keyspace.
 *
 * @param context the task attempt context
 * @throws IOException/* w w  w.j av a 2s.co  m*/
 */
ColumnFamilyRecordWriter(TaskAttemptContext context) throws IOException {
    this(context.getConfiguration());
    this.progressable = new Progressable(context);
}