Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:ir.ac.ut.snl.mrcd.MyFileInputFormat.java

@Override
public RecordReader<Text, ShortestPathTuple> createRecordReader(InputSplit is, TaskAttemptContext tac)
        throws IOException, InterruptedException {
    tac.setStatus(is.toString());/*from   w ww  .j a  v a  2  s.  co m*/
    return new MyRecordReader(tac.getConfiguration(), (FileSplit) is);
}

From source file:it.crs4.features.BioImgRecordReader.java

License:Apache License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    globalPlaneIdx = (int) split.getStart();
    nPlanes = (int) split.getLength();
    Path file = split.getPath();/*from   ww w. j ava  2s  .co m*/
    FileSystem fs = file.getFileSystem(context.getConfiguration());
    String absPathName = fs.getFileStatus(file).getPath().toString();
    reader = new ImageReader();
    try {
        reader.setId(absPathName);
    } catch (FormatException e) {
        throw new RuntimeException("FormatException: " + e.getMessage());
    }
    planesPerSeries = reader.getImageCount();
    factory = new BioImgFactory(reader, absPathName);
    name = PathTools.stripext(PathTools.basename(absPathName));
    planeCounter = 0;
}

From source file:it.crs4.pydoop.mapreduce.pipes.PydoopAvroOutputFormatBase.java

License:Apache License

protected static Schema getOutputSchema(TaskAttemptContext context, String propName) throws IOException {
    Properties props = Submitter.getPydoopProperties();
    Configuration conf = context.getConfiguration();
    String schemaJSON = conf.get(props.getProperty(propName));
    if (null == schemaJSON) {
        throw new IOException("Avro output requires an output schema");
    }//from   w  w  w.  j a va 2 s  .c o m
    return Schema.parse(schemaJSON);
}

From source file:it.crs4.pydoop.mapreduce.pipes.PydoopAvroRecordReaderBase.java

License:Apache License

@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    if (!(inputSplit instanceof FileSplit)) {
        throw new IllegalArgumentException("Only compatible with FileSplits.");
    }//from   w  ww. j a  va2  s.com
    FileSplit fileSplit = (FileSplit) inputSplit;
    SeekableInput seekableFileInput = createSeekableInput(context.getConfiguration(), fileSplit.getPath());
    mAvroFileReader = new DataFileReader<GenericRecord>(seekableFileInput,
            new GenericDatumReader<GenericRecord>(mReaderSchema));
    // We will read the first block that begins after the input split
    // start; we will read up to but not including the first block
    // that begins after the input split end.
    mAvroFileReader.sync(fileSplit.getStart());
    mStartPosition = mAvroFileReader.previousSync();
    mEndPosition = fileSplit.getStart() + fileSplit.getLength();
}

From source file:it.crs4.pydoop.NoSeparatorTextOutputFormat.java

License:Apache License

public RecordWriter<Text, Text> getRecordWriter(TaskAttemptContext job)
        throws IOException, InterruptedException {
    final String keyValueSeparator = "";
    Configuration conf = job.getConfiguration();
    boolean isCompressed = getCompressOutput(job);
    CompressionCodec codec = null;// ww  w.j  a va2  s .  c o m
    String extension = "";
    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    if (!isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<Text, Text>(fileOut, keyValueSeparator);
    } else {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<Text, Text>(new DataOutputStream(codec.createOutputStream(fileOut)),
                keyValueSeparator);
    }
}

From source file:it.crs4.seal.demux.DemuxOutputFormat.java

License:Open Source License

public RecordWriter<Text, SequencedFragment> getRecordWriter(TaskAttemptContext job) throws IOException {
    Path defaultFile = getDefaultWorkFile(job, "");
    FileSystem fs = defaultFile.getFileSystem(job.getConfiguration());
    return new DemuxMultiFileLineRecordWriter(job, fs, defaultFile);
}

From source file:it.crs4.seal.tsv_sort.TextValueOutputFormat.java

License:Apache License

public RecordWriter<Text, Text> getRecordWriter(TaskAttemptContext task) throws IOException {
    Configuration conf = task.getConfiguration();
    boolean isCompressed = getCompressOutput(task);

    CompressionCodec codec = null;// www . ja v  a  2 s.c  o m
    String extension = "";

    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(task, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }

    Path file = getDefaultWorkFile(task, extension);
    FileSystem fs = file.getFileSystem(conf);

    DataOutputStream output;

    if (isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        output = new DataOutputStream(codec.createOutputStream(fileOut));
    } else
        output = fs.create(file, false);

    return new ValueRecordWriter(output);
}

From source file:it.crs4.seal.tsv_sort.TsvInputFormat.java

License:Apache License

@Override
public RecordReader<Text, Text> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {
    setConf(context.getConfiguration());
    return new TsvRecordReader(getConf(), keyFields);
}

From source file:it.prz.jmatrw4spark.JMATFileRecordReader.java

License:Open Source License

public void initialize(InputSplit baseSplit, TaskAttemptContext ctx) throws IOException, InterruptedException {
    Configuration cfg = ctx.getConfiguration();

    FileSplit fileSplit = (FileSplit) baseSplit;
    Path filePath = fileSplit.getPath();

    FileSystem fs = filePath.getFileSystem(cfg);
    FSDataInputStream dis = fs.open(fileSplit.getPath());

    //Initialise the block boundaries.
    lBlockStart = fileSplit.getStart();/*  w w  w.j  a v a2 s.co m*/
    lBlockLength = fileSplit.getLength();
    lBlockEnd = lBlockStart + lBlockLength;
    lBlockCurPos = lBlockStart;

    //Initialise the object to read the *.mat file.
    _matReader = new JMATReader(dis);

    //move the file pointer to the start location.
    _matReader.seek(lBlockStart, new Seeker() {
        @Override
        public boolean seekTo(long lBytePos, InputStream is) throws IOException {
            if (is instanceof FSDataInputStream == false)
                throw new UnsupportedSeekOperation("Unknown input stream " + is.getClass().getName());

            ((FSDataInputStream) is).seek(lBytePos);

            return true;
        }
    });
}

From source file:ivory.core.preprocess.PositionalSequenceFileRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit fileSplit = (FileSplit) split;
    conf = context.getConfiguration();
    Path path = fileSplit.getPath();
    FileSystem fs = path.getFileSystem(conf);
    this.in = new SequenceFile.Reader(fs, path, conf);
    this.end = fileSplit.getStart() + fileSplit.getLength();

    if (fileSplit.getStart() > in.getPosition()) {
        in.sync(fileSplit.getStart()); // sync to start
    }//from  w  ww.  jav a 2 s .  c  o  m

    this.start = in.getPosition();
    more = start < end;
}