Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsMapFileOutputFormat.java

License:Apache License

@Override
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context)
        throws IOException {
    Configuration conf = context.getConfiguration();
    CompressionCodec codec = null;//from  ww w .j a  va  2  s  .  c o  m
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = HirodsSequenceFileOutputFormat.getOutputCompressionType(context);

        // find the right codec
        Class<?> codecClass = getOutputCompressorClass(context, DefaultCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
    }

    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    // ignore the progress parameter, since MapFile is local
    final MapFile.Writer out = new MapFile.Writer(conf, fs, file.toString(),
            context.getOutputKeyClass().asSubclass(WritableComparable.class),
            context.getOutputValueClass().asSubclass(Writable.class), compressionType, codec, context);

    return new RecordWriter<WritableComparable<?>, Writable>() {
        public void write(WritableComparable<?> key, Writable value) throws IOException {
            out.append(key, value);
        }

        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsMultipleOutputs.java

License:Apache License

@SuppressWarnings("unchecked")
private synchronized RecordWriter getRecordWriter(TaskAttemptContext taskContext, String baseFileName)
        throws IOException, InterruptedException {

    // look for record-writer in the cache
    RecordWriter writer = recordWriters.get(baseFileName);

    // If not in cache, create a new one
    if (writer == null) {
        // get the record writer from context output format
        HirodsFileOutputFormat.setOutputName(taskContext, baseFileName);
        try {//from   ww  w  . java 2  s  . co m
            writer = ((OutputFormat) ReflectionUtils.newInstance(taskContext.getOutputFormatClass(),
                    taskContext.getConfiguration())).getRecordWriter(taskContext);
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }

        // if counters are enabled, wrap the writer with context 
        // to increment counters 
        if (countersEnabled) {
            writer = new RecordWriterWithCounter(writer, baseFileName, context);
        }

        // add the record-writer to the cache
        recordWriters.put(baseFileName, writer);
    }
    return writer;
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsSequenceFileAsBinaryOutputFormat.java

License:Apache License

protected SequenceFile.Writer getSequenceWriter(TaskAttemptContext context, Class<?> keyClass,
        Class<?> valueClass) throws IOException {
    Configuration conf = context.getConfiguration();

    CompressionCodec codec = null;//from w w w.j a v  a  2  s . com
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = getOutputCompressionType(context);
        // find the right codec
        Class<?> codecClass = getOutputCompressorClass(context, DefaultCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
    }
    // get the path of the temporary output file
    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    return SequenceFile.createWriter(fs, conf, file, keyClass, valueClass, compressionType, codec, context);
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsSequenceFileOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    CompressionCodec codec = null;//from  ww  w .j  a  va 2s  .c  om
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(context)) {
        // find the kind of compression to do
        compressionType = getOutputCompressionType(context);

        // find the right codec
        Class<?> codecClass = getOutputCompressorClass(context, DefaultCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
    }
    // get the path of the temporary output file 
    Path file = getDefaultWorkFile(context, "");
    FileSystem fs = file.getFileSystem(conf);
    final SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, file, context.getOutputKeyClass(),
            context.getOutputValueClass(), compressionType, codec, context);

    return new RecordWriter<K, V>() {

        @Override
        public void write(K key, V value) throws IOException {

            out.append(key, value);
        }

        @Override
        public void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    };
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsTextOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = conf.get("mapred.textoutputformat.separator", "\t");
    CompressionCodec codec = null;/*from  ww  w  . j a va2  s. c om*/
    String extension = "";
    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    if (!isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
    } else {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)),
                keyValueSeparator);
    }
}

From source file:edu.berkeley.cs.amplab.adam.io.InterleavedFastqInputFormat.java

License:Apache License

public RecordReader<Void, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    context.setStatus(genericSplit.toString());
    return new InterleavedFastqRecordReader(context.getConfiguration(), (FileSplit) genericSplit); // cast as per example in TextInputFormat
}

From source file:edu.gslis.streamcorpus.ThriftRecordReader.java

License:Apache License

public ThriftRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index)
        throws IOException {
    this.path = split.getPath(index);
    fs = this.path.getFileSystem(context.getConfiguration());
    this.startOffset = split.getOffset(index);
    this.end = startOffset + split.getLength(index);
    this.pos = startOffset;

    in = fs.open(path);// w w w.  j  av a 2 s . co  m

    if (path.toUri().toString().endsWith("xz"))
        tp = new TBinaryProtocol.Factory().getProtocol(new TIOStreamTransport(new XZInputStream(in)));
    else
        tp = new TBinaryProtocol.Factory().getProtocol(new TIOStreamTransport(in));

}

From source file:edu.indiana.d2i.htrc.io.dataapi.IDRecorderReader.java

License:Apache License

@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    split = (IDInputSplit) inputSplit;/* ww  w  .  jav a  2 s  .  c  o m*/
    iditerator = split.getIDIterator();

    logger.info("split has " + split.getLength() + " books");

    conf = taskAttemptContext.getConfiguration();
    maxIdRetrieved = conf.getInt(HTRCConstants.MAX_ID_RETRIEVED, 100);
    dataEPR = split.getLocations()[0];
    delimitor = conf.get(HTRCConstants.DATA_API_URL_DELIMITOR, "|");
    clientID = conf.get(HTRCConstants.DATA_API_CLIENTID, "yim");
    clientSecrete = conf.get(HTRCConstants.DATA_API_CLIENTSECRETE, "yim");
    tokenLoc = conf.get(HTRCConstants.DATA_API_TOKENLOC,
            "https://129-79-49-119.dhcp-bl.indiana.edu:25443/oauth2/token?grant_type=client_credentials");
    selfsigned = conf.getBoolean(HTRCConstants.DATA_API_SELFSIGNED, true);

    if (dataEPR.equals(HTRCConstants.DATA_API_DEFAULT_URL)) {
        dataEPR = HTRCConstants.DATA_API_DEFAULT_URL_PREFIX + dataEPR;
    }

    dataClient = new HTRCDataAPIClient.Builder(dataEPR, delimitor).authentication(true).selfsigned(selfsigned)
            .clientID(clientID).clientSecrete(clientSecrete).tokenLocation(tokenLoc).build();

    //      dataClient = Utilities.creatDataAPIClient(conf);

    key = new Text();
    value = new Text();
}

From source file:edu.indiana.d2i.htrc.io.index.lucene.LuceneRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    split = (IDInputSplit) inputSplit;/* www  .j a v  a2  s . co m*/
    iditerator = split.getIDIterator();
    conf = context.getConfiguration();
    client = LuceneClient.createLuceneClient(conf);
}

From source file:edu.indiana.d2i.htrc.io.index.solr.SolrRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    split = (IDInputSplit) inputSplit;//from  w ww  . j a  v a 2  s . c o m
    iditerator = split.getIDIterator();
    conf = context.getConfiguration();
    client = new SolrClient(conf, true);
}