Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.linkedin.whiteelephant.mapreduce.MyAvroMultipleOutputs.java

License:Apache License

@SuppressWarnings("unchecked")
private synchronized RecordWriter getRecordWriter(TaskAttemptContext taskContext, String baseFileName)
        throws IOException, InterruptedException {

    // look for record-writer in the cache
    RecordWriter writer = recordWriters.get(baseFileName);

    // If not in cache, create a new one
    if (writer == null) {
        // get the record writer from context output format
        //FileOutputFormat.setOutputName(taskContext, baseFileName);

        // (mhayes) Fix so can write to different baseFileName
        // * Bug reported here:            https://issues.apache.org/jira/browse/AVRO-1215
        // * Original code submitted here: https://issues.apache.org/jira/browse/AVRO-1106
        taskContext.getConfiguration().set("avro.mo.config.namedOutput", baseFileName);

        try {/*from  w  w w .  ja v a2s. c o m*/
            writer = ((OutputFormat) ReflectionUtils.newInstance(taskContext.getOutputFormatClass(),
                    taskContext.getConfiguration())).getRecordWriter(taskContext);
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }

        // if counters are enabled, wrap the writer with context 
        // to increment counters 
        if (countersEnabled) {
            writer = new RecordWriterWithCounter(writer, baseFileName, context);
        }

        // add the record-writer to the cache
        recordWriters.put(baseFileName, writer);
    }
    return writer;
}

From source file:com.mapred.EmplRecordReader.java

@Override
public void initialize(InputSplit is, TaskAttemptContext tac) throws IOException, InterruptedException {
    FileSplit split = (FileSplit) is;//from   w ww .j  a v  a 2 s.c  om
    position = 0L;
    end = split.getLength();

    Configuration job = tac.getConfiguration();
    final Path file = split.getPath();
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());

    in = new LineRecordReader();
    in.initialize(is, tac);
}

From source file:com.marcolotz.lung.io.inputFormat.MultipleFilesRecordReader.java

License:Creative Commons License

/**
 * Implementation detail: This constructor is built to be called via
 * reflection from within CombineFileRecordReader.
 * //ww  w  .  ja  v  a 2  s.c  om
 * @param fileSplit
 *            The CombineFileSplit that this will read from.
 * @param context
 *            The context for this task.
 * @param pathToProcess
 *            The path index from the CombineFileSplit to process in this
 *            record.
 */
public MultipleFilesRecordReader(CombineFileSplit fileSplit, TaskAttemptContext context,
        Integer pathToProcess) {
    isProcessed = false;

    mFileToRead = fileSplit.getPath(pathToProcess);
    mFileLength = fileSplit.getLength(pathToProcess);

    mConf = context.getConfiguration();

    /* never used in production, just for code integrity */
    assert 0 == fileSplit.getOffset(pathToProcess);

    if (LOG.isDebugEnabled()) {
        LOG.debug("FileToRead is: " + mFileToRead.toString());
        LOG.debug("Processing path " + pathToProcess + " out of " + fileSplit.getNumPaths());

        try {
            FileSystem fs = FileSystem.get(mConf);

            /* never used in production, just for code integrity */
            assert fs.getFileStatus(mFileToRead).getLen() == mFileLength;
        } catch (IOException ioe) {
            LOG.debug("Problem in file length");
        }
    }

    fileContent = new BytesWritable();
}

From source file:com.marcolotz.lung.io.inputFormat.WholeFileRecordReader.java

License:Creative Commons License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    this.fileSplit = (FileSplit) split;
    this.conf = context.getConfiguration();
}

From source file:com.marklogic.contentpump.AggregateXMLReader.java

License:Apache License

@Override
public void initialize(InputSplit inSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    initConfig(context);//  w  w w .j  a  v a2 s.  co m
    initAggConf(context);

    f = XMLInputFactory.newInstance();
    setFile(((FileSplit) inSplit).getPath());
    fs = file.getFileSystem(context.getConfiguration());
    FileStatus status = fs.getFileStatus(file);
    if (status.isDirectory()) {
        iterator = new FileIterator((FileSplit) inSplit, context);
        inSplit = iterator.next();
    }
    initStreamReader(inSplit);
}

From source file:com.marklogic.contentpump.AggregateXMLReader.java

License:Apache License

protected void initAggConf(TaskAttemptContext context) {
    Configuration conf = context.getConfiguration();
    idName = conf.get(ConfigConstants.CONF_INPUT_URI_ID);
    if (idName == null) {
        useAutomaticId = true;/*w w  w  .jav a2s.c  om*/
    }
    recordName = conf.get(ConfigConstants.CONF_AGGREGATE_RECORD_ELEMENT);
    recordNamespace = conf.get(ConfigConstants.CONF_AGGREGATE_RECORD_NAMESPACE);

}

From source file:com.marklogic.contentpump.ArchiveOutputFormat.java

License:Apache License

@Override
public RecordWriter<DocumentURI, MarkLogicDocument> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    Path path = new Path(conf.get(ConfigConstants.CONF_OUTPUT_FILEPATH));
    return new ArchiveWriter(path, context);
}

From source file:com.marklogic.contentpump.ArchiveRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit inSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    initConfig(context);//ww  w .j a  v a  2s  . c  o  m
    allowEmptyMeta = conf.getBoolean(CONF_INPUT_ARCHIVE_METADATA_OPTIONAL, false);

    setFile(((FileSplit) inSplit).getPath());
    fs = file.getFileSystem(context.getConfiguration());
    FileStatus status = fs.getFileStatus(file);
    if (status.isDirectory()) {
        iterator = new FileIterator((FileSplit) inSplit, context);
        inSplit = iterator.next();
    }
    initStream(inSplit);
}

From source file:com.marklogic.contentpump.ArchiveWriter.java

License:Apache License

public ArchiveWriter(Path path, TaskAttemptContext context) {
    dir = path.toString();/*from  ww  w.  ja  v a  2 s  .  co m*/
    this.context = context;
    Configuration conf = context.getConfiguration();
    encoding = conf.get(OUTPUT_CONTENT_ENCODING, DEFAULT_ENCODING);
    String type = conf.get(CONF_OUTPUT_TYPE, DEFAULT_OUTPUT_TYPE);
    ExportOutputType outputType = ExportOutputType.valueOf(type.toUpperCase());
    if (outputType.equals(ExportOutputType.DOCUMENT)) {
        isExportDoc = true;
    } else {
        //archive uses DatabaseContentReader
        isExportDoc = false;
    }
}

From source file:com.marklogic.contentpump.CompressedAggXMLReader.java

License:Apache License

@Override
public void initialize(InputSplit inSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    initConfig(context);/*w  w  w.  ja  va 2 s. c o  m*/
    initAggConf(context);
    f = XMLInputFactory.newInstance();
    setFile(((FileSplit) inSplit).getPath());
    fs = file.getFileSystem(context.getConfiguration());

    FileStatus status = fs.getFileStatus(file);
    if (status.isDirectory()) {
        iterator = new FileIterator((FileSplit) inSplit, context);
        inSplit = iterator.next();
    }
    initStreamReader(inSplit);
}