Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.marklogic.contentpump.CompressedDelimitedJSONInputFormat.java

License:Apache License

@Override
public RecordReader<DocumentURIWithSourceInfo, Text> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException, InterruptedException {
    String codec = context.getConfiguration().get(ConfigConstants.CONF_INPUT_COMPRESSION_CODEC,
            CompressionCodec.ZIP.toString());
    if (codec.equalsIgnoreCase(CompressionCodec.ZIP.toString())) {
        return new ZipDelimitedJSONReader();
    } else if (codec.equalsIgnoreCase(CompressionCodec.GZIP.toString())) {
        return new GzipDelimitedJSONReader();
    } else {/*from   www  . java  2  s.  co m*/
        throw new UnsupportedOperationException("Unsupported codec: " + codec);
    }
}

From source file:com.marklogic.contentpump.CompressedDelimitedTextReader.java

License:Apache License

@Override
public void initialize(InputSplit inSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    initConfig(context);//from   w w  w. jav  a  2s. com
    initDocType();
    initDelimConf();

    setFile(((FileSplit) inSplit).getPath());
    fs = file.getFileSystem(context.getConfiguration());
    FileStatus status = fs.getFileStatus(file);
    if (status.isDirectory()) {
        iterator = new FileIterator((FileSplit) inSplit, context);
        inSplit = iterator.next();
    }

    initStream(inSplit);
}

From source file:com.marklogic.contentpump.CompressedDocumentInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
@Override/* w w  w  .j  a v  a  2  s  .c o m*/
public RecordReader<DocumentURIWithSourceInfo, VALUE> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    boolean streaming = conf.getBoolean(MarkLogicConstants.OUTPUT_STREAMING, false);
    if (streaming) {
        return (RecordReader<DocumentURIWithSourceInfo, VALUE>) new CompressedStreamingReader();
    } else {
        return new CompressedDocumentReader<VALUE>();
    }
}

From source file:com.marklogic.contentpump.DatabaseContentInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
@Override/* w  w  w. ja  v  a 2  s  .c o  m*/
public RecordReader<DocumentURI, VALUE> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException, InterruptedException {
    DatabaseContentReader reader = new DatabaseContentReader(context.getConfiguration());
    return (RecordReader<DocumentURI, VALUE>) reader;
}

From source file:com.marklogic.contentpump.DatabaseContentOutputFormat.java

License:Apache License

@Override
public RecordWriter<DocumentURI, DatabaseDocumentWithMeta> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    fastLoad = Boolean.valueOf(conf.get(OUTPUT_FAST_LOAD));
    Map<String, ContentSource> sourceMap = getSourceMap(fastLoad, context);
    // construct the DatabaseContentWriter
    return new DatabaseContentWriter<DatabaseDocumentWithMeta>(conf, sourceMap, fastLoad, am);

}

From source file:com.marklogic.contentpump.DatabaseTransformOutputFormat.java

License:Apache License

@Override
public RecordWriter<DocumentURI, DatabaseDocumentWithMeta> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    fastLoad = Boolean.valueOf(conf.get(OUTPUT_FAST_LOAD));
    Map<String, ContentSource> sourceMap = getSourceMap(fastLoad, context);
    // construct the DatabaseTransformContentWriter
    return new DatabaseTransformWriter<DatabaseDocumentWithMeta>(conf, sourceMap, fastLoad, am);
}

From source file:com.marklogic.contentpump.DelimitedJSONReader.java

License:Apache License

@Override
public void initialize(InputSplit inSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    /* Initialization in super class */
    initConfig(context);//from  w w w . ja v  a2  s.  c om
    /*  Get file(s) in input split */
    setFile(((FileSplit) inSplit).getPath());
    // Initialize reader properties
    generateId = conf.getBoolean(CONF_INPUT_GENERATE_URI, false);
    if (generateId) {
        idGen = new IdGenerator(file.toUri().getPath() + "-" + ((FileSplit) inSplit).getStart());
    } else {
        uriName = conf.get(CONF_INPUT_URI_ID, null);
        mapper = new ObjectMapper();
    }
    bytesRead = 0;
    totalBytes = inSplit.getLength();
    /* Check file status */
    fs = file.getFileSystem(context.getConfiguration());
    FileStatus status = fs.getFileStatus(file);
    if (status.isDirectory()) {
        iterator = new FileIterator((FileSplit) inSplit, context);
        inSplit = iterator.next();
    }
    /* Initialize buffered reader */
    initFileStream(inSplit);
}

From source file:com.marklogic.contentpump.DelimitedTextInputFormat.java

License:Apache License

@Override
public RecordReader<DocumentURIWithSourceInfo, Text> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException, InterruptedException {
    if (isSplitInput(context.getConfiguration())) {
        return new SplitDelimitedTextReader<Text>();
    } else {/* ww  w .j  a  va 2  s .  co  m*/
        return new DelimitedTextReader<Text>();
    }
}

From source file:com.marklogic.contentpump.DelimitedTextReader.java

License:Apache License

@Override
public void initialize(InputSplit inSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    initConfig(context);/*from w  w  w .  ja va  2s. co  m*/
    initDocType();
    initDelimConf();
    setFile(((FileSplit) inSplit).getPath());
    fs = file.getFileSystem(context.getConfiguration());
    FileStatus status = fs.getFileStatus(file);
    if (status.isDirectory()) {
        iterator = new FileIterator((FileSplit) inSplit, context);
        inSplit = iterator.next();
    }
    initParser(inSplit);
}

From source file:com.marklogic.contentpump.ImportRecordReader.java

License:Apache License

@SuppressWarnings("unchecked")
protected void initConfig(TaskAttemptContext context) {
    conf = context.getConfiguration();
    String type = conf.get(MarkLogicConstants.CONTENT_TYPE, MarkLogicConstants.DEFAULT_CONTENT_TYPE);
    if (!conf.getBoolean(MarkLogicConstants.OUTPUT_STREAMING, false)) {
        ContentType contentType = ContentType.valueOf(type);
        Class<? extends Writable> valueClass = contentType.getWritableClass();
        value = (VALUEIN) ReflectionUtils.newInstance(valueClass, conf);
    }/* www  .j  a  v a  2s.co m*/
    encoding = conf.get(MarkLogicConstants.OUTPUT_CONTENT_ENCODING, DEFAULT_ENCODING);
}