Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.conversantmedia.mapreduce.output.BloomFilterOutputFormat.java

License:Apache License

protected int getExpectedInsertions(TaskAttemptContext job) {
    return this.insertionSize > 0 ? this.insertionSize
            : job.getConfiguration().getInt(CONF_KEY_EXPECTED_INSERTIONS, -1);
}

From source file:com.conversantmedia.mapreduce.tool.AnnotatedDelegatingCombiner.java

License:Apache License

@Override
public Reducer<K1, V1, K2, V2> getDelegate(TaskAttemptContext context) {
    if (this.reducer == null) {
        Class<? extends Reducer> reducerClass = (Class<? extends Reducer>) context.getConfiguration()
                .getClass(CONFKEY_DELEGATE_COMBINER_CLASS, Reducer.class);
        reducer = (Reducer<K1, V1, K2, V2>) ReflectionUtils.newInstance(reducerClass,
                context.getConfiguration());
    }//from  www  . j a v a2s  . c o  m
    return this.reducer;
}

From source file:com.conversantmedia.mapreduce.tool.AnnotatedDelegatingMapper.java

License:Apache License

@Override
public Mapper<K1, V1, K2, V2> getDelegate(TaskAttemptContext context) {
    if (mapper == null) {
        Class<? extends Mapper> mapperClass = (Class<? extends Mapper>) context.getConfiguration()
                .getClass(CONFKEY_DELEGATE_MAPPER_CLASS, Mapper.class);
        mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(mapperClass, context.getConfiguration());
    }/*from www  .j  a  va 2 s.  co  m*/
    return this.mapper;
}

From source file:com.conversantmedia.mapreduce.tool.AnnotatedDelegatingReducer.java

License:Apache License

@Override
public Reducer<K1, V1, K2, V2> getDelegate(TaskAttemptContext context) {
    if (this.reducer == null) {
        Class<? extends Reducer> reducerClass = (Class<? extends Reducer>) context.getConfiguration()
                .getClass(CONFKEY_DELEGATE_REDUCER_CLASS, Reducer.class);
        reducer = (Reducer<K1, V1, K2, V2>) ReflectionUtils.newInstance(reducerClass,
                context.getConfiguration());
    }/* w  ww. j a va 2s .  co  m*/
    return this.reducer;
}

From source file:com.cotdp.hadoop.BrotliFileRecordReader.java

License:Apache License

/**
 * Initialize and open the ZIP file from the FileSystem
 *//*from  w  w  w  .  j a v a 2s  . c  om*/
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    FileSplit split = (FileSplit) inputSplit;
    Configuration conf = taskAttemptContext.getConfiguration();
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);

    // Set the file path as the key
    currentKey.set(path.getName());
    // Open the stream
    fsin = fs.open(path);

    String cmd = "/bin/cat";
    ProcessBuilder pb = new ProcessBuilder();
    pb.redirectOutput();
    pb.command(cmd);

    try {
        decompressor = pb.start();

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

From source file:com.cotdp.hadoop.ZipFileRecordReader.java

License:Apache License

/**
 * Initialise and open the ZIP file from the FileSystem
 *//*from  w  w  w. j  a  v a 2 s .com*/

public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    FileSplit split = (FileSplit) inputSplit;
    Configuration conf = taskAttemptContext.getConfiguration();
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);

    // Open the stream
    fsin = fs.open(path);
    zip = new ZipInputStream(fsin);
    try {
        entry = zip.getNextEntry();
    } catch (ZipException e) {
        if (ZipFileInputFormat.getLenient() == false)
            throw e;
    }
}

From source file:com.couchbase.sqoop.mapreduce.db.CouchbaseInputFormat.java

License:Apache License

@Override
/** {@inheritDoc} */
public RecordReader<Text, T> createRecordReader(final InputSplit split, final TaskAttemptContext context)
        throws IOException, InterruptedException {
    System.out.printf("Creating Record Reader %s", split.getLocations().toString());
    return createRecordReader(split, context.getConfiguration());
}

From source file:com.couchbase.sqoop.mapreduce.db.CouchbaseOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(final TaskAttemptContext context)
        throws IOException, InterruptedException {
    return new CouchbaseRecordWriter(new CouchbaseConfiguration(context.getConfiguration()));
}

From source file:com.datasalt.pangool.solr.SolrRecordWriter.java

License:Apache License

public SolrRecordWriter(int batchSize, boolean outputZipFile, int threadCount, int queueSize,
        String localSolrHome, String zipName, TupleDocumentConverter converter, TaskAttemptContext context) {
    this.localSolrHome = localSolrHome;
    this.zipName = zipName;
    conf = context.getConfiguration();
    this.batchSize = batchSize;

    setLogLevel("org.apache.solr.core", "WARN");
    setLogLevel("org.apache.solr.update", "WARN");
    Logger.getLogger("org.apache.solr.core").setLevel(Level.WARN);
    Logger.getLogger("org.apache.solr.update").setLevel(Level.WARN);
    java.util.logging.Logger.getLogger("org.apache.solr.core").setLevel(java.util.logging.Level.WARNING);
    java.util.logging.Logger.getLogger("org.apache.solr.update").setLevel(java.util.logging.Level.WARNING);

    setLogLevel("org.apache.solr", "WARN");
    Logger.getLogger("org.apache.solr").setLevel(Level.WARN);
    java.util.logging.Logger.getLogger("org.apache.solr").setLevel(java.util.logging.Level.WARNING);

    heartBeater = new HeartBeater(context);
    try {// www  . java2 s  .c  o m
        heartBeater.needHeartBeat();
        /** The actual file in hdfs that holds the configuration. */
        this.outputZipFile = outputZipFile;

        this.fs = FileSystem.get(conf);
        perm = new Path(FileOutputFormat.getOutputPath(context), getOutFileName(context, "part"));

        // Make a task unique name that contains the actual index output name to
        // make debugging simpler
        // Note: if using JVM reuse, the sequence number will not be reset for a
        // new task using the jvm

        Path temp = conf.getLocalPath("mapred.local.dir",
                "solr_" + conf.get("mapred.task.id") + '.' + sequence.incrementAndGet());

        if (outputZipFile && !perm.getName().endsWith(".zip")) {
            perm = perm.suffix(".zip");
        }
        fs.delete(temp, true); // delete old, if any
        fs.delete(perm, true); // delete old, if any
        local = fs.startLocalOutput(perm, temp);

        solrHome = findSolrConfig(conf);

        // }
        // Verify that the solr home has a conf and lib directory
        if (solrHome == null) {
            throw new IOException("Unable to find solr home setting");
        }

        // Setup a solr instance that we can batch writes to
        LOG.info("SolrHome: " + solrHome.toUri());
        String dataDir = new File(local.toString(), "data").getAbsoluteFile().toString();
        // copy the schema to the conf dir
        File confDir = new File(local.toString(), "conf");
        confDir.mkdirs();

        File unpackedSolrHome = new File(solrHome.toString());
        FileUtils.copyDirectory(new File(unpackedSolrHome, "conf"), confDir);

        Properties props = new Properties();
        props.setProperty("solr.data.dir", dataDir);
        props.setProperty("solr.home", solrHome.toString());
        SolrResourceLoader loader = new SolrResourceLoader(solrHome.toString(), null, props);
        LOG.info(String.format(
                "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to temporary directory %s, with permdir %s",
                solrHome, solrHome.toUri(), loader.getInstanceDir(), loader.getConfigDir(), dataDir, perm));
        CoreContainer container = new CoreContainer(loader);
        CoreDescriptor descr = new CoreDescriptor(container, "core1", solrHome.toString());
        descr.setDataDir(dataDir);
        descr.setCoreProperties(props);
        core = container.create(descr);
        container.register(core, false);
        solr = new EmbeddedSolrServer(container, "core1");
        batchWriter = new BatchWriter(solr, batchSize, context.getTaskAttemptID().getTaskID(), threadCount,
                queueSize);

        this.converter = converter;
    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
        throw new IllegalStateException(String.format("Failed to initialize record writer for %s, %s",
                context.getJobName(), conf.get("mapred.task.id")), e);
    } finally {
        heartBeater.cancelHeartBeat();
    }
}

From source file:com.datasalt.pangool.tuplemr.avro.AvroInputFormat.java

License:Apache License

@Override
public RecordReader<AvroWrapper<T>, NullWritable> createRecordReader(InputSplit inputSplit,
        TaskAttemptContext context) throws IOException, InterruptedException {
    context.setStatus(inputSplit.toString());
    return new AvroRecordReader<T>(getSchema(), isReflect, context.getConfiguration(), (FileSplit) inputSplit);
}