Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:edu.indiana.d2i.htrc.io.mem.MemCachedOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    return new MemCachedRecordWriter<K, V>(context.getConfiguration());
}

From source file:edu.isi.mavuno.input.ClueWarcInputFormat.java

License:Open Source License

/**
 * Just return the record reader//from w w  w  .ja v a 2s  . com
 */
@Override
public RecordReader<LongWritable, ClueWarcRecord> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException {
    return new ClueWarcRecordReader(context.getConfiguration(), (FileSplit) split);
}

From source file:edu.isi.mavuno.input.TextFileInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, TextDocument> createRecordReader(InputSplit split,
        TaskAttemptContext context) {
    Configuration conf = context.getConfiguration();
    try {//from   w ww.  jav  a2  s .  c  om
        return new TextFileRecordReader((FileSplit) split, conf);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

From source file:edu.isi.mavuno.input.TrecInputFormat.java

License:Apache License

/**
 * Returns a <code>RecordReader</code> for this <code>InputFormat</code>.
 *///from   ww w.jav  a2 s  . c  om
@Override
public RecordReader<LongWritable, TrecDocument> createRecordReader(InputSplit inputSplit,
        TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    return new TrecDocumentRecordReader((FileSplit) inputSplit, conf);
}

From source file:edu.isi.mavuno.input.XMLInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    Configuration conf = context.getConfiguration();
    try {//from   w w w  .jav a2s . c om
        return new XMLRecordReader((FileSplit) split, conf);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

From source file:edu.purdue.cs.HSPGiST.HadoopClasses.LocalHSPGiSTOutputFormat.java

License:Apache License

protected edu.purdue.cs.HSPGiST.HadoopClasses.NullableSequenceFile.Writer getWriter(TaskAttemptContext context,
        Class<?> keyClass, Class<?> valueClass) throws IOException {
    Configuration conf = context.getConfiguration();

    Path file = getDefaultWorkFile(context, "");
    return edu.purdue.cs.HSPGiST.HadoopClasses.NullableSequenceFile.createWriter(conf, file, keyClass,
            valueClass);/*from   ww  w  .  j a v  a  2s  . c om*/
}

From source file:edu.rutgers.ess.crs.utility.CSVInputFormat.java

License:Apache License

public RecordReader<LongWritable, TextArrayWritable> createRecordReader(final InputSplit split,
        final TaskAttemptContext context) {
    final String csvDelimiter = context.getConfiguration().get(CSVInputFormat.CSV_TOKEN_SEPARATOR_CONFIG);
    Character separator = null;//from  ww  w.ja v  a  2  s . co m
    if (csvDelimiter != null && csvDelimiter.length() == 1) {
        separator = csvDelimiter.charAt(0);
    }
    return new CSVRecordReader(separator);
}

From source file:edu.rutgers.ess.crs.utility.KeyValueCSVInputFormat.java

License:Apache License

public RecordReader<Text, TextArrayWritable> createRecordReader(final InputSplit split,
        final TaskAttemptContext context) throws IOException {
    final String tokenDelimiter = context.getConfiguration()
            .get(KeyValueCSVInputFormat.CSV_TOKEN_SEPARATOR_CONFIG);
    final String keyValueDelimiter = context.getConfiguration()
            .get(KeyValueCSVInputFormat.CSV_KEYVALUE_SEPARATOR_CONFIG);
    if (tokenDelimiter.equals(keyValueDelimiter)) {
        throw new IllegalArgumentException(
                "CSV_TOKEN_SEPARATOR_CONFIG is the same as CSV_KEYVALUE_SEPARATOR_CONFIG");
    }/*from  w w w .j  a  va 2 s. c  om*/
    return new KeyValueCSVRecordReader(tokenDelimiter, context.getConfiguration());
}

From source file:edu.rutgers.ess.crs.utility.KeyValueCSVOutputFormat.java

License:Apache License

public RecordWriter<Text, TextArrayWritable> getRecordWriter(final TaskAttemptContext context)
        throws IOException, InterruptedException {
    final Configuration conf = context.getConfiguration();
    final boolean isCompressed = getCompressOutput((JobContext) context);
    final String tokenSeparator = conf.get(KeyValueCSVOutputFormat.CSV_TOKEN_SEPARATOR_CONFIG);
    final String keyValueSeparator = conf.get(KeyValueCSVOutputFormat.CSV_KEYVALUE_SEPARATOR_CONFIG);
    CompressionCodec codec = null;//from ww  w.  j a  v  a  2 s . co  m
    String extension = "";
    if (isCompressed) {
        final Class<? extends CompressionCodec> codecClass = (Class<? extends CompressionCodec>) getOutputCompressorClass(
                (JobContext) context, GzipCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }
    final Path file = this.getDefaultWorkFile(context, extension);
    final FileSystem fs = file.getFileSystem(conf);
    if (!isCompressed) {
        final FSDataOutputStream fileOut = fs.create(file, false);
        return new KeyValueCSVRecordWriter((DataOutputStream) fileOut, tokenSeparator, keyValueSeparator);
    }
    final FSDataOutputStream fileOut = fs.create(file, false);
    return new KeyValueCSVRecordWriter(
            new DataOutputStream((OutputStream) codec.createOutputStream((OutputStream) fileOut)),
            tokenSeparator, keyValueSeparator);
}

From source file:edu.uci.ics.hyracks.hdfs2.dataflow.HDFSReadOperatorDescriptor.java

License:Apache License

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
        IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
        throws HyracksDataException {
    final List<FileSplit> inputSplits = splitsFactory.getSplits();

    return new AbstractUnaryOutputSourceOperatorNodePushable() {
        private String nodeName = ctx.getJobletContext().getApplicationContext().getNodeId();
        private ContextFactory ctxFactory = new ContextFactory();

        @SuppressWarnings("unchecked")
        @Override//ww  w  .ja v a2 s  .com
        public void initialize() throws HyracksDataException {
            ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
            try {
                Thread.currentThread().setContextClassLoader(ctx.getJobletContext().getClassLoader());
                Job job = confFactory.getConf();
                job.getConfiguration().setClassLoader(ctx.getJobletContext().getClassLoader());
                IKeyValueParser parser = tupleParserFactory.createKeyValueParser(ctx);
                writer.open();
                InputFormat inputFormat = ReflectionUtils.newInstance(job.getInputFormatClass(),
                        job.getConfiguration());
                int size = inputSplits.size();
                for (int i = 0; i < size; i++) {
                    /**
                     * read all the partitions scheduled to the current node
                     */
                    if (scheduledLocations[i].equals(nodeName)) {
                        /**
                         * pick an unread split to read synchronize among
                         * simultaneous partitions in the same machine
                         */
                        synchronized (executed) {
                            if (executed[i] == false) {
                                executed[i] = true;
                            } else {
                                continue;
                            }
                        }

                        /**
                         * read the split
                         */
                        TaskAttemptContext context = ctxFactory.createContext(job.getConfiguration(), i);
                        context.getConfiguration().setClassLoader(ctx.getJobletContext().getClassLoader());
                        RecordReader reader = inputFormat.createRecordReader(inputSplits.get(i), context);
                        reader.initialize(inputSplits.get(i), context);
                        while (reader.nextKeyValue() == true) {
                            parser.parse(reader.getCurrentKey(), reader.getCurrentValue(), writer,
                                    inputSplits.get(i).toString());
                        }
                    }
                }
                parser.close(writer);
                writer.close();
            } catch (Exception e) {
                throw new HyracksDataException(e);
            } finally {
                Thread.currentThread().setContextClassLoader(ctxCL);
            }
        }
    };
}