Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:BamRecordReader.java

License:Apache License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();//from w w w  .java  2s. com
    split_length = split.getLength();
    System.out.println("start: " + start);
    System.out.println("split_length: " + split_length);
    fileInfo = split.getPath();
    //String fileName = fileInfo.toString().split("-")[0];
    //Path file = new Path(fileName);
    //compressionCodecs = new CompressionCodecFactory(job);
    //final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    //FileSystem fs = file.getFileSystem(job);
    //fileIn = fs.open(file);
    //fileIn.seek(start);
    //this.pos = start;
}

From source file:HtmlInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    try {/*from ww  w.j a  v  a  2 s  .  c o  m*/
        return new HtmlRecordReader((FileSplit) split, context.getConfiguration());
    } catch (IOException ioe) {
        return null;
    }
}

From source file:ZipFileRecordReader.java

License:Apache License

/**
 * Initialise and open the ZIP file from the FileSystem
 *///www.ja  v  a  2 s  . c o m
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    FileSplit split = (FileSplit) inputSplit;
    Configuration conf = taskAttemptContext.getConfiguration();
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);

    // Open the stream
    fsin = fs.open(path);
    zip = new ZipInputStream(fsin);
}

From source file:alluxio.hadoop.mapreduce.KeyValueOutputCommitter.java

License:Apache License

/**
 * @param taskContext MapReduce task configuration
 * @return true if the task output directory exists, otherwise false
 * @throws IOException if fails to determine whether the output directory exists
 *//* w w w.j a  v a  2s. com*/
@Override
public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
    Path taskOutputPath = new Path(KeyValueOutputFormat.getTaskOutputURI(taskContext).toString());
    FileSystem fs = taskOutputPath.getFileSystem(taskContext.getConfiguration());
    return fs.exists(taskOutputPath);
}

From source file:andromache.hadoop.CassandraRecordReader.java

License:Apache License

public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    this.context = context;
    this.split = (CassandraSplit) split;
    Configuration conf = context.getConfiguration();
    KeyRange jobRange = CassandraConfigHelper.getInputKeyRange(conf);
    filter = jobRange == null ? null : jobRange.row_filter;
    predicate = CassandraConfigHelper.getInputSlicePredicate(conf);
    boolean widerows = CassandraConfigHelper.getInputIsWide(conf);
    isEmptyPredicate = isEmptyPredicate(predicate);
    totalRowCount = CassandraConfigHelper.getInputSplitSize(conf);
    batchSize = CassandraConfigHelper.getRangeBatchSize(conf);
    cfName = ((CassandraSplit) split).getCf();
    consistencyLevel = CassandraConfigHelper.getReadConsistencyLevel(conf);

    keyspace = CassandraConfigHelper.getInputKeyspace(conf);

    try {/*from  ww  w .  j a v  a  2  s.c o m*/
        // only need to connect once
        if (socket != null && socket.isOpen()) {
            return;
        }

        // create connection using thrift
        String location = getLocation();
        socket = new TSocket(location, CassandraConfigHelper.getInputRpcPort(conf), timemout);
        TTransport transport = CassandraConfigHelper.getInputTransportFactory(conf).openTransport(socket);
        TBinaryProtocol binaryProtocol = new TBinaryProtocol(transport);
        client = new Cassandra.Client(binaryProtocol);

        // log in
        client.set_keyspace(keyspace);
        if (CassandraConfigHelper.getInputKeyspaceUserName(conf) != null) {
            Map<String, String> creds = new HashMap<String, String>();
            creds.put(IAuthenticator.USERNAME_KEY, CassandraConfigHelper.getInputKeyspaceUserName(conf));
            creds.put(IAuthenticator.PASSWORD_KEY, CassandraConfigHelper.getInputKeyspacePassword(conf));
            AuthenticationRequest authRequest = new AuthenticationRequest(creds);
            client.login(authRequest);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    iter = widerows ? new WideRowIterator() : new StaticRowIterator();
    logger.debug("created {}", iter);
}

From source file:andromache.hadoop.CassandraRecordWriter.java

License:Apache License

CassandraRecordWriter(TaskAttemptContext context) throws IOException {
    this.progressable = context;
    Configuration conf = context.getConfiguration();
    int queueSize = conf.getInt(QUEUE_SIZE, 256);
    int batchSize = conf.getInt(BATCH_SIZE, 32);
    long batchDelay = conf.getLong(BATCH_DELAY_MS, 200);
    ConsistencyLevel consistencyLevel = CassandraConfigHelper.getWriteConsistencyLevel(conf);

    CassandraClientFactory cassandraClientFactory = new CassandraClientFactory(
            CassandraConfigHelper.getOutputTransportFactory(conf),
            CassandraConfigHelper.getOutputRpcPort(conf));

    this.rangeThreadsCache = new RangeThreadsCache(conf, cassandraClientFactory, queueSize, batchSize,
            batchDelay, progressable, consistencyLevel);

    log.info("Using consistency level of {}", consistencyLevel);
}

From source file:authordetect.input.SingleBookReader.java

/**
 * @param inputSplit//from w w w .  ja v  a  2 s  .c o  m
 * @param context    the information about the task
 * @throws java.io.IOException
 * @throws InterruptedException
 */
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {

    FileSplit split = (FileSplit) inputSplit;
    Configuration configuration = context.getConfiguration();

    // get the option from configuration:
    // 0 for group by author, 1 for group by book
    int option = configuration.getInt("GROUP_OPTION", 0);

    Path path = split.getPath();
    filename = path.getName();
    FileSystem fileSystem = path.getFileSystem(configuration);
    FSDataInputStream inputStream = fileSystem.open(path);
    lineReader = new LineReader(inputStream, configuration);

    //initial start point and end point
    start = split.getStart();
    end = start + split.getLength();

    inputStream.seek(start);
    if (start != 0) {
        start += lineReader.readLine(new Text(), 0, (int) Math.min(Integer.MAX_VALUE, end - start));
    }

    start += lineReader.readLine(currentLine);

    prepareToScanBook(option);
}

From source file:authors.XmlInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    try {//  www.j  a v a  2 s  .  co m
        return new XmlRecordReader((FileSplit) split, context.getConfiguration());
    } catch (IOException ioe) {
        log.warn("Error while creating XmlRecordReader", ioe);
        return null;
    }
}

From source file:be.ugent.intec.halvade.uploader.mapreduce.MyFastqOutputFormat.java

public RecordWriter<PairedIdWritable, FastqRecord> getRecordWriter(TaskAttemptContext task) throws IOException {
    Configuration conf = task.getConfiguration();
    boolean isCompressed = getCompressOutput(task);

    CompressionCodec codec = null;/* w ww . j  a v  a 2  s  .com*/
    String extension = "";

    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(task, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }

    Path file = getDefaultWorkFile(task, extension);
    FileSystem fs = file.getFileSystem(conf);

    OutputStream output;

    if (isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        output = new DataOutputStream(codec.createOutputStream(fileOut));
    } else {
        output = fs.create(file, false);
    }

    return new FastqRecordWriter(conf, output);
}

From source file:br.ufpr.inf.hpath.XmlItemInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    try {/*from   w w w  .  j av a  2  s  .c om*/
        return new XmlRecordReader((FileSplit) split, context.getConfiguration());
    } catch (IOException ioe) {
        log.warn("Error while creating XmlRecordReader", ioe);
        return null;
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}