Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskAttemptContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.splunk.hunk.input.packet.PcapRecordReader.java

License:Apache License

@Override
public void vixInitialize(VixInputSplit split, TaskAttemptContext context)
        throws IOException, InterruptedException {
    //gLogger.info("takashi:TgzPacketRecordReader:vixInitialize is called");

    FileSystem fs = FileSystem.get(context.getConfiguration());
    fsDataIn = fs.open(split.getPath());
    pcapReader = initPcapReader(packet_type_class, new DataInputStream(fsDataIn));

    packetIterator = pcapReader.iterator();
    totalBytesToRead = split.getLength() - split.getStart();
    objectPreProcessor = new PacketEventProcessor();
}

From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANInputFormat.java

License:Open Source License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    // create a reader... it will need its own backend instace
    String ckanHost = context.getConfiguration().get(INPUT_CKAN_HOST);
    String ckanPort = context.getConfiguration().get(INPUT_CKAN_PORT);
    boolean ckanSSL = context.getConfiguration().get(INPUT_CKAN_SSL).equals("true");
    String ckanAPIKey = context.getConfiguration().get(INPUT_CKAN_API_KEY);
    int splitsLength = new Integer(context.getConfiguration().get(INPUT_CKAN_SPLITS_LENGTH));
    logger.info("Creating record reader, the backend is at " + (ckanSSL ? "https://" : "http://") + ckanHost
            + ":" + ckanPort + " (API key=" + ckanAPIKey + ")");
    return new CKANRecordReader(new CKANBackend(ckanHost, ckanPort, ckanSSL, ckanAPIKey, splitsLength), split,
            context);/*from w w  w  . j a  va  2  s  .  com*/
}

From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANOutputFormat.java

License:Open Source License

@Override
public RecordWriter<Text, IntWritable> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    // get the package identifier from the URL
    String pkgURL = context.getConfiguration().get(OUTPUT_CKAN_URL);
    String[] urlParts = pkgURL.split("/");
    String pkgId = urlParts[urlParts.length - 1];

    // create a reader... it will need its own backend instace
    String ckanHost = context.getConfiguration().get(OUTPUT_CKAN_HOST);
    String ckanPort = context.getConfiguration().get(OUTPUT_CKAN_PORT);
    boolean ckanSSL = context.getConfiguration().get(OUTPUT_CKAN_SSL).equals("true");
    String ckanAPIKey = context.getConfiguration().get(OUTPUT_CKAN_API_KEY);
    logger.info("Creating record reader, the backend is at " + (ckanSSL ? "https://" : "http://") + ckanHost
            + ":" + ckanPort + " (API key=" + ckanAPIKey + ")");
    return new CKANRecordWriter(new CKANBackend(ckanHost, ckanPort, ckanSSL, ckanAPIKey, 0), pkgId, context);
}

From source file:com.tomslabs.grid.avro.AvroFileOutputFormat.java

License:Apache License

@Override
public RecordWriter<T, Object> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration config = context.getConfiguration();

    Schema schema = getWriteSchema(config);
    DatumWriter<T> datumWriter = getDatumWriter(config);

    final DataFileWriter<T> writer = new DataFileWriter<T>(datumWriter);

    if (getCompressOutput(context)) {
        int level = config.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
        writer.setCodec(CodecFactory.deflateCodec(level));
    }//  w  w  w .  j  a va 2 s .c o m

    Path file = getDefaultWorkFile(context, EXT);
    FileSystem fs = file.getFileSystem(config);

    writer.create(schema, fs.create(file));

    return new AvroRecordWriter<T>(writer);
}

From source file:com.tomslabs.grid.avro.AvroRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit fileSplit = (FileSplit) split;
    Configuration config = context.getConfiguration();
    Path path = fileSplit.getPath();

    this.in = new FsInput(path, config);

    DatumReader<T> datumReader = getDatumReader(config);

    this.reader = new DataFileReader<T>(in, datumReader);
    reader.sync(fileSplit.getStart()); // sync to start

    this.start = in.tell();
    this.end = fileSplit.getStart() + split.getLength();
}

From source file:com.toshiba.mwcloud.gs.hadoop.mapreduce.GSRowRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    reader_ = new GDRecordReader(((GSContainerSplit) split).getDelegate(), conf);

    key_ = new GSColumnKeyWritable(reader_.getKeyType());
    value_ = new GSRowWritable();
}

From source file:com.toshiba.mwcloud.gs.hadoop.util.GDRecordWriterBase.java

License:Apache License

/**
 * <div lang="ja">/*from w  w w .  jav a 2s  .  com*/
 * 
 * @param context TaskAttemptContext
 * @throws GSException GridDB??????
 * </div><div lang="en">
 * Constructor
 * @param context TaskAttemptContext object
 * @throws GSException an exception occurred in GridDB
 * </div>
 */
public GDRecordWriterBase(TaskAttemptContext context) throws GSException {
    context_ = context;
    Configuration conf = context.getConfiguration();

    conf_ = conf;

    Properties prop = GDPropertyUtils.getGSProp(conf, GDPropertyUtils.getGSPropertiesToWrite());
    gridstore_ = GridStoreFactory.getInstance().getGridStore(prop);

    rowBufferSize_ = GDPropertyUtils.getInteger(conf, GS_OUTPUT_ROW_BUFFER_SIZE);

    rowBuffer_ = null;
    rowBufferCount_ = 0;
}

From source file:com.tuplejump.calliope.hadoop.cql3.CqlRecordReader.java

License:Apache License

public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    if (CqlConfigHelper.getMultiRangeInputSplit(context.getConfiguration())) {
        logger.info("Initializing Record reader with MultiRangeSplit");
        initializeWithMultiRangeSplit(split, context);
    } else {/*from   w  w  w . jav a  2 s  .c om*/
        logger.info("Initializing Record reader with SingleRangeSplit");
        initializeWithColumnFamilySplit(split, context);
    }
}

From source file:com.tuplejump.calliope.hadoop.cql3.CqlRecordReader.java

License:Apache License

private void initializeWithColumnFamilySplit(InputSplit split, TaskAttemptContext context) throws IOException {
    this.split = split;
    ColumnFamilySplit cfSplit = (ColumnFamilySplit) split;
    Configuration conf = context.getConfiguration();
    totalRowCount = (cfSplit.getLength() < Long.MAX_VALUE) ? (int) cfSplit.getLength()
            : ConfigHelper.getInputSplitSize(conf);
    cfName = quote(ConfigHelper.getInputColumnFamily(conf));
    keyspace = quote(ConfigHelper.getInputKeyspace(conf));
    cqlQuery = CqlConfigHelper.getInputCql(conf);
    partitioner = ConfigHelper.getInputPartitioner(context.getConfiguration());

    try {//from ww w .j ava2  s  .c  om
        if (cluster != null)
            return;
        // create connection using thrift
        String[] locations = split.getLocations();

        Exception lastException = null;
        for (String location : locations) {
            try {
                cluster = CqlConfigHelper.getInputCluster(location, conf);
                break;
            } catch (Exception e) {
                lastException = e;
                logger.warn("Failed to create authenticated client to {}", location);
            }
        }
        if (cluster == null && lastException != null)
            throw lastException;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    if (cluster != null) {
        try {
            session = cluster.connect(keyspace);
        } catch (NoHostAvailableException nha) {
            Map<InetSocketAddress, Throwable> errors = nha.getErrors();
            logger.error(errors.toString());
            for (InetSocketAddress isa : errors.keySet()) {
                logger.error("ERROR ON HOST [" + isa.getAddress() + "/" + isa.getPort() + "] ");
                logger.error(errors.get(isa).getMessage());
                logger.error("Connection Timeout:  "
                        + cluster.getConfiguration().getSocketOptions().getConnectTimeoutMillis());
                logger.error("Local connection limit:  " + cluster.getConfiguration().getPoolingOptions()
                        .getCoreConnectionsPerHost(HostDistance.LOCAL));
                logger.error("Remote connection limit:  " + cluster.getConfiguration().getPoolingOptions()
                        .getCoreConnectionsPerHost(HostDistance.REMOTE));
                //logger.error("Connection Timeout:  " + cluster.getConfiguration().getSocketOptions().);
            }
            throw nha;
        }
    }
    rowIterator = new SingleRangeRowIterator();
    logger.debug("created {}", rowIterator);
}

From source file:com.tuplejump.calliope.hadoop.cql3.CqlRecordReader.java

License:Apache License

private void initializeWithMultiRangeSplit(InputSplit split, TaskAttemptContext context) throws IOException {
    this.split = split;
    MultiRangeSplit cfSplit = (MultiRangeSplit) split;
    Configuration conf = context.getConfiguration();
    totalRowCount = (cfSplit.getLength() < Long.MAX_VALUE) ? (int) cfSplit.getLength()
            : ConfigHelper.getInputSplitSize(conf);
    cfName = quote(ConfigHelper.getInputColumnFamily(conf));
    keyspace = quote(ConfigHelper.getInputKeyspace(conf));
    cqlQuery = CqlConfigHelper.getInputCql(conf);
    partitioner = ConfigHelper.getInputPartitioner(context.getConfiguration());

    try {/*from  ww w.ja v  a  2 s .  com*/
        if (cluster != null)
            return;
        // create connection using thrift
        String[] locations = split.getLocations();

        Exception lastException = null;
        for (String location : locations) {
            try {
                cluster = CqlConfigHelper.getInputCluster(location, conf);
                break;
            } catch (Exception e) {
                lastException = e;
                logger.warn("Failed to create authenticated client to {}", location);
            }
        }
        if (cluster == null && lastException != null)
            throw lastException;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    if (cluster != null) {
        try {
            session = cluster.connect(keyspace);
        } catch (NoHostAvailableException nha) {
            Map<InetSocketAddress, Throwable> errors = nha.getErrors();
            logger.error(errors.toString());
            for (InetSocketAddress isa : errors.keySet()) {
                logger.error("ERROR ON HOST [" + isa.getAddress() + "/" + isa.getPort() + "] ");
                logger.error(errors.get(isa).getMessage());
                logger.error("Connection Timeout:  "
                        + cluster.getConfiguration().getSocketOptions().getConnectTimeoutMillis());
                logger.error("Local connection limit:  " + cluster.getConfiguration().getPoolingOptions()
                        .getCoreConnectionsPerHost(HostDistance.LOCAL));
                logger.error("Remote connection limit:  " + cluster.getConfiguration().getPoolingOptions()
                        .getCoreConnectionsPerHost(HostDistance.REMOTE));
                //logger.error("Connection Timeout:  " + cluster.getConfiguration().getSocketOptions().);
            }
            throw nha;
        }
    }
    rowIterator = new MultiRangeRowIterator();
    logger.debug("created {}", rowIterator);
}