Example usage for org.apache.hadoop.mapred JobConf get

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.rim.logdriver.mapred.boom.BoomInputFormat.java

License:Apache License

@Override
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    // Ensure we have sensible defaults for how we build blocks.
    if (job.get("mapreduce.job.max.split.locations") == null) {
        job.setLong("mapreduce.job.max.split.locations", MAX_SPLIT_LOCATIONS);
    }/*from   w w w.j  a  va2  s.  com*/
    if (job.get("mapred.max.split.size") == null) {
        // Try to set the split size to the default block size. In case of
        // failure, we'll use this 128MB default.
        long blockSize = 128 * 1024 * 1024; // 128MB
        try {
            blockSize = FileSystem.get(job).getDefaultBlockSize();
        } catch (IOException e) {
            LOG.error("Error getting filesystem to get get default block size (this does not bode well).");
        }
        job.setLong("mapred.max.split.size", blockSize);
    }

    return super.getSplits(job, numSplits);
}

From source file:com.run.mapred.hbase2tsv.HFileInputFormat_mr2.java

License:Apache License

@Override
public RecordReader<NullWritable, KeyValue> getRecordReader(InputSplit split, JobConf job, Reporter reporter)
        throws IOException {
    FileSplit fileSplit = (FileSplit) split;
    Path path = fileSplit.getPath();
    FileSystem fs = path.getFileSystem(job);
    LOG.info("Initialize HFileRecordReader for {}", path);
    Reader in = HFile.createReader(fs, path, new CacheConfig(job), job);

    // The file info must be loaded before the scanner can be used.
    // This seems like a bug in HBase, but it's easily worked around.
    in.loadFileInfo();/*from   w  w  w  .  j  a va 2  s  .co  m*/
    HFileScanner scanner = in.getScanner(false, false);

    String startRowStr = job.get(START_ROW_KEY);
    byte[] startRow = null;
    if (startRowStr != null) {
        startRow = decodeHexOrDie(startRowStr);
    }
    String stopRowStr = job.get(STOP_ROW_KEY);
    byte[] stopRow = null;
    if (stopRowStr != null) {
        stopRow = decodeHexOrDie(stopRowStr);
    }
    return new HFileRecordReader(in, scanner, startRow, stopRow);
}

From source file:com.scaleoutsoftware.soss.hserver.HiveNamedMapInputFormatMapred.java

License:Apache License

@Override
public InputSplit[] getSplits(JobConf jobConf, int i) throws IOException {
    String mapName = jobConf.get(HServerHiveStorageHandler.MAP_NAME);
    if (mapName == null || mapName.length() == 0) {
        throw new IOException("Input format is not configured with a valid NamedMap.");
    }//from w  w w . j  a va 2 s .c  o m

    int mapId = NamedMapFactory.getMap(mapName, new StubSerializer(), new StubSerializer()).getMapId();
    List<org.apache.hadoop.mapreduce.InputSplit> splits;

    try {
        splits = GridInputFormat.getSplits(mapId, i);
    } catch (InterruptedException e) {
        throw new IOException(e);
    }

    InputSplit[] wrappedSpilts = new InputSplit[splits.size()];

    Iterator splitIterator = splits.iterator();

    Path dummyPath = FileInputFormat.getInputPaths(jobConf)[0];
    LOG.debug("Using dummy path for splits:" + dummyPath);

    //Wrap splits to conform to mapred API
    for (i = 0; i < wrappedSpilts.length; i++) {
        wrappedSpilts[i] = new BucketSplitMapred((BucketSplit) splitIterator.next(), dummyPath);
    }

    return wrappedSpilts;
}

From source file:com.scaleoutsoftware.soss.hserver.HiveNamedMapInputFormatMapred.java

License:Apache License

@Override
public RecordReader getRecordReader(InputSplit inputSplit, JobConf configuration, Reporter reporter)
        throws IOException {
    String mapName = configuration.get(HServerHiveStorageHandler.MAP_NAME);

    int mapId = NamedMapFactory.getMap(mapName, new StubSerializer(), new StubSerializer()).getMapId();
    return new JsonRecordReader(inputSplit, configuration, mapId);
}

From source file:com.scaleoutsoftware.soss.hserver.NamedMapOutputFormatMapred.java

License:Apache License

@Override
public RecordWriter getRecordWriter(FileSystem fileSystem, JobConf configuration, String s,
        Progressable progressable) throws IOException {
    String mapName = configuration.get(outputNamedMapProperty);
    Class<CustomSerializer<K>> keySerializerClass = (Class<CustomSerializer<K>>) configuration
            .getClass(outputNamedMapKeySerializerProperty, null);
    Class<CustomSerializer<V>> valueSerializerClass = (Class<CustomSerializer<V>>) configuration
            .getClass(outputNamedMapValueSerializerProperty, null);
    int smOrdinal = configuration.getInt(SERIALIZATION_MODE, SerializationMode.DEFAULT.ordinal());
    int amOrdinal = configuration.getInt(AVAILABILITY_MODE, AvailabilityMode.USE_REPLICAS.ordinal());
    SerializationMode serializationMode = SerializationMode.values()[smOrdinal];
    AvailabilityMode availabilityMode = AvailabilityMode.values()[amOrdinal];

    if (mapName == null || mapName.length() == 0 || keySerializerClass == null
            || valueSerializerClass == null) {
        throw new IOException("Input format is not configured with a valid NamedMap.");
    }//  w  ww .  ja v  a  2  s. c  om

    CustomSerializer<K> keySerializer = ReflectionUtils.newInstance(keySerializerClass, configuration);
    keySerializer.setObjectClass((Class<K>) configuration.getClass(outputNamedMapKeyProperty, null));
    CustomSerializer<V> valueSerializer = ReflectionUtils.newInstance(valueSerializerClass, configuration);
    valueSerializer.setObjectClass((Class<V>) configuration.getClass(outputNamedMapValueProperty, null));
    NamedMap<K, V> namedMap = NamedMapFactory.getMap(mapName, keySerializer, valueSerializer);
    namedMap.setAvailabilityMode(availabilityMode);
    namedMap.setSerializationMode(serializationMode);

    return new NamedMapRecordWriter<K, V>(namedMap);
}

From source file:com.scaleoutsoftware.soss.hserver.NamedMapOutputFormatMapred.java

License:Apache License

@Override
public void checkOutputSpecs(FileSystem fileSystem, JobConf configuration) throws IOException {
    String mapName = configuration.get(outputNamedMapProperty);
    Class<CustomSerializer<K>> keySerializerClass = (Class<CustomSerializer<K>>) configuration
            .getClass(outputNamedMapKeySerializerProperty, null);
    Class<CustomSerializer<V>> valueSerializerClass = (Class<CustomSerializer<V>>) configuration
            .getClass(outputNamedMapValueSerializerProperty, null);

    if (mapName == null || mapName.length() == 0 || keySerializerClass == null
            || valueSerializerClass == null) {
        throw new IOException("Input format is not configured with a valid NamedMap.");
    }/*  ww  w .  j av a 2  s  .  co m*/
}

From source file:com.scaleunlimited.cascading.FlowMonitor.java

License:Apache License

private boolean isJobLocal(Config config) {
    if (config instanceof JobConf) {
        JobConf conf = (JobConf) config;
        return conf.get("mapred.job.tracker").equalsIgnoreCase("local");
    } else {/* w  w  w .  java2  s . co  m*/
        return true;
    }
}

From source file:com.scaleunlimited.cascading.GangliaFlowReporter.java

License:Apache License

private boolean isLocalJob() {
    JobConf jobConf = new JobConf();
    return jobConf.get("mapred.job.tracker").equalsIgnoreCase("local");
}

From source file:com.spotify.hdfs2cass.CassandraPartitioner.java

License:Apache License

@SuppressWarnings("unchecked")
@Override//w  w  w .ja  va2  s .co  m
public void configure(JobConf entries) {
    final String partitionerParam = entries.get(ClusterInfo.SPOTIFY_CASSANDRA_PARTITIONER_PARAM);
    if (partitionerParam == null) {
        throw new RuntimeException("Didn't get any cassandra partitioner information");
    }

    try {
        partitioner = (AbstractPartitioner<BigIntegerToken>) Class.forName(partitionerParam).newInstance();
    } catch (Exception ex) {
        throw new RuntimeException("Invalid partitioner class name: " + partitionerParam);
    }

    final String tokenNodesParam = entries.get(ClusterInfo.SPOTIFY_CASSANDRA_TOKENS_PARAM);
    if (tokenNodesParam == null) {
        throw new RuntimeException("Didn't get any cassandra information");
    }

    final String[] parts = StringUtils.splitByWholeSeparatorPreserveAllTokens(tokenNodesParam, ",");
    if ((parts == null) || (parts.length == 0)) {
        throw new RuntimeException("Didn't get any valid cassandra nodes information");
    }

    tokenNodes = new ArrayList<TokenNode>();
    for (String part : parts) {
        tokenNodes.add(new TokenNode(part));
    }

    Collections.sort(tokenNodes, new Comparator<TokenNode>() {
        @Override
        public int compare(TokenNode o1, TokenNode o2) {
            if (o1.equals(o2)) {
                return 0;
            }

            return o1.getStartToken().compareTo(o2.getStartToken());
        }
    });
}

From source file:com.spotify.hdfs2cass.misc.ClusterInfoTest.java

License:Apache License

@Test
public void testSetConf() throws Exception {
    final ClusterInfo clusterInfo = createClusterInfo();
    clusterInfo.init("foo");

    final JobConf conf = new JobConf();
    clusterInfo.setConf(conf);//from   ww w. ja va  2 s.  com

    final String result = conf.get(ClusterInfo.SPOTIFY_CASSANDRA_TOKENS_PARAM);
    assertEquals(4, StringUtils.splitPreserveAllTokens(result, ",").length);
}