Example usage for org.apache.hadoop.mapred JobConf get

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.rapleaf.hank.hadoop.DomainBuilderDefaultOutputFormat.java

License:Apache License

public RecordWriter<KeyAndPartitionWritable, ValueWritable> getRecordWriter(FileSystem fs, JobConf conf,
        String name, Progressable progressable) throws IOException {

    // Implicitly relies on the FileOutputCommitter
    String outputPath = conf.get("mapred.work.output.dir");
    if (outputPath == null) {
        throw new RuntimeException("Path was not set in mapred.work.output.dir");
    }//from w w w. ja  va2 s . c o m

    // Load configuration items
    String domainName = DomainBuilderProperties.getDomainName(conf);
    VersionType versionType = DomainBuilderProperties.getVersionType(domainName, conf);
    // Load config
    Domain domain = DomainBuilderProperties.getDomain(conf);
    // Build RecordWriter with the Domain
    return new DomainBuilderRecordWriter(domain, versionType, new HDFSOutputStreamFactory(fs, outputPath));
}

From source file:com.redgate.hadoop.hive.azuretables.AzureTablesInputFormat.java

License:Apache License

/**
 * Sets up a RecordReader for an Azure Table, and build a connection string
 * from the table properties.//from   w w w . jav a 2 s .c  o m
 */
@Override
public RecordReader<Text, MapWritable> getRecordReader(InputSplit split, JobConf conf, Reporter reporter)
        throws IOException {
    String table = conf.get(ConfigurationUtil.TABLE);
    String accountName = ConfigurationUtil.accountName(conf);
    String storageConnectionString = ConfigurationUtil.getStorageConnectionString(conf);
    LOG.info(String.format("Connecting to table %s on account %s", table, accountName));
    return new AzureTablesRecordReader(storageConnectionString, table, split);
}

From source file:com.redgate.hadoop.hive.azuretables.AzureTablesInputFormat.java

License:Apache License

/**
 * Determine InputSplits for the Azure Table reader, using the partition
 * keys given in the table definition as the input split boundaries.
 * // w  ww .ja va2s  . c  o  m
 * @param conf
 *            The Hadoop job configuration
 * @param numSplits
 *            The desired number of splits, which is pretty much ignored at
 *            the moment. This is not ideal, since there should be a
 *            secondary mechanism to further split a table partition
 */
@Override
public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException {
    LOG.info("Partition keys: " + conf.get(ConfigurationUtil.PARTITION_KEYS));
    String[] partitionKeys = ConfigurationUtil.partitionKeys(conf);
    return AzureTablesSplit.getSplits(conf, partitionKeys, numSplits);
}

From source file:com.redgate.hadoop.hive.azuretables.ConfigurationUtil.java

License:Apache License

public static String accountName(JobConf conf) {
    return conf.get(ACCOUNT_NAME);
}

From source file:com.redgate.hadoop.hive.azuretables.ConfigurationUtil.java

License:Apache License

public static String accessKey(JobConf conf) {
    return conf.get(ACCESS_KEY);
}

From source file:com.redgate.hadoop.hive.azuretables.ConfigurationUtil.java

License:Apache License

public static String table(JobConf conf) {
    return conf.get(TABLE);
}

From source file:com.redgate.hadoop.hive.azuretables.ConfigurationUtil.java

License:Apache License

public static String[] partitionKeys(JobConf conf) {
    // TODO - need an actually safe way of splitting partition keys
    String keys = conf.get(PARTITION_KEYS);
    return keys.split(PARTITION_SPLIT);
}

From source file:com.redgate.hadoop.hive.azuretables.ConfigurationUtil.java

License:Apache License

public static String outputPartitionKey(JobConf conf) {
    String string = conf.get(OUTPUT_PARTITION_KEY);
    if (string == null) {
        string = "OUTPUT";
    }// w w w.  j a va2s  . co m
    return string;
}

From source file:com.ricemap.spateDB.core.SpatialSite.java

License:Apache License

public static CellInfo[] getCells(JobConf job) throws IOException {
    CellInfo[] cells = null;/* www.j a v a2  s  .  c o  m*/
    String cells_file = job.get(OUTPUT_CELLS);
    if (cells_file != null) {
        Path[] cacheFiles = DistributedCache.getLocalCacheFiles(job);
        for (Path cacheFile : cacheFiles) {
            if (cacheFile.getName().contains(cells_file)) {
                FSDataInputStream in = FileSystem.getLocal(job).open(cacheFile);

                int cellCount = in.readInt();
                cells = new CellInfo[cellCount];
                for (int i = 0; i < cellCount; i++) {
                    cells[i] = new CellInfo();
                    cells[i].readFields(in);
                }

                in.close();
            }
        }
    }
    return cells;
}

From source file:com.rim.logdriver.mapred.avro.AvroBlockInputFormat.java

License:Apache License

@Override
public RecordReader<AvroFileHeader, BytesWritable> getRecordReader(InputSplit split, JobConf job,
        Reporter reporter) throws IOException {

    // Ensure we have sensible defaults for how we build blocks.
    if (job.get("mapreduce.job.max.split.locations") == null) {
        job.setLong("mapreduce.job.max.split.locations", MAX_SPLIT_LOCATIONS);
    }//from  w  w  w.j a  v  a  2s.c om
    if (job.get("mapred.max.split.size") == null) {
        // Try to set the split size to the default block size. In case of
        // failure, we'll use this 128MB default.
        long blockSize = 128 * 1024 * 1024; // 128MB
        try {
            blockSize = FileSystem.get(job).getDefaultBlockSize();
        } catch (IOException e) {
            LOG.error("Error getting filesystem to get get default block size (this does not bode well).");
        }
        job.setLong("mapred.max.split.size", blockSize);
    }

    return new AvroBlockRecordReader(split, job);
}