Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.aliyun.openservices.tablestore.hadoop.TableStore.java

License:Apache License

/**
 * Set an endpoint of TableStore into a JobContext.
 *//*  ww w  . jav  a 2s.  com*/
public static void setEndpoint(JobContext job, String endpoint) {
    Preconditions.checkNotNull(job, "job must be nonnull");
    setEndpoint(job.getConfiguration(), new Endpoint(endpoint));
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStore.java

License:Apache License

/**
 * Set both an endpoint and an instance name into a JobContext.
 *//*  w  w w  .j  a  va2s  .  c  o  m*/
public static void setEndpoint(JobContext job, String endpoint, String instance) {
    Preconditions.checkNotNull(job, "job must be nonnull");
    setEndpoint(job.getConfiguration(), new Endpoint(endpoint, instance));
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreInputFormat.java

License:Apache License

/**
 * Add a RangeRowQueryCriteria object as data source.
 *//*from  w w  w . ja  va  2  s  . c  o  m*/
public static void addCriteria(JobContext job, RangeRowQueryCriteria criteria) {
    Preconditions.checkNotNull(job, "job must be nonnull");
    addCriteria(job.getConfiguration(), criteria);
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreInputFormat.java

License:Apache License

/**
 * Clear TableStore data sources./*  w ww  . ja v a2 s  .  c o m*/
 */
public static void clearCriteria(JobContext job) {
    Preconditions.checkNotNull(job, "job must be nonnull");
    clearCriteria(job.getConfiguration());
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    SyncClientInterface ots = TableStore.newOtsClient(conf);
    try {/*from w w  w  .  java 2  s .  com*/
        return getSplits(conf, ots);
    } finally {
        ots.shutdown();
    }
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreOutputFormat.java

License:Apache License

/**
 * Set a table in TableStore as data sink.
 *//*from  w w w  . jav  a  2s  . c  om*/
public static void setOutputTable(JobContext job, String outputTable) {
    setOutputTable(job.getConfiguration(), outputTable);
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreOutputFormat.java

License:Apache License

/**
 * Set max batch size for BatchWriteRow requests to TableStore.
 * This is optional.//w ww  . j a  va  2 s  . c om
 */
public static void setMaxBatchSize(JobContext job, int maxBatchSize) {
    setMaxBatchSize(job.getConfiguration(), maxBatchSize);
}

From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    checkTable(conf);/* www .j  a  va2  s  .  co  m*/
}

From source file:com.ambiata.ivory.operation.hadoop.DelegatingInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    Job jobCopy = new Job(conf);
    List<InputSplit> splits = new ArrayList<InputSplit>();
    Map<Path, InputFormat> formatMap = MultipleInputs.getInputFormatMap(job);
    Map<Path, Class<? extends Mapper>> mapperMap = MultipleInputs.getMapperTypeMap(job);
    Map<Class<? extends InputFormat>, List<Path>> formatPaths = new HashMap<Class<? extends InputFormat>, List<Path>>();

    // First, build a map of InputFormats to Paths
    for (Entry<Path, InputFormat> entry : formatMap.entrySet()) {
        if (!formatPaths.containsKey(entry.getValue().getClass())) {
            formatPaths.put(entry.getValue().getClass(), new LinkedList<Path>());
        }// w  ww.j  av a2  s.c o m

        formatPaths.get(entry.getValue().getClass()).add(entry.getKey());
    }

    for (Entry<Class<? extends InputFormat>, List<Path>> formatEntry : formatPaths.entrySet()) {
        Class<? extends InputFormat> formatClass = formatEntry.getKey();
        InputFormat format = (InputFormat) ReflectionUtils.newInstance(formatClass, conf);
        List<Path> paths = formatEntry.getValue();

        Map<Class<? extends Mapper>, List<Path>> mapperPaths = new HashMap<Class<? extends Mapper>, List<Path>>();

        // Now, for each set of paths that have a common InputFormat, build
        // a map of Mappers to the paths they're used for
        for (Path path : paths) {
            Class<? extends Mapper> mapperClass = mapperMap.get(path);
            if (!mapperPaths.containsKey(mapperClass)) {
                mapperPaths.put(mapperClass, new LinkedList<Path>());
            }

            mapperPaths.get(mapperClass).add(path);
        }

        // Now each set of paths that has a common InputFormat and Mapper can
        // be added to the same job, and split together.
        for (Entry<Class<? extends Mapper>, List<Path>> mapEntry : mapperPaths.entrySet()) {
            paths = mapEntry.getValue();
            Class<? extends Mapper> mapperClass = mapEntry.getKey();

            if (mapperClass == null) {
                try {
                    mapperClass = job.getMapperClass();
                } catch (ClassNotFoundException e) {
                    throw new IOException("Mapper class is not found", e);
                }
            }

            FileInputFormat.setInputPaths(jobCopy, paths.toArray(new Path[paths.size()]));

            // Get splits for each input path and tag with InputFormat
            // and Mapper types by wrapping in a TaggedInputSplit.
            List<InputSplit> pathSplits = format.getSplits(jobCopy);
            for (InputSplit pathSplit : pathSplits) {
                splits.add(new TaggedInputSplit(pathSplit, conf, format.getClass(), mapperClass));
            }
        }
    }

    return splits;
}

From source file:com.ambiata.ivory.operation.hadoop.MultipleInputs.java

License:Apache License

/**
 * Retrieves a map of {@link Path}s to the {@link InputFormat} class
 * that should be used for them./* ww  w.ja  va  2  s .co  m*/
 *
 * @param job The {@link JobContext}
 * @see #addInputPath(JobConf, Path, Class)
 * @return A map of paths to inputformats for the job
 */
@SuppressWarnings("unchecked")
static Map<Path, InputFormat> getInputFormatMap(JobContext job) {
    Map<Path, InputFormat> m = new HashMap<Path, InputFormat>();
    Configuration conf = job.getConfiguration();
    String[] pathMappings = conf.get(DIR_FORMATS).split(",");
    for (String pathMappingEncoded : pathMappings) {
        /* WAS not decoded */
        String pathMapping = decode(pathMappingEncoded);
        String[] split = pathMapping.split(";");
        InputFormat inputFormat;
        try {
            inputFormat = (InputFormat) ReflectionUtils.newInstance(conf.getClassByName(split[1]), conf);
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        m.put(new Path(split[0]), inputFormat);
    }
    return m;
}