Example usage for org.apache.hadoop.conf Configuration getClass

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getClass.

Prototype

public Class<?> getClass(String name, Class<?> defaultValue)

Source Link

Document

Get the value of the name property as a Class.

Usage

From source file:com.hortonworks.hbase.replication.bridge.HBaseRPC.java

License:Apache License

/**
 * Returns a new instance of the configured {@link RpcEngine} implementation.
 *//*from   w w w .j  av a  2 s . c  o  m*/
public static synchronized RpcEngine getProtocolEngine(Configuration conf) {
    // check for a configured default engine
    Class<?> impl = conf.getClass(RPC_ENGINE_PROP, WritableRpcEngine.class);

    LOG.debug("Using RpcEngine: " + impl.getName());
    RpcEngine engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
    return engine;
}

From source file:com.ibm.jaql.io.hadoop.AbstractDb2InputFormat.java

License:Apache License

public static Connection connect(Configuration conf, Properties overrides) throws IOException, SQLException {
    String url = conf.get(URL_KEY);
    String propRec = conf.get(PROPERTIES_KEY);
    Class<? extends Driver> driverClass = conf.getClass(DRIVER_KEY, Driver.class).asSubclass(Driver.class);

    Driver driver;/*from   w  ww  . ja va 2 s .com*/
    try {
        driver = driverClass.newInstance();
    } catch (Exception e) {
        throw new UndeclaredThrowableException(e);// IOException("Error constructing jdbc driver", e);
    }

    Properties props = new Properties();

    if (propRec != null && !"".equals(propRec)) {
        try {
            JsonParser parser = new JsonParser(new StringReader(propRec));
            JsonRecord jrec = (JsonRecord) parser.JsonVal();
            for (Entry<JsonString, JsonValue> f : jrec) {
                JsonString key = f.getKey();
                JsonValue value = f.getValue();
                props.setProperty(key.toString(), value.toString());
            }
        } catch (ParseException pe) {
            throw new UndeclaredThrowableException(pe); // IOException("couldn't parse "+PROPERTIES_KEY+" = "+jsonRec, pe);
        }
    }
    // props.put("readOnly", true);
    if (overrides != null) {
        props.putAll(overrides);
    }

    //    DriverPropertyInfo[] info = driver.getPropertyInfo(url, props);

    return driver.connect(url, props);
}

From source file:com.marklogic.contentpump.MultithreadedMapper.java

License:Apache License

/**
 * Get the application's mapper class.// w  w  w  .  ja  v  a 2s . c  o m
 * 
 * @param <K1>
 *            the map's input key type
 * @param <V1>
 *            the map's input value type
 * @param <K2>
 *            the map's output key type
 * @param <V2>
 *            the map's output value type
 * @param job
 *            the job
 * @return the mapper class to run
 */
@SuppressWarnings("unchecked")
public static <K1, V1, K2, V2> Class<BaseMapper<K1, V1, K2, V2>> getMapperClass(JobContext job) {
    Configuration conf = job.getConfiguration();
    return (Class<BaseMapper<K1, V1, K2, V2>>) conf.getClass(ConfigConstants.CONF_MULTITHREADEDMAPPER_CLASS,
            BaseMapper.class);
}

From source file:com.mongodb.hadoop.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<?> getMapperOutputKey(final Configuration conf) {
    return conf.getClass(JOB_MAPPER_OUTPUT_KEY, null);
}

From source file:com.mongodb.hadoop.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<?> getMapperOutputValue(final Configuration conf) {
    return conf.getClass(JOB_MAPPER_OUTPUT_VALUE, null);
}

From source file:com.mongodb.hadoop.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<?> getOutputKey(final Configuration conf) {
    return conf.getClass(JOB_OUTPUT_KEY, null);
}

From source file:com.mongodb.hadoop.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<?> getOutputValue(final Configuration conf) {
    return conf.getClass(JOB_OUTPUT_VALUE, null);
}

From source file:com.mongodb.hadoop.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<?> getBSONPathFilter(final Configuration conf) {
    return conf.getClass(BSON_PATHFILTER, null);
}

From source file:com.moz.fiji.mapreduce.IntegrationTestJobHistoryFijiTable.java

License:Apache License

/**
 * Test of all the basic information recorded by a mapper.
 *///from   ww  w .ja  v a2 s.co  m
@Test
public void testMappers() throws Exception {
    createAndPopulateFooTable();
    final Configuration jobConf = getConf();
    // Set a value in the configuration. We'll check to be sure we can retrieve it later.
    jobConf.set("conf.test.animal.string", "squirrel");
    final Fiji fiji = Fiji.Factory.open(getFijiURI());
    try {
        final FijiURI fooTableURI = FijiURI.newBuilder(getFijiURI()).withTableName("foo").build();
        final JobHistoryFijiTable jobHistory = JobHistoryFijiTable.open(fiji);

        try {
            // Construct a Producer for this table.
            final FijiProduceJobBuilder builder = FijiProduceJobBuilder.create().withConf(jobConf)
                    .withInputTable(fooTableURI).withProducer(EmailDomainProducer.class)
                    .withOutput(MapReduceJobOutputs.newDirectFijiTableMapReduceJobOutput(fooTableURI));
            FijiMapReduceJob mrJob = builder.build();

            // Record the jobId and run the job.
            String jobName = mrJob.getHadoopJob().getJobName();
            LOG.info("About to run job: " + jobName);
            assertTrue(mrJob.run());
            String jobId = mrJob.getHadoopJob().getJobID().toString();
            LOG.info("Job was run with id: " + jobId);

            // Retrieve the recorded values and sanity test them.
            JobHistoryEntry jobEntry = jobHistory.getJobDetails(jobId);
            assertEquals(jobEntry.getJobName(), jobName);
            assertEquals(jobEntry.getJobId(), jobId);
            assertTrue(jobEntry.getJobStartTime() < jobEntry.getJobEndTime());
            assertEquals("SUCCEEDED", jobEntry.getJobEndStatus());

            // Check counters. We don't know the exact number of rows in the foo table, so just check if
            // it's greater than 0.
            final String countersString = jobEntry.getJobCounters();
            final Pattern countersPattern = Pattern.compile("PRODUCER_ROWS_PROCESSED=(\\d+)");
            final Matcher countersMatcher = countersPattern.matcher(countersString);
            assertTrue(countersMatcher.find());
            assertTrue(Integer.parseInt(countersMatcher.group(1)) > 0);

            // Test to make sure the Configuration has the correct producer class, and records the value
            // we set previously.
            final String configString = jobEntry.getJobConfiguration();
            final Configuration config = new Configuration();
            config.addResource(new ByteArrayInputStream(configString.getBytes()));
            assertTrue(EmailDomainProducer.class == config.getClass(FijiConfKeys.FIJI_PRODUCER_CLASS, null));
            assertEquals("Couldn't retrieve configuration field from deserialized configuration.", "squirrel",
                    config.get("conf.test.animal.string"));
        } finally {
            jobHistory.close();
        }
    } finally {
        fiji.release();
    }
}

From source file:com.moz.fiji.mapreduce.kvstore.TestKeyValueStoreConfiguration.java

License:Apache License

@Test
public void testStoreClass() {
    Configuration parent = new Configuration(false);
    KeyValueStoreConfiguration isolated = KeyValueStoreConfiguration.createInConfiguration(parent, 0);
    isolated.setClass("foo-key", String.class, Object.class);
    assertEquals(String.class, isolated.getClass("foo-key", null));
    assertEquals(String.class, isolated.getClass("foo-key", null, Object.class));

    // Check that this value is stored in the namespace on the parent:
    Configuration delegate = isolated.getDelegate();
    assertEquals(String.class,
            delegate.getClass(KeyValueStoreConfiguration.confKeyAtIndex("foo-key", 0), null));
}