Example usage for org.apache.hadoop.mapreduce.lib.db DBConfiguration configureDB

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.db DBConfiguration configureDB.

Prototype

public static void configureDB(Configuration conf, String driverClass, String dbUrl, String userName,
        String passwd) 

Source Link

Document

Sets the DB access related fields in the Configuration .

Usage

From source file:link.neolink.datamonster.Datamonster.java

License:Apache License

/**
 * Set the SQL credentials/* w w w .  j  a  v a 2 s. co m*/
 *
 * @param URL
 *    The SQL URI, in JDBC format
 * @param user
 *    The SQL username
 * @param password
 *    The SQL password
 */
public void setSQLCredentials(String URL, String user, String password) {
    DBConfiguration.configureDB(job.getConfiguration(), this.sqlDriver, URL, user, password);
    this.sqlConfig = new DBConfiguration(job.getConfiguration());
}

---

From source file:nl.sanoma.hdt.report.generator.ReportGeneratorDriver.java

License:Open Source License

/**
 * Job to import the values from DB to a MapFile in HDFS
 *
 * @param dBOutputPath the path where the MapFile output will be created
 * @return returns the exitCode of the job
 * @throws ConfigurationException//w ww . j ava  2 s  . c om
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
public Boolean importDBToMapFile(String dBOutputPath)
        throws ConfigurationException, IOException, InterruptedException, ClassNotFoundException {

    PropertiesConfiguration dBConfig = new PropertiesConfiguration(DB_CONFIG_FILE_NAME);
    //TODO: error handling if config file is missing

    //create MapFile from DB job
    Job job = new Job(getConf());
    Configuration conf = job.getConfiguration();
    job.setJobName("DB to MapFile import");
    job.setJarByClass(ReportGeneratorDriver.class);
    job.setMapperClass(DBToMapFileMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(Text.class);
    job.setInputFormatClass(DBInputFormat.class);
    job.setOutputFormatClass(MapFileOutputFormat.class);
    MapFileOutputFormat.setOutputPath(job, new Path(dBOutputPath));
    //TODO: add compression to MapFile
    job.setNumReduceTasks(0);
    DBConfiguration.configureDB(conf, dBConfig.getString(DB_CONFIG_DRIVER), dBConfig.getString(DB_CONFIG_URL),
            dBConfig.getString(DB_CONFIG_USER), dBConfig.getString(DB_CONFIG_PASSWORD));
    String[] fields = { "id", "name", "category", "price" };
    DBInputFormat.setInput(job, DBInputWritable.class, dBConfig.getString(DB_CONFIG_TABLE_NAME), null, "id",
            fields);

    return job.waitForCompletion(true);
}

---

From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOIT.java

License:Apache License

private static void setupHadoopConfiguration(PostgresIOTestPipelineOptions options) {
    Configuration conf = new Configuration();
    DBConfiguration.configureDB(conf, "org.postgresql.Driver", DatabaseTestHelper.getPostgresDBUrl(options),
            options.getPostgresUsername(), options.getPostgresPassword());

    conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tableName);
    conf.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, "id", "name");
    conf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, "id ASC");
    conf.setClass(DBConfiguration.INPUT_CLASS_PROPERTY, TestRowDBWritable.class, DBWritable.class);

    conf.setClass("key.class", LongWritable.class, Object.class);
    conf.setClass("value.class", TestRowDBWritable.class, Object.class);
    conf.setClass("mapreduce.job.inputformat.class", DBInputFormat.class, InputFormat.class);

    conf.set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, tableName);
    conf.set(DBConfiguration.OUTPUT_FIELD_COUNT_PROPERTY, "2");
    conf.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, "id", "name");

    conf.setClass(HadoopFormatIO.OUTPUT_KEY_CLASS, TestRowDBWritable.class, Object.class);
    conf.setClass(HadoopFormatIO.OUTPUT_VALUE_CLASS, NullWritable.class, Object.class);
    conf.setClass(HadoopFormatIO.OUTPUT_FORMAT_CLASS_ATTR, DBOutputFormat.class, OutputFormat.class);
    conf.set(HadoopFormatIO.JOB_ID, String.valueOf(1));

    hadoopConfiguration = new SerializableConfiguration(conf);
}

---

From source file:org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT.java

License:Apache License

private static void setupHadoopConfiguration(PostgresIOTestPipelineOptions options) {
    Configuration conf = new Configuration();
    DBConfiguration.configureDB(conf, "org.postgresql.Driver", DatabaseTestHelper.getPostgresDBUrl(options),
            options.getPostgresUsername(), options.getPostgresPassword());
    conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tableName);
    conf.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, "id", "name");
    conf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, "id ASC");
    conf.setClass(DBConfiguration.INPUT_CLASS_PROPERTY, TestRowDBWritable.class, DBWritable.class);

    conf.setClass("key.class", LongWritable.class, Object.class);
    conf.setClass("value.class", TestRowDBWritable.class, Object.class);
    conf.setClass("mapreduce.job.inputformat.class", DBInputFormat.class, InputFormat.class);

    hadoopConfiguration = new SerializableConfiguration(conf);
}

---