List of usage examples for org.apache.hadoop.mapreduce.lib.db DBOutputFormat setOutput
public static void setOutput(Job job, String tableName, int fieldCount) throws IOException
From source file:cn.itcast.hadoop.mr.wordcount.DBCountPageView.java
License:Apache License
@Override //Usage DBCountPageView [driverClass dburl] public int run(String[] args) throws Exception { //?MySql/*from w w w . j a va 2s . c om*/ String driverClassName = DRIVER_CLASS; String url = DB_URL; //?? //???? if (args.length > 1) { driverClassName = args[0]; url = args[1]; } //driverClassNameurl?? initialize(driverClassName, url); //hdfs? Configuration conf = getConf(); //?? DBConfiguration.configureDB(conf, driverClassName, url); //??? //job Job job = Job.getInstance(conf); //job?? job.setJobName("Count Pageviews of URLs"); //job job.setJarByClass(DBCountPageView.class); //Map job.setMapperClass(PageviewMapper.class); //Combiner job.setCombinerClass(LongSumReducer.class); //reduce job.setReducerClass(PageviewReducer.class); //DB? // setInput(Job job, Class<? extends DBWritable> inputClass, String tableName, String conditions, String orderBy, String... fieldNames) DBInputFormat.setInput(job, AccessRecord.class, "HAccess", null, "url", AccessFieldNames); //? //FileOutputFormat.setoutput ? DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);// //Mapkey? job.setMapOutputKeyClass(Text.class); //MapValue? job.setMapOutputValueClass(LongWritable.class); //Reducekey? job.setOutputKeyClass(PageviewRecord.class); //Reducevalue? job.setOutputValueClass(NullWritable.class); int ret;//job????? try { ret = job.waitForCompletion(true) ? 0 : 1; boolean correct = verify(); if (!correct) { throw new RuntimeException("Evaluation was not correct!"); } } finally { shutdown(); } return ret; }
From source file:com.phantom.hadoop.examples.DBCountPageView.java
License:Apache License
@Override // Usage DBCountPageView [driverClass dburl] public int run(String[] args) throws Exception { String driverClassName = DRIVER_CLASS; String url = DB_URL;/*from w w w . j av a 2 s .c o m*/ if (args.length > 1) { driverClassName = args[0]; url = args[1]; } initialize(driverClassName, url); Configuration conf = getConf(); DBConfiguration.configureDB(conf, driverClassName, url); Job job = new Job(conf); job.setJobName("Count Pageviews of URLs"); job.setJarByClass(DBCountPageView.class); job.setMapperClass(PageviewMapper.class); job.setCombinerClass(LongSumReducer.class); job.setReducerClass(PageviewReducer.class); DBInputFormat.setInput(job, AccessRecord.class, "Access", null, "url", AccessFieldNames); DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(PageviewRecord.class); job.setOutputValueClass(NullWritable.class); int ret; try { ret = job.waitForCompletion(true) ? 0 : 1; boolean correct = verify(); if (!correct) { throw new RuntimeException("Evaluation was not correct!"); } } finally { shutdown(); } return ret; }
From source file:hadoop.examples.DBCountPageView.java
License:Apache License
public int run(String[] args) throws Exception { String driverClassName = DRIVER_CLASS; String url = DB_URL;//from ww w . j a v a 2s . c om if (args.length > 1) { driverClassName = args[0]; url = args[1]; } initialize(driverClassName, url); Configuration conf = getConf(); DBConfiguration.configureDB(conf, driverClassName, url); Job job = new Job(conf); job.setJobName("Count Pageviews of URLs"); job.setJarByClass(DBCountPageView.class); job.setMapperClass(PageviewMapper.class); job.setCombinerClass(LongSumReducer.class); job.setReducerClass(PageviewReducer.class); DBInputFormat.setInput(job, AccessRecord.class, "Access", null, "url", AccessFieldNames); DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(PageviewRecord.class); job.setOutputValueClass(NullWritable.class); int ret; try { ret = job.waitForCompletion(true) ? 0 : 1; boolean correct = verify(); if (!correct) { throw new RuntimeException("Evaluation was not correct!"); } } finally { shutdown(); } return ret; }
From source file:io.apigee.lembos.node.types.DBOutputFormatWrap.java
License:Apache License
/** * Java wrapper for {@link DBOutputFormat#setOutput(org.apache.hadoop.mapreduce.Job, String, int)} and * {@link DBOutputFormat#setOutput(org.apache.hadoop.mapreduce.Job, String, String...)}. * * @param ctx the JavaScript context//from w w w. ja v a 2 s . c o m * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called */ @JSStaticFunction public static void setOutput(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { final Object arg0 = args.length >= 1 ? args[0] : Undefined.instance; final Object arg1 = args.length >= 2 ? args[1] : Undefined.instance; final Object arg2 = args.length >= 3 ? args[2] : Undefined.instance; if (args.length != 3) { throw Utils.makeError(ctx, thisObj, LembosMessages.THREE_ARGS_EXPECTED); } else if (!JavaScriptUtils.isDefined(arg0)) { throw Utils.makeError(ctx, thisObj, LembosMessages.FIRST_ARG_REQUIRED); } else if (!JavaScriptUtils.isDefined(arg1)) { throw Utils.makeError(ctx, thisObj, LembosMessages.SECOND_ARG_REQUIRED); } else if (!JavaScriptUtils.isDefined(arg0)) { throw Utils.makeError(ctx, thisObj, LembosMessages.THIRD_ARG_REQUIRED); } else if (!(arg0 instanceof JobWrap)) { throw Utils.makeError(ctx, thisObj, LembosMessages.FIRST_ARG_MUST_BE_JOB); } else if (!(arg2 instanceof NativeArray) && !(arg2 instanceof Number)) { throw Utils.makeError(ctx, thisObj, LembosMessages.THIRD_ARG_MUST_BE_ARR_OR_NUM); } try { if (arg2 instanceof NativeArray) { final NativeArray jsFieldNames = (NativeArray) arg2; final String[] fieldNames = new String[jsFieldNames.size()]; for (int i = 0; i < jsFieldNames.size(); i++) { fieldNames[i] = jsFieldNames.get(i).toString(); } DBOutputFormat.setOutput(((JobWrap) arg0).getJob(), arg1.toString(), fieldNames); } else { DBOutputFormat.setOutput(((JobWrap) arg0).getJob(), arg1.toString(), JavaScriptUtils.fromNumber(arg2).intValue()); } } catch (IOException e) { throw Utils.makeError(ctx, thisObj, e.getMessage()); } }
From source file:link.neolink.datamonster.Datamonster.java
License:Apache License
/** * Reducer configuration, with SQL// www. ja v a 2 s . c om * * @param outType * Kind of output query, either "INSERT", "UPDATE" or "DELINSERT" * @param outTable * The output table * @param outFields * The output table's fields * @param reducer * The reducer class */ public void reducerConfiguration(String outType, String outTable, String[] outFields, Class<? extends Reducer> reducer) throws Exception { if (this.outType != Type.MAGIC) { System.err.println("Output type already defined"); System.exit(-1); } this.job.setReducerClass(reducer); this.setOutType(outType); job.setOutputValueClass(NullWritable.class); this.outTable = outTable; if (this.outputWritable != null) { job.setOutputKeyClass(this.outputWritable); } else if (this.ioWritable != null) { job.setOutputKeyClass(this.ioWritable); } else { System.err.println("Output is SQL, but no output Writable class defined"); System.exit(-1); } // Define the OutputFormatClass (it's not the same between an INSERT/DELINSERT and a UPDATE) if (this.outType == Type.INSERT || this.outType == Type.DELINSERT) { this.job.setOutputFormatClass(DBOutputFormat.class); } else if (this.outType == Type.UPDATE) { this.job.setOutputFormatClass(DBUpdateOutputFormat.class); } // Configura the DBOutputFormat (same thing, it differs bewteen INSERT and UPDATE) if (this.outType == Type.INSERT || this.outType == Type.DELINSERT) { DBOutputFormat.setOutput(job, outTable, outFields); } else if (this.outType == Type.UPDATE) { DBUpdateOutputFormat.setOutput(job, outTable, outFields); } }
From source file:org.apache.hadoop.examples.DBCountPageView.java
License:Apache License
@Override //Usage DBCountPageView [driverClass dburl] public int run(String[] args) throws Exception { String driverClassName = DRIVER_CLASS; String url = DB_URL;//from ww w. ja v a2s . c o m if (args.length > 1) { driverClassName = args[0]; url = args[1]; } initialize(driverClassName, url); Configuration conf = getConf(); DBConfiguration.configureDB(conf, driverClassName, url); Job job = Job.getInstance(conf); job.setJobName("Count Pageviews of URLs"); job.setJarByClass(DBCountPageView.class); job.setMapperClass(PageviewMapper.class); job.setCombinerClass(LongSumReducer.class); job.setReducerClass(PageviewReducer.class); DBInputFormat.setInput(job, AccessRecord.class, "HAccess", null, "url", AccessFieldNames); DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setOutputKeyClass(PageviewRecord.class); job.setOutputValueClass(NullWritable.class); int ret; try { ret = job.waitForCompletion(true) ? 0 : 1; boolean correct = verify(); if (!correct) { throw new RuntimeException("Evaluation was not correct!"); } } finally { shutdown(); } return ret; }