List of usage examples for org.apache.hadoop.mapreduce Job isSuccessful
public boolean isSuccessful() throws IOException
From source file:io.vitess.hadoop.MapReduceIT.java
License:Apache License
/** * Run a mapper only MR job and verify all the rows in the source table were outputted into HDFS. *//*from w w w . j ava2 s . c o m*/ public void testDumpTableToHDFS() throws Exception { // Configurations for the job, output from mapper as Text Configuration conf = createJobConf(); Job job = Job.getInstance(conf); job.setJobName("table"); job.setJarByClass(VitessInputFormat.class); job.setMapperClass(TableMapper.class); VitessInputFormat.setInput(job, "localhost:" + testEnv.getPort(), testEnv.getKeyspace(), "select id, name, age from vtgate_test", ImmutableList.<String>of(), 4 /* splitCount */, 0 /* numRowsPerQueryPart */, Algorithm.EQUAL_SPLITS, TestUtil.getRpcClientFactory().getClass()); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(RowWritable.class); job.setOutputFormatClass(TextOutputFormat.class); job.setNumReduceTasks(0); Path outDir = new Path(testEnv.getTestOutputPath(), "mrvitess/output"); FileSystem fs = FileSystem.get(conf); if (fs.exists(outDir)) { fs.delete(outDir, true); } FileOutputFormat.setOutputPath(job, outDir); job.waitForCompletion(true); assertTrue(job.isSuccessful()); String[] outputLines = MapReduceTestUtil.readOutput(outDir, conf).split("\n"); // there should be one line per row in the source table assertEquals(NUM_ROWS, outputLines.length); Set<Long> actualAges = new HashSet<>(); Set<String> actualNames = new HashSet<>(); // Parse and verify we've gotten all the ages and rows. Gson gson = new Gson(); for (String line : outputLines) { String[] parts = line.split("\t"); actualAges.add(Long.valueOf(parts[0])); // Rows are written as JSON since this is TextOutputFormat. String rowJson = parts[1]; Type mapType = new TypeToken<Map<String, String>>() { }.getType(); @SuppressWarnings("unchecked") Map<String, String> map = (Map<String, String>) gson.fromJson(rowJson, mapType); actualNames.add(map.get("name")); } Set<Long> expectedAges = new HashSet<>(); Set<String> expectedNames = new HashSet<>(); for (long i = 1; i <= NUM_ROWS; i++) { // Generate values that match TestUtil.insertRows(). expectedAges.add(i % 10); expectedNames.add("name_" + i); } assertEquals(expectedAges.size(), actualAges.size()); assertTrue(actualAges.containsAll(expectedAges)); assertEquals(NUM_ROWS, actualNames.size()); assertTrue(actualNames.containsAll(expectedNames)); }
From source file:io.vitess.hadoop.MapReduceIT.java
License:Apache License
/** * Map all rows and aggregate by age at the reducer. *//* ww w . j a v a 2s . co m*/ public void testReducerAggregateRows() throws Exception { Configuration conf = createJobConf(); Job job = Job.getInstance(conf); job.setJobName("table"); job.setJarByClass(VitessInputFormat.class); job.setMapperClass(TableMapper.class); VitessInputFormat.setInput(job, "localhost:" + testEnv.getPort(), testEnv.getKeyspace(), "select id, name, age from vtgate_test", ImmutableList.<String>of(), 1 /* splitCount */, 0 /* numRowsPerQueryPart */, Algorithm.EQUAL_SPLITS, TestUtil.getRpcClientFactory().getClass()); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(RowWritable.class); job.setReducerClass(CountReducer.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(IntWritable.class); job.setOutputFormatClass(TextOutputFormat.class); Path outDir = new Path(testEnv.getTestOutputPath(), "mrvitess/output"); FileSystem fs = FileSystem.get(conf); if (fs.exists(outDir)) { fs.delete(outDir, true); } FileOutputFormat.setOutputPath(job, outDir); job.waitForCompletion(true); assertTrue(job.isSuccessful()); String[] outputLines = MapReduceTestUtil.readOutput(outDir, conf).split("\n"); // There should be 10 different ages, because age = i % 10. assertEquals(10, outputLines.length); // All rows should be accounted for. int totalRowsReduced = 0; for (String line : outputLines) { totalRowsReduced += Integer.parseInt(line); } assertEquals(NUM_ROWS, totalRowsReduced); }
From source file:mvm.rya.joinselect.mr.FullTableSize.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); String inTable = conf.get(SPO_TABLE); String outTable = conf.get(SELECTIVITY_TABLE); String auths = conf.get(AUTHS); assert inTable != null && outTable != null; Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); JoinSelectStatsUtil.initTableMRJob(job, inTable, outTable, auths); job.setMapperClass(FullTableMapper.class); job.setCombinerClass(FullTableCombiner.class); job.setReducerClass(FullTableReducer.class); job.setNumReduceTasks(1);//from w w w . j a v a 2s . com job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:mvm.rya.joinselect.mr.JoinSelectAggregate.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); String inPath1 = conf.get(PROSPECTS_OUTPUTPATH); String inPath2 = conf.get(SPO_OUTPUTPATH); String auths = conf.get(AUTHS); String outPath = conf.get(OUTPUTPATH); assert inPath1 != null && inPath2 != null && outPath != null; Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); JoinSelectStatsUtil.initJoinMRJob(job, inPath1, inPath2, JoinSelectAggregateMapper.class, outPath, auths); job.setSortComparatorClass(JoinSelectSortComparator.class); job.setGroupingComparatorClass(JoinSelectGroupComparator.class); job.setPartitionerClass(JoinSelectPartitioner.class); job.setReducerClass(JoinReducer.class); job.setNumReduceTasks(32);// w w w . ja v a 2s. c om job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:mvm.rya.joinselect.mr.JoinSelectProspectOutput.java
License:Apache License
@Override public int run(String[] args) throws AccumuloSecurityException, IOException, ClassNotFoundException, InterruptedException { Configuration conf = getConf(); String inTable = conf.get(PROSPECTS_TABLE); String auths = conf.get(AUTHS); String outPath = conf.get(PROSPECTS_OUTPUTPATH); assert inTable != null && outPath != null; Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths); job.setMapperClass(CardinalityMapper.class); job.setNumReduceTasks(0);//from w w w . j av a 2 s . c o m job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:mvm.rya.joinselect.mr.JoinSelectSpoTableOutput.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); String inTable = conf.get(SPO_TABLE); String auths = conf.get(AUTHS); String outPath = conf.get(SPO_OUTPUTPATH); assert inTable != null && outPath != null; Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths); job.setMapperClass(JoinSelectMapper.class); job.setNumReduceTasks(0);/* ww w . jav a 2s . c o m*/ job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:mvm.rya.joinselect.mr.JoinSelectStatisticsSum.java
License:Apache License
@Override public int run(String[] args) throws AccumuloSecurityException, IOException, ClassNotFoundException, InterruptedException { Configuration conf = getConf(); String outTable = conf.get(SELECTIVITY_TABLE); String auths = conf.get(AUTHS); String inPath = conf.get(INPUTPATH); assert outTable != null && inPath != null; Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); JoinSelectStatsUtil.initSumMRJob(job, inPath, outTable, auths); job.setMapperClass(CardinalityIdentityMapper.class); job.setCombinerClass(CardinalityIdentityCombiner.class); job.setReducerClass(CardinalityIdentityReducer.class); job.setNumReduceTasks(32);//from ww w. j a v a 2 s.c o m job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.examples.filedata.CharacterHistogram.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = Job.getInstance(getConf()); job.setJobName(this.getClass().getSimpleName()); job.setJarByClass(this.getClass()); Opts opts = new Opts(); opts.parseArgs(CharacterHistogram.class.getName(), args); job.setInputFormatClass(ChunkInputFormat.class); opts.setAccumuloConfigs(job);//from ww w . j ava2 s.co m job.getConfiguration().set(VIS, opts.visibilities.toString()); job.setMapperClass(HistMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.examples.mapreduce.NGramIngest.java
License:Apache License
@Override public int run(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(getClass().getName(), args); Job job = Job.getInstance(getConf()); job.setJobName(getClass().getSimpleName()); job.setJarByClass(getClass());/* w ww .j a v a 2s. c om*/ opts.setAccumuloConfigs(job); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(AccumuloOutputFormat.class); job.setMapperClass(NGramMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setNumReduceTasks(0); job.setSpeculativeExecution(false); if (!opts.getConnector().tableOperations().exists(opts.getTableName())) { log.info("Creating table " + opts.getTableName()); opts.getConnector().tableOperations().create(opts.getTableName()); SortedSet<Text> splits = new TreeSet<>(); String numbers[] = "1 2 3 4 5 6 7 8 9".split("\\s"); String lower[] = "a b c d e f g h i j k l m n o p q r s t u v w x y z".split("\\s"); String upper[] = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split("\\s"); for (String[] array : new String[][] { numbers, lower, upper }) { for (String s : array) { splits.add(new Text(s)); } } opts.getConnector().tableOperations().addSplits(opts.getTableName(), splits); } TextInputFormat.addInputPath(job, new Path(opts.inputDirectory)); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
From source file:org.apache.accumulo.examples.mapreduce.RegexExample.java
License:Apache License
@Override public int run(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(getClass().getName(), args); Job job = Job.getInstance(getConf()); job.setJobName(getClass().getSimpleName()); job.setJarByClass(getClass());//from w w w .j av a 2 s. c o m job.setInputFormatClass(AccumuloInputFormat.class); opts.setAccumuloConfigs(job); IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, opts.rowRegex, opts.columnFamilyRegex, opts.columnQualifierRegex, opts.valueRegex, false); AccumuloInputFormat.addIterator(job, regex); job.setMapperClass(RegexMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setNumReduceTasks(0); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, new Path(opts.destination)); System.out.println("setRowRegex: " + opts.rowRegex); System.out.println("setColumnFamilyRegex: " + opts.columnFamilyRegex); System.out.println("setColumnQualifierRegex: " + opts.columnQualifierRegex); System.out.println("setValueRegex: " + opts.valueRegex); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }