List of usage examples for org.apache.hadoop.mapred JobConf JobConf
public JobConf()
From source file:cascading.tuple.hadoop.TupleSerialization.java
License:Open Source License
@Override public Configuration getConf() { if (super.getConf() == null) setConf(new JobConf()); return super.getConf(); }
From source file:cascading.tuple.SpillableTupleTest.java
License:Open Source License
public void testSpillCompressed() { GzipCodec codec = ReflectionUtils.newInstance(GzipCodec.class, new JobConf()); long time = System.currentTimeMillis(); performSpillTest(5, 50, codec);// w w w .j av a 2 s .c om performSpillTest(49, 50, codec); performSpillTest(50, 50, codec); performSpillTest(51, 50, codec); performSpillTest(499, 50, codec); performSpillTest(500, 50, codec); performSpillTest(501, 50, codec); System.out.println("time = " + (System.currentTimeMillis() - time)); }
From source file:cascading.util.Util.java
License:Open Source License
public static Thread getHDFSShutdownHook() { Exception caughtException = null; try {// ww w . j a v a 2 s.co m // we must init the FS so the finalizer is registered FileSystem.getLocal(new JobConf()); Field field = FileSystem.class.getDeclaredField("clientFinalizer"); field.setAccessible(true); Thread finalizer = (Thread) field.get(null); if (finalizer != null) Runtime.getRuntime().removeShutdownHook(finalizer); return finalizer; } catch (NoSuchFieldException exception) { caughtException = exception; } catch (IllegalAccessException exception) { caughtException = exception; } catch (IOException exception) { caughtException = exception; } LOG.info("unable to find and remove client hdfs shutdown hook, received exception: " + caughtException.getClass().getName()); return null; }
From source file:co.cask.cdap.data.runtime.main.ResourcesClassLoaderTest.java
License:Apache License
@SuppressWarnings("AccessStaticViaInstance") @Test// w w w . ja va 2s . com public void testCustomResourceLoading() throws Exception { // Using default classloader JobConf jobConf = new JobConf(); // foo-loader is not defined in default classloader Assert.assertNull(jobConf.get("foo-loader")); // On first load, TestClass.init should be false Assert.assertFalse(TestClass.init); TestClass.init = true; // Using ResourcesClassLoader with URL /test-conf URL url = getClass().getResource("/test-conf/mapred-site.xml"); ClassLoader previousClassLoader = ClassLoaders.setContextClassLoader( new ResourcesClassLoader(new URL[] { getParentUrl(url) }, getClass().getClassLoader())); jobConf = new JobConf(); Assert.assertEquals("bar-loader", jobConf.get("foo-loader")); // TestClass is already initialzed earlier, hence TestClass.init should be true TestClass testClass = (TestClass) Thread.currentThread().getContextClassLoader() .loadClass(TestClass.class.getName()).newInstance(); Assert.assertTrue(testClass.init); ClassLoaders.setContextClassLoader(previousClassLoader); // Using ResourcesClassLoader with URL /test-app-conf url = getClass().getResource("/test-app-conf/mapred-site.xml"); previousClassLoader = ClassLoaders.setContextClassLoader( new ResourcesClassLoader(new URL[] { getParentUrl(url) }, getClass().getClassLoader())); jobConf = new JobConf(); Assert.assertEquals("baz-app-loader", jobConf.get("foo-loader")); // TestClass is already initialzed earlier, hence TestClass.init should be true testClass = (TestClass) Thread.currentThread().getContextClassLoader().loadClass(TestClass.class.getName()) .newInstance(); Assert.assertTrue(testClass.init); ClassLoaders.setContextClassLoader(previousClassLoader); }
From source file:co.cask.hydrator.plugin.CopybookTest.java
License:Apache License
@BeforeClass public static void setupTest() throws Exception { setupBatchArtifacts(BATCH_APP_ARTIFACT_ID, ETLBatchApplication.class); // add artifact for batch sources and sinks addPluginArtifact(NamespaceId.DEFAULT.artifact("copybookreader-plugins", "1.0.0"), BATCH_APP_ARTIFACT_ID, CopybookSource.class); FileInputFormat.setInputPaths(new JobConf(), new Path("src/test/resources")); }
From source file:colossal.pipe.AvroGroupPartitionerTests.java
License:Open Source License
@SuppressWarnings("deprecation") @Before//w w w . j av a 2 s .c om public void setup() { partitioner = new AvroGroupPartitioner<GenericData.Record, GenericData.Record>(); keyRec = new Record( Schema.parse(("{'type':'record', 'name':'key', 'fields': [ {'name': 'group', 'type': 'string'}," + " {'name' : 'extra', 'type' : 'string'}," + " {'name' : 'subsort', 'type' : 'string'}]}") .replaceAll("\\'", "\""))); valRec = new Record( Schema.parse(("{'type':'record', 'name':'val', 'fields': [ {'name': 'group', 'type': 'string'}," + " {'name' : 'extra', 'type' : 'string'}," + " {'name' : 'extra2', 'type' : 'string'}," + " {'name' : 'subsort', 'type' : 'string'}]}").replaceAll("\\'", "\""))); key = new AvroKey<Record>(keyRec); value = new AvroValue<Record>(valRec); keyRec.put("group", "A"); keyRec.put("subsort", "one"); valRec.put("extra", "one"); valRec.put("group", "A"); valRec.put("subsort", "one"); valRec.put("extra", "one"); valRec.put("extra2", "one"); conf = new JobConf(); }
From source file:com.acme.extensions.mr.WordCount.java
License:Apache License
public static void main(String[] args) throws Exception { JobConf job = new JobConf(); String[] otherArgs = new GenericOptionsParser(job, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: wordcount <in> <out>"); System.exit(2);//w ww . j a v a2 s.c o m } job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); JobClient.runJob(job); }
From source file:com.aerospike.spark.examples.SparkSessionRollup.java
License:Apache License
public static void main(String[] args) { com.aerospike.client.Log.setCallback(new AerospikeLogger()); com.aerospike.client.Log.setLevel(com.aerospike.client.Log.Level.DEBUG); SparkConf conf = new SparkConf().setAppName(appName).set("spark.executor.memory", "2g").setMaster(master); JavaSparkContext sc = new JavaSparkContext(conf); sc.addJar("build/libs/spark_session_rollup-1.0.0-worker.jar"); JavaRDD<String> entries = sc.textFile("hdfs://as0:9000/worldcup"); JavaPairRDD<Long, Iterable<Long>> userhits = entries.mapToPair(new ExtractHits()).groupByKey(); JavaPairRDD<String, Session> sessions = userhits.flatMapToPair(new FindSessions()); System.err.println(sessions.count()); JobConf job = new JobConf(); job.setOutputKeyClass(String.class); job.setOutputValueClass(Session.class); job.setOutputFormat(SessionOutputFormat.class); AerospikeConfigUtil.setOutputHost(job, "localhost"); AerospikeConfigUtil.setOutputPort(job, 3000); AerospikeConfigUtil.setOutputNamespace(job, "test"); AerospikeConfigUtil.setOutputSetName(job, "sessions3"); sessions.saveAsHadoopDataset(job);/*from w w w . j av a2 s.c o m*/ }
From source file:com.alexholmes.hadooputils.combine.avro.mapred.CombineAvroInputFormatTest.java
License:Apache License
@SuppressWarnings("deprecation") public void testJob() throws Exception { JobConf job = new JobConf(); String dir = System.getProperty("test.dir", ".") + "/mapred"; Path outputPath = new Path(dir + "/out"); outputPath.getFileSystem(job).delete(outputPath); WordCountUtil.writeLinesFile();//w ww . j a va2 s . c o m job.setJobName("wordcount"); AvroJob.setInputSchema(job, Schema.create(Schema.Type.STRING)); AvroJob.setOutputSchema(job, new Pair<Utf8, Long>(new Utf8(""), 0L).getSchema()); AvroJob.setMapperClass(job, MapImpl.class); AvroJob.setCombinerClass(job, ReduceImpl.class); AvroJob.setReducerClass(job, ReduceImpl.class); job.setInputFormat(CombineAvroInputFormat.class); FileInputFormat.setInputPaths(job, new Path(dir + "/in")); FileOutputFormat.setOutputPath(job, outputPath); FileOutputFormat.setCompressOutput(job, true); WordCountUtil.setMeta(job); JobClient.runJob(job); WordCountUtil.validateCountsFile(); }
From source file:com.alexholmes.hadooputils.combine.avro.mapred.CombineAvroInputFormatTest.java
License:Apache License
@SuppressWarnings("deprecation") public void testProjection() throws Exception { JobConf job = new JobConf(); Integer defaultRank = new Integer(-1); String jsonSchema = "{\"type\":\"record\"," + "\"name\":\"org.apache.avro.mapred.Pair\"," + "\"fields\": [ " + "{\"name\":\"rank\", \"type\":\"int\", \"default\": -1}," + "{\"name\":\"value\", \"type\":\"long\"}" + "]}"; Schema readerSchema = Schema.parse(jsonSchema); AvroJob.setInputSchema(job, readerSchema); String dir = System.getProperty("test.dir", ".") + "/mapred"; Path inputPath = new Path(dir + "/out" + "/part-00000" + AvroOutputFormat.EXT); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroRecordReader<Pair<Integer, Long>> recordReader = new AvroRecordReader<Pair<Integer, Long>>(job, fileSplit);/*from www . j a v a 2s.co m*/ AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<Pair<Integer, Long>>(null); NullWritable ignore = NullWritable.get(); long sumOfCounts = 0; long numOfCounts = 0; while (recordReader.next(inputPair, ignore)) { Assert.assertEquals((Integer) inputPair.datum().get(0), defaultRank); sumOfCounts += (Long) inputPair.datum().get(1); numOfCounts++; } Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size()); long actualSumOfCounts = 0; for (Long count : WordCountUtil.COUNTS.values()) { actualSumOfCounts += count; } Assert.assertEquals(sumOfCounts, actualSumOfCounts); }