List of usage examples for org.apache.hadoop.mapred JobConf JobConf
public JobConf()
From source file:com.google.mr4c.hadoop.ClusterTest.java
License:Open Source License
@Test public void testRoundTrip() throws Exception { JobConf conf = new JobConf(); conf.clear();/*from ww w .j a v a 2 s.c o m*/ m_cluster1.applyToConfig(conf); Cluster cluster = Cluster.extractFromConfig(conf); assertEquals(m_cluster1, cluster); }
From source file:com.google.mr4c.hadoop.HadoopAlgoRunner.java
License:Open Source License
protected HadoopAlgoRunner(String[] args, Logger log) { m_fullArgs = args;/*www . ja v a 2s . c om*/ m_log = log; m_jobConf = new JobConf(); m_bbJob = new MR4CMRJob(false); // not on cluster }
From source file:com.google.mr4c.hadoop.yarn.YarnTestBinding.java
License:Open Source License
private void startMrCluster() throws IOException { Configuration conf = new JobConf(); FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri()); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); String addr = MiniYARNCluster.getHostname() + ":0"; conf.set(YarnConfiguration.RM_ADDRESS, addr); conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr); m_mrCluster = MiniMRClientClusterFactory.create(HadoopTestUtils.class, "MR4CTests", 1, // num node managers conf);// ww w . j a v a2 s .c o m // make sure startup is finished for (int i = 0; i < 60; i++) { String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); if (newAddr.equals(addr)) { s_log.warn("MiniYARNCluster startup not complete"); try { Thread.sleep(1000); } catch (InterruptedException ie) { throw new IOException(ie); } } else { s_log.info("MiniYARNCluster now available at {}", newAddr); return; } } throw new IOException("MiniYARNCluster taking too long to startup"); }
From source file:com.hadoop.mapreduce.TestLzoLazyLoading.java
License:Open Source License
public void testWithLocal() throws Exception { MiniMRCluster mr = null;// ww w. j a v a 2s . c om try { JobConf jconf = new JobConf(); jconf.set("mapred.queue.names", "default"); mr = new MiniMRCluster(2, "file:///", 3, null, null, jconf); Configuration cf = mr.createJobConf(); cf.set("io.compression.codecs", LzoCodec.class.getName()); runWordCount(cf, false, false); runWordCount(cf, false, true); runWordCount(cf, true, false); } finally { if (mr != null) { mr.shutdown(); } } }
From source file:com.hazelcast.jet.benchmark.wordcount.HdfsToMap.java
License:Open Source License
private static void fillMap(JetInstance client, String name, String inputPath, int parallelism) throws Exception { DAG dag = new DAG(); JobConf conf = new JobConf(); conf.setInputFormat(TextInputFormat.class); TextInputFormat.addInputPath(conf, new Path(inputPath)); Vertex reader = dag.newVertex("reader", readHdfsP(conf, Util::entry)); Vertex mapper = dag.newVertex("mapper", mapP((Map.Entry<LongWritable, Text> e) -> entry(e.getKey().get(), e.getValue().toString()))); Vertex writer = dag.newVertex("writer", writeMapP(name)); reader.localParallelism(parallelism); mapper.localParallelism(parallelism); writer.localParallelism(parallelism); dag.edge(between(reader, mapper));// ww w . jav a 2s. c o m dag.edge(between(mapper, writer)); JobConfig jobConfig = new JobConfig(); jobConfig.addClass(HdfsToMap.class); client.newJob(dag, jobConfig).join(); }
From source file:com.hazelcast.jet.hadoop.impl.ReadHdfsPTest.java
License:Open Source License
@Before public void setup() throws IOException { instance = createJetMember();/*from w ww. j a v a 2 s .c o m*/ jobConf = new JobConf(); jobConf.setInputFormat(inputFormatClass); writeToFile(); for (Path path : paths) { FileInputFormat.addInputPath(jobConf, path); } }
From source file:com.hazelcast.jet.hadoop.impl.WriteHdfsPTest.java
License:Open Source License
@Test public void testWriteFile() throws Exception { int messageCount = 320; String mapName = randomMapName(); JetInstance instance = createJetMember(); createJetMember();/*from w ww .ja v a 2 s . c o m*/ Map<IntWritable, IntWritable> map = IntStream.range(0, messageCount).boxed() .collect(toMap(IntWritable::new, IntWritable::new)); instance.getMap(mapName).putAll(map); Path path = getPath(); JobConf conf = new JobConf(); conf.setOutputFormat(outputFormatClass); conf.setOutputCommitter(FileOutputCommitter.class); conf.setOutputKeyClass(IntWritable.class); conf.setOutputValueClass(IntWritable.class); if (outputFormatClass.equals(LazyOutputFormat.class)) { LazyOutputFormat.setOutputFormatClass(conf, TextOutputFormat.class); } FileOutputFormat.setOutputPath(conf, path); Pipeline p = Pipeline.create(); p.drawFrom(Sources.map(mapName)).drainTo(HdfsSinks.hdfs(conf)) // we use higher value to increase the race chance for LazyOutputFormat .setLocalParallelism(8); Future<Void> future = instance.newJob(p).getFuture(); assertCompletesEventually(future); JobConf readJobConf = new JobConf(); readJobConf.setInputFormat(inputFormatClass); FileInputFormat.addInputPath(readJobConf, path); p = Pipeline.create(); p.drawFrom(HdfsSources.hdfs(readJobConf)).drainTo(Sinks.list("results")); future = instance.newJob(p).getFuture(); assertCompletesEventually(future); IList<Object> results = instance.getList("results"); assertEquals(messageCount, results.size()); }
From source file:com.hazelcast.jet.impl.connector.hadoop.WriteHdfsPTest.java
License:Open Source License
@Test public void testWriteFile() throws Exception { int messageCount = 20; String mapName = randomMapName(); JetInstance instance = createJetMember(); createJetMember();/*from w w w .j a va 2 s . c o m*/ Map<IntWritable, IntWritable> map = IntStream.range(0, messageCount).boxed() .collect(toMap(IntWritable::new, IntWritable::new)); instance.getMap(mapName).putAll(map); DAG dag = new DAG(); Vertex producer = dag.newVertex("producer", readMap(mapName)).localParallelism(1); Path path = getPath(); JobConf conf = new JobConf(); conf.setOutputFormat(outputFormatClass); conf.setOutputCommitter(FileOutputCommitter.class); conf.setOutputKeyClass(IntWritable.class); conf.setOutputValueClass(IntWritable.class); FileOutputFormat.setOutputPath(conf, path); Vertex consumer = dag.newVertex("consumer", writeHdfs(conf)).localParallelism(4); dag.edge(between(producer, consumer)); Future<Void> future = instance.newJob(dag).execute(); assertCompletesEventually(future); dag = new DAG(); JobConf readJobConf = new JobConf(); readJobConf.setInputFormat(inputFormatClass); FileInputFormat.addInputPath(readJobConf, path); producer = dag.newVertex("producer", readHdfs(readJobConf)).localParallelism(8); consumer = dag.newVertex("consumer", writeList("results")).localParallelism(1); dag.edge(between(producer, consumer)); future = instance.newJob(dag).execute(); assertCompletesEventually(future); IList<Object> results = instance.getList("results"); assertEquals(messageCount, results.size()); }
From source file:com.hotels.corc.cascading.OrcFileSourcePerformanceTest.java
License:Apache License
@Before public void before() throws IOException { structTypeInfo = createTypeInfo();//from www. java2 s . c om writeOrcFile(); tap = createTap(); when(flowProcess.getConfigCopy()).thenReturn(new JobConf()); }
From source file:com.hotels.plunger.TapDataReader.java
License:Apache License
private TupleEntryIterator getHadoopTupleEntryIterator() throws IOException { @SuppressWarnings("unchecked") Tap<JobConf, ?, ?> hadoopTap = (Tap<JobConf, ?, ?>) source; JobConf conf = new JobConf(); FlowProcess<JobConf> flowProcess = new HadoopFlowProcess(conf); hadoopTap.sourceConfInit(flowProcess, conf); return hadoopTap.openForRead(flowProcess); }