List of usage examples for org.apache.hadoop.mapreduce TaskInputOutputContext getConfiguration
public Configuration getConfiguration();
From source file:org.apache.crunch.io.hcatalog.HCatRecordDataReadable.java
License:Apache License
@Override public Iterable<HCatRecord> read(TaskInputOutputContext<?, ?, ?, ?> context) throws IOException { return new HCatRecordDataIterable(bundle, context.getConfiguration()); }
From source file:org.apache.crunch.io.hcatalog.HCatSourceITSpec.java
License:Apache License
@Test public void testReadable() throws Exception { String tableName = testName.getMethodName(); Path tableRootLocation = temporaryPath.getPath(tableName); String data = "17,josh\n29,indiana\n"; writeDataToHdfs(data, tableRootLocation, conf); HCatTestUtils.createUnpartitionedTable(client, tableName, TableType.MANAGED_TABLE, tableRootLocation); Pipeline p = new MRPipeline(HCatSourceITSpec.class, conf); HCatSourceTarget src = (HCatSourceTarget) FromHCat.table(tableName); HCatSchema schema = src.getTableSchema(p.getConfiguration()); PCollection<HCatRecord> records = p.read(src); ReadableData<HCatRecord> readable = records.asReadable(true); TaskInputOutputContext mockTIOC = Mockito.mock(TaskInputOutputContext.class); when(mockTIOC.getConfiguration()).thenReturn(conf); readable.configure(conf);//from w ww . ja va 2 s. c o m Iterator<HCatRecord> iterator = readable.read(mockTIOC).iterator(); HCatTestUtils.Fns.MapPairFn fn = new HCatTestUtils.Fns.MapPairFn(schema); List<Pair<Integer, String>> results = new ArrayList<>(); while (iterator.hasNext()) { results.add(fn.map(iterator.next())); } p.done(); assertEquals(ImmutableList.of(Pair.of(17, "josh"), Pair.of(29, "indiana")), results); }
From source file:org.apache.crunch.io.impl.ReadableDataImpl.java
License:Apache License
@Override public Iterable<T> read(TaskInputOutputContext<?, ?, ?, ?> ctxt) throws IOException { final Configuration conf = ctxt.getConfiguration(); final FileReaderFactory<T> readerFactory = getFileReaderFactory(); return Iterables.concat(Lists.transform(paths, new Function<String, Iterable<T>>() { @Override//from ww w. j a va 2 s. co m public Iterable<T> apply(@Nullable String input) { Path path = getCacheFilePath(input, conf); try { FileSystem fs = path.getFileSystem(conf); return CompositePathIterable.create(fs, path, readerFactory); } catch (IOException e) { throw new RuntimeException(e); } } })); }
From source file:org.apache.crunch.test.CrunchTestSupport.java
License:Apache License
/** * The method creates a {@linkplain TaskInputOutputContext} which can be used * in unit tests. The context serves very limited purpose. You can only * operate with counters, taskAttempId, status and configuration while using * this context./*ww w . j a va 2 s . co m*/ */ public static <KI, VI, KO, VO> TaskInputOutputContext<KI, VI, KO, VO> getTestContext( final Configuration config) { TaskInputOutputContext<KI, VI, KO, VO> context = Mockito.mock(TaskInputOutputContext.class); TestCounters.clearCounters(); final StateHolder holder = new StateHolder(); Mockito.when(context.getCounter(Mockito.any(Enum.class))).then(new Answer<Counter>() { @Override public Counter answer(InvocationOnMock invocation) throws Throwable { Enum<?> counter = (Enum<?>) invocation.getArguments()[0]; return TestCounters.getCounter(counter); } }); Mockito.when(context.getCounter(Mockito.anyString(), Mockito.anyString())).then(new Answer<Counter>() { @Override public Counter answer(InvocationOnMock invocation) throws Throwable { String group = (String) invocation.getArguments()[0]; String name = (String) invocation.getArguments()[1]; return TestCounters.getCounter(group, name); } }); Mockito.when(context.getConfiguration()).thenReturn(config); Mockito.when(context.getTaskAttemptID()).thenReturn(new TaskAttemptID()); Mockito.when(context.getStatus()).then(new Answer<String>() { @Override public String answer(InvocationOnMock invocation) throws Throwable { return holder.internalStatus; } }); Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { holder.internalStatus = (String) invocation.getArguments()[0]; return null; } }).when(context).setStatus(Mockito.anyString()); return context; }
From source file:org.apache.crunch.test.CrunchTestSupportTest.java
License:Apache License
@Test public void testContext() { Configuration sampleConfig = new Configuration(); String status = "test"; TaskInputOutputContext<?, ?, ?, ?> testContext = CrunchTestSupport.getTestContext(sampleConfig); assertEquals(sampleConfig, testContext.getConfiguration()); TaskAttemptID taskAttemptID = testContext.getTaskAttemptID(); assertEquals(taskAttemptID, testContext.getTaskAttemptID()); assertNotNull(taskAttemptID);//from www .j av a 2 s. c om assertNull(testContext.getStatus()); testContext.setStatus(status); assertEquals(status, testContext.getStatus()); assertEquals(0, testContext.getCounter(CT.ONE).getValue()); testContext.getCounter(CT.ONE).increment(1); assertEquals(1, testContext.getCounter(CT.ONE).getValue()); testContext.getCounter(CT.ONE).increment(4); assertEquals(5, testContext.getCounter(CT.ONE).getValue()); }
From source file:org.apache.kudu.mapreduce.KuduTableMapReduceUtil.java
License:Apache License
/** * Use this method when setting up a task to get access to the KuduTable in order to create * Inserts, Updates, and Deletes.//from w w w .ja v a 2s .c om * @param context Map context * @return The kudu table object as setup by the output format */ @SuppressWarnings("rawtypes") public static KuduTable getTableFromContext(TaskInputOutputContext context) { String multitonKey = context.getConfiguration().get(KuduTableOutputFormat.MULTITON_KEY); return KuduTableOutputFormat.getKuduTable(multitonKey); }
From source file:org.apache.nutch.mapreduce.NutchCounter.java
License:Apache License
@SuppressWarnings("rawtypes") public NutchCounter(TaskInputOutputContext context) { this.context = context; this.conf = context.getConfiguration(); this.id = counterSequence.incrementAndGet(); this.name = "NutchCounter" + "-" + id; String jobName = context.getJobName(); jobName = StringUtils.substringBeforeLast(jobName, "-"); jobName = jobName.replaceAll("(\\[.+\\])", ""); this.LOG = LoggerFactory.getLogger(name + "-" + jobName); this.hostname = NetUtil.getHostname(); crawlFilters = CrawlFilters.create(conf); }
From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl.java
License:Apache License
public MapReducePOStoreImpl(TaskInputOutputContext<?, ?, ?, ?> context) { // get a copy of the Configuration so that changes to the // configuration below (like setting the output location) do // not affect the caller's copy Configuration outputConf = new Configuration(context.getConfiguration()); reporter = PigStatusReporter.getInstance(); reporter.setContext(new MRTaskContext(context)); // make a copy of the Context to use here - since in the same // task (map or reduce) we could have multiple stores, we should // make this copy so that the same context does not get over-written // by the different stores. this.context = HadoopShims.createTaskAttemptContext(outputConf, context.getTaskAttemptID()); }
From source file:org.apache.sqoop.mapreduce.ProgressThread.java
License:Apache License
public ProgressThread(final TaskInputOutputContext ctxt, Log log) { this.context = ctxt; this.log = log; this.keepGoing = true; configureAutoProgress(ctxt.getConfiguration()); }
From source file:org.kiji.mapreduce.context.DirectKijiTableWriterContext.java
License:Apache License
/** * Constructs a new context that can write cells directly to a Kiji table. * * @param hadoopContext is the Hadoop {@link TaskInputOutputContext} that will be used to perform * the writes./*from w ww .ja v a2 s . c o m*/ * @throws IOException on I/O error. */ public DirectKijiTableWriterContext(TaskInputOutputContext<?, ?, ?, ?> hadoopContext) throws IOException { super(hadoopContext); final Configuration conf = new Configuration(hadoopContext.getConfiguration()); final KijiURI outputURI = KijiURI.newBuilder(conf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build(); mKiji = Kiji.Factory.open(outputURI, conf); mTable = mKiji.openTable(outputURI.getTable()); mPutter = mTable.openTableWriter(); mEntityIdFactory = mTable.getEntityIdFactory(); }