List of usage examples for org.apache.hadoop.yarn.util AuxiliaryServiceHelper setServiceDataIntoEnv
public static void setServiceDataIntoEnv(String serviceName, ByteBuffer metaData, Map<String, String> env)
From source file:org.apache.tez.dag.app.launcher.LocalContainerLauncher.java
License:Apache License
public LocalContainerLauncher(AppContext context, TaskAttemptListener taskAttemptListener, String workingDirectory) throws UnknownHostException { super(LocalContainerLauncher.class.getName()); this.context = context; this.taskAttemptListener = taskAttemptListener; this.workingDirectory = workingDirectory; AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(0), localEnv); executionContext = new ExecutionContextImpl(InetAddress.getLocalHost().getHostName()); // User cannot be set here since it isn't available till a DAG is running. }
From source file:org.apache.tez.mapreduce.processor.MapUtils.java
License:Apache License
public static LogicalIOProcessorRuntimeTask createLogicalTask(FileSystem fs, Path workDir, JobConf jobConf, int mapId, Path mapInput, TezUmbilical umbilical, String dagName, String vertexName, List<InputSpec> inputSpecs, List<OutputSpec> outputSpecs) throws Exception { jobConf.setInputFormat(SequenceFileInputFormat.class); ProcessorDescriptor mapProcessorDesc = ProcessorDescriptor.create(MapProcessor.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)); Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(); TaskSpec taskSpec = new TaskSpec(TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0), dagName, vertexName, -1, mapProcessorDesc, inputSpecs, outputSpecs, null); Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>(); serviceConsumerMetadata.put(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ShuffleUtils.convertJobTokenToBytes(shuffleToken)); Map<String, String> envMap = new HashMap<String, String>(); ByteBuffer shufflePortBb = ByteBuffer.allocate(4).putInt(0, 8000); AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, shufflePortBb, envMap);//from w w w . j a v a 2s .com LogicalIOProcessorRuntimeTask task = new LogicalIOProcessorRuntimeTask(taskSpec, 0, jobConf, new String[] { workDir.toString() }, umbilical, serviceConsumerMetadata, envMap, HashMultimap.<String, String>create(), null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory()); return task; }
From source file:org.apache.tez.mapreduce.processor.reduce.TestReduceProcessor.java
License:Apache License
@Test(timeout = 5000) public void testReduceProcessor() throws Exception { final String dagName = "mrdag0"; String mapVertexName = MultiStageMRConfigUtil.getInitialMapVertexName(); String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName(); JobConf jobConf = new JobConf(defaultConf); setUpJobConf(jobConf);//from w w w .ja v a 2s . c o m MRHelpers.translateMRConfToTez(jobConf); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); Path mapInput = new Path(workDir, "map0"); MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(MRRuntimeProtos.MRInputUserPayloadProto .newBuilder().setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)) .build().toByteArray()))), 1); OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); // Run a map TestUmbilical testUmbilical = new TestUmbilical(); LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0, mapInput, testUmbilical, dagName, mapVertexName, Collections.singletonList(mapInputSpec), Collections.singletonList(mapOutputSpec)); mapTask.initialize(); mapTask.run(); mapTask.close(); // One VME, One DME Assert.assertEquals(2, testUmbilical.getEvents().size()); Assert.assertEquals(EventType.VERTEX_MANAGER_EVENT, testUmbilical.getEvents().get(0).getEventType()); Assert.assertEquals(EventType.COMPOSITE_DATA_MOVEMENT_EVENT, testUmbilical.getEvents().get(1).getEventType()); CompositeDataMovementEvent cdmEvent = (CompositeDataMovementEvent) testUmbilical.getEvents().get(1) .getEvent(); Assert.assertEquals(1, cdmEvent.getCount()); DataMovementEvent dme = cdmEvent.getEvents().iterator().next(); dme.setTargetIndex(0); LOG.info("Starting reduce..."); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(dagName)); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(identifier, jobTokenSecretManager); shuffleToken.setService(identifier.getJobId()); jobConf.setOutputFormat(SequenceFileOutputFormat.class); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true); FileOutputFormat.setOutputPath(jobConf, new Path(workDir, "output")); ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create(ReduceProcessor.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)); InputSpec reduceInputSpec = new InputSpec(mapVertexName, InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex", OutputDescriptor .create(MROutputLegacy.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); // Now run a reduce TaskSpec taskSpec = new TaskSpec(TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0), dagName, reduceVertexName, -1, reduceProcessorDesc, Collections.singletonList(reduceInputSpec), Collections.singletonList(reduceOutputSpec), null); Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>(); serviceConsumerMetadata.put(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, ShuffleUtils.convertJobTokenToBytes(shuffleToken)); Map<String, String> serviceProviderEnvMap = new HashMap<String, String>(); ByteBuffer shufflePortBb = ByteBuffer.allocate(4).putInt(0, 8000); AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, shufflePortBb, serviceProviderEnvMap); LogicalIOProcessorRuntimeTask task = new LogicalIOProcessorRuntimeTask(taskSpec, 0, jobConf, new String[] { workDir.toString() }, new TestUmbilical(), serviceConsumerMetadata, serviceProviderEnvMap, HashMultimap.<String, String>create(), null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory()); List<Event> destEvents = new LinkedList<Event>(); destEvents.add(dme); task.initialize(); OrderedGroupedInputLegacy sortedOut = (OrderedGroupedInputLegacy) task.getInputs().values().iterator() .next(); sortedOut.handleEvents(destEvents); task.run(); task.close(); // MRTask mrTask = (MRTask)t.getProcessor(); // TODO NEWTEZ Verify the partitioner has not been created // Likely not applicable anymore. // Assert.assertNull(mrTask.getPartitioner()); // Only a task commit happens, hence the data is still in the temporary directory. Path reduceOutputDir = new Path(new Path(workDir, "output"), "_temporary/0/" + IDConverter.toMRTaskIdForOutput(TezTestUtils.getMockTaskId(0, 1, 0))); Path reduceOutputFile = new Path(reduceOutputDir, "part-v001-o000-00000"); SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, jobConf); LongWritable key = new LongWritable(); Text value = new Text(); long prev = Long.MIN_VALUE; while (reader.next(key, value)) { if (prev != Long.MIN_VALUE) { Assert.assertTrue(prev < key.get()); prev = key.get(); } } reader.close(); }
From source file:org.apache.tez.runtime.library.output.TestOnFileUnorderedKVOutput.java
License:Apache License
@Test(timeout = 5000) public void testGeneratedDataMovementEvent() throws Exception { Configuration conf = new Configuration(); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName()); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName()); int appAttemptNumber = 1; TezUmbilical tezUmbilical = null;/*from www .j av a 2 s.co m*/ String dagName = "currentDAG"; String taskVertexName = "currentVertex"; String destinationVertexName = "destinationVertex"; TezDAGID dagID = TezDAGID.getInstance("2000", 1, 1); TezVertexID vertexID = TezVertexID.getInstance(dagID, 1); TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); TezCounters counters = new TezCounters(); UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf); RuntimeTask runtimeTask = mock(RuntimeTask.class); int shufflePort = 2112; Map<String, String> auxEnv = new HashMap<String, String>(); ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(shufflePort); bb.position(0); AuxiliaryServiceHelper.setServiceDataIntoEnv(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID, bb, auxEnv); OutputDescriptor outputDescriptor = mock(OutputDescriptor.class); when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor"); OutputContext outputContext = new TezOutputContextImpl(conf, new String[] { workDir.toString() }, appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName, -1, taskAttemptID, counters, 0, userPayload, runtimeTask, null, auxEnv, new MemoryDistributor(1, 1, conf), outputDescriptor, null, new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory()); UnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(outputContext, 1); List<Event> events = null; events = kvOutput.initialize(); assertTrue(events != null && events.size() == 0); KeyValueWriter kvWriter = kvOutput.getWriter(); List<KVPair> data = KVDataGen.generateTestData(true); for (KVPair kvp : data) { kvWriter.write(kvp.getKey(), kvp.getvalue()); } events = kvOutput.close(); assertTrue(events != null && events.size() == 1); DataMovementEvent dmEvent = (DataMovementEvent) events.get(0); assertEquals("Invalid source index", 0, dmEvent.getSourceIndex()); DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto .parseFrom(ByteString.copyFrom(dmEvent.getUserPayload())); assertFalse(shufflePayload.hasEmptyPartitions()); assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent()); assertEquals(shufflePort, shufflePayload.getPort()); assertEquals("host", shufflePayload.getHost()); }
From source file:org.apache.tez.service.impl.ContainerRunnerImpl.java
License:Apache License
public void setShufflePort(int shufflePort) { AuxiliaryServiceHelper.setServiceDataIntoEnv(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(shufflePort), localEnv); }