List of usage examples for org.apache.hadoop.mapreduce TaskInputOutputContext getTaskAttemptID
public TaskAttemptID getTaskAttemptID();
From source file:com.asakusafw.runtime.compatibility.hadoop1.JobCompatibilityHadoop1.java
License:Apache License
@Override public Counter getTaskOutputRecordCounter(TaskInputOutputContext<?, ?, ?, ?> context) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ }//from ww w . ja v a 2 s . com if (context.getTaskAttemptID().isMap()) { return context.getCounter(Task.Counter.MAP_OUTPUT_RECORDS); } else { return context.getCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); } }
From source file:com.asakusafw.runtime.compatibility.hadoop2.JobCompatibilityHadoop2.java
License:Apache License
@Override public Counter getTaskOutputRecordCounter(TaskInputOutputContext<?, ?, ?, ?> context) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ }//from ww w.java 2 s .c o m if (context.getTaskAttemptID().getTaskType() == TaskType.MAP) { return context.getCounter(TaskCounter.MAP_OUTPUT_RECORDS); } else { return context.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS); } }
From source file:com.moz.fiji.mapreduce.util.SerializeLoggerAspect.java
License:Apache License
/** * Logic to serialize collected profiling content to a file on HDFS. The files are stored * in the current working directory for this context, in a folder specified by STATS_DIR. The per * task file is named by the task attempt id. * We obtain the profiling stats collected by the LogTimerAspect in FijiSchema. The format of the * file is as follows: Job Name, Job ID, Task Attempt, Function Signature, * Aggregate Time (nanoseconds), Number of Invocations, Time per call (nanoseconds)'\n' * * @param context The {@link TaskInputOutputContext} for this job. * @throws IOException If the writes to HDFS fail. *//* w w w .j a va 2 s. c o m*/ private void serializeToFile(TaskInputOutputContext context) throws IOException { Path parentPath = new Path(context.getWorkingDirectory(), STATS_DIR); FileSystem fs = parentPath.getFileSystem(context.getConfiguration()); fs.mkdirs(parentPath); Path path = new Path(parentPath, context.getTaskAttemptID().toString()); OutputStreamWriter out = new OutputStreamWriter(fs.create(path, true), "UTF-8"); try { out.write("Job Name, Job ID, Task Attempt, Function Signature, Aggregate Time (nanoseconds), " + "Number of Invocations, Time per call (nanoseconds)\n"); ConcurrentHashMap<String, LoggingInfo> signatureTimeMap = mLogTimerAspect.getSignatureTimeMap(); for (Map.Entry<String, LoggingInfo> entrySet : signatureTimeMap.entrySet()) { writeProfileInformation(out, context, entrySet.getKey(), entrySet.getValue()); } signatureTimeMap = mMRLogTimerAspect.getSignatureTimeMap(); for (Map.Entry<String, LoggingInfo> entrySet : signatureTimeMap.entrySet()) { writeProfileInformation(out, context, entrySet.getKey(), entrySet.getValue()); } } finally { out.close(); } }
From source file:com.moz.fiji.mapreduce.util.SerializeLoggerAspect.java
License:Apache License
/** * Logic to write a profiling content for a single method signature to a file on HDFS. * The format of the file is as follows: Job Name, Job ID, Task Attempt, Function Signature, * Aggregate Time (nanoseconds), Number of Invocations, Time per call (nanoseconds)'\n' * * @param out The {@link OutputStreamWriter} for writing to the file. * @param context The {@link TaskInputOutputContext} for this job. * @param signature The method signature for the profile. * @param loggingInfo The profiling information for the method. * @throws IOException If the writes to HDFS fail. *//*from www. j a va 2 s . c o m*/ private void writeProfileInformation(OutputStreamWriter out, TaskInputOutputContext context, String signature, LoggingInfo loggingInfo) throws IOException { // ensure that files do not end up with x.yzE7 format for floats. Instead of 1.0E3, we want // 1000.000 NumberFormat nf = NumberFormat.getInstance(); nf.setGroupingUsed(false); nf.setMinimumFractionDigits(1); nf.setMaximumFractionDigits(3); out.write(context.getJobName() + ", " + context.getJobID() + ", " + context.getTaskAttemptID() + ", " + signature + ", " + loggingInfo.toString() + ", " + nf.format(loggingInfo.perCallTime()) + "\n"); }
From source file:de.l3s.concatgz.io.ImmediateOutput.java
License:Open Source License
public ImmediateOutput(TaskInputOutputContext context, boolean flushOnWrite) throws IOException { this.context = context; this.flushOnWrite = flushOnWrite; Configuration conf = context.getConfiguration(); this.dir = getPath(conf); this.fs = FileSystem.newInstance(conf); this.bufferSize = conf.getInt("io.file.buffer.size", 4096); this.replication = getReplication(conf); String idPrefix = getIdPrefix(conf); file = "" + context.getTaskAttemptID().getTaskID().getId(); while (file.length() < 5) file = "0" + file; if (idPrefix.length() > 0) file = idPrefix + "-" + file; file = "-" + file; }
From source file:it.crs4.pydoop.mapreduce.pipes.Application.java
License:Apache License
/** * Start the child process to handle the task for us. * @throws IOException/* w w w.j a va 2s .com*/ * @throws InterruptedException */ Application(TaskInputOutputContext<K1, V1, K2, V2> context, DummyRecordReader input) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); serverSocket = new ServerSocket(0); Map<String, String> env = new HashMap<String, String>(); // add TMPDIR environment variable with the value of java.io.tmpdir env.put("TMPDIR", System.getProperty("java.io.tmpdir")); env.put(Submitter.PORT, Integer.toString(serverSocket.getLocalPort())); //Add token to the environment if security is enabled Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(context.getCredentials()); // This password is used as shared secret key between this application and // child pipes process byte[] password = jobToken.getPassword(); String localPasswordFile = new File(".") + Path.SEPARATOR + "jobTokenPassword"; writePasswordToLocalFile(localPasswordFile, password, conf); // FIXME why is this not Submitter.SECRET_LOCATION ? env.put("hadoop.pipes.shared.secret.location", localPasswordFile); List<String> cmd = new ArrayList<String>(); String interpretor = conf.get(Submitter.INTERPRETOR); if (interpretor != null) { cmd.add(interpretor); } String executable = context.getLocalCacheFiles()[0].toString(); if (!(new File(executable).canExecute())) { // LinuxTaskController sets +x permissions on all distcache files already. // In case of DefaultTaskController, set permissions here. FileUtil.chmod(executable, "u+x"); } cmd.add(executable); // wrap the command in a stdout/stderr capture // we are starting map/reduce task of the pipes job. this is not a cleanup // attempt. TaskAttemptID taskid = context.getTaskAttemptID(); File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT); File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR); long logLength = TaskLog.getTaskLogLength(conf); cmd = TaskLog.captureOutAndError(null, cmd, stdout, stderr, logLength, false); process = runClient(cmd, env); clientSocket = serverSocket.accept(); String challenge = getSecurityChallenge(); String digestToSend = createDigest(password, challenge); String digestExpected = createDigest(password, digestToSend); handler = new OutputHandler<K2, V2>(context, input, digestExpected); K2 outputKey = (K2) ReflectionUtils.newInstance(context.getOutputKeyClass(), conf); V2 outputValue = (V2) ReflectionUtils.newInstance(context.getOutputValueClass(), conf); downlink = new BinaryProtocol<K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue, conf); downlink.authenticate(digestToSend, challenge); waitForAuthentication(); LOG.debug("Authentication succeeded"); downlink.start(); downlink.setJobConf(conf); }
From source file:org.apache.crunch.test.CrunchTestSupport.java
License:Apache License
/** * The method creates a {@linkplain TaskInputOutputContext} which can be used * in unit tests. The context serves very limited purpose. You can only * operate with counters, taskAttempId, status and configuration while using * this context./*from w w w . ja va2s. c om*/ */ public static <KI, VI, KO, VO> TaskInputOutputContext<KI, VI, KO, VO> getTestContext( final Configuration config) { TaskInputOutputContext<KI, VI, KO, VO> context = Mockito.mock(TaskInputOutputContext.class); TestCounters.clearCounters(); final StateHolder holder = new StateHolder(); Mockito.when(context.getCounter(Mockito.any(Enum.class))).then(new Answer<Counter>() { @Override public Counter answer(InvocationOnMock invocation) throws Throwable { Enum<?> counter = (Enum<?>) invocation.getArguments()[0]; return TestCounters.getCounter(counter); } }); Mockito.when(context.getCounter(Mockito.anyString(), Mockito.anyString())).then(new Answer<Counter>() { @Override public Counter answer(InvocationOnMock invocation) throws Throwable { String group = (String) invocation.getArguments()[0]; String name = (String) invocation.getArguments()[1]; return TestCounters.getCounter(group, name); } }); Mockito.when(context.getConfiguration()).thenReturn(config); Mockito.when(context.getTaskAttemptID()).thenReturn(new TaskAttemptID()); Mockito.when(context.getStatus()).then(new Answer<String>() { @Override public String answer(InvocationOnMock invocation) throws Throwable { return holder.internalStatus; } }); Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { holder.internalStatus = (String) invocation.getArguments()[0]; return null; } }).when(context).setStatus(Mockito.anyString()); return context; }
From source file:org.apache.crunch.test.CrunchTestSupportTest.java
License:Apache License
@Test public void testContext() { Configuration sampleConfig = new Configuration(); String status = "test"; TaskInputOutputContext<?, ?, ?, ?> testContext = CrunchTestSupport.getTestContext(sampleConfig); assertEquals(sampleConfig, testContext.getConfiguration()); TaskAttemptID taskAttemptID = testContext.getTaskAttemptID(); assertEquals(taskAttemptID, testContext.getTaskAttemptID()); assertNotNull(taskAttemptID);//from w w w.j ava2 s .c o m assertNull(testContext.getStatus()); testContext.setStatus(status); assertEquals(status, testContext.getStatus()); assertEquals(0, testContext.getCounter(CT.ONE).getValue()); testContext.getCounter(CT.ONE).increment(1); assertEquals(1, testContext.getCounter(CT.ONE).getValue()); testContext.getCounter(CT.ONE).increment(4); assertEquals(5, testContext.getCounter(CT.ONE).getValue()); }
From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl.java
License:Apache License
public MapReducePOStoreImpl(TaskInputOutputContext<?, ?, ?, ?> context) { // get a copy of the Configuration so that changes to the // configuration below (like setting the output location) do // not affect the caller's copy Configuration outputConf = new Configuration(context.getConfiguration()); reporter = PigStatusReporter.getInstance(); reporter.setContext(new MRTaskContext(context)); // make a copy of the Context to use here - since in the same // task (map or reduce) we could have multiple stores, we should // make this copy so that the same context does not get over-written // by the different stores. this.context = HadoopShims.createTaskAttemptContext(outputConf, context.getTaskAttemptID()); }
From source file:org.kiji.mapreduce.util.SerializeLoggerAspect.java
License:Apache License
/** * Logic to serialize collected profiling content to a file on HDFS. The files are stored * in the current working directory for this context, in a folder specified by STATS_DIR. The per * task file is named by the task attempt id. * We obtain the profiling stats collected by the LogTimerAspect in KijiSchema. The format of the * file is as follows: Job Name, Job ID, Task Attempt, Function Signature, * Aggregate Time (nanoseconds), Number of Invocations, Time per call (nanoseconds)'\n' * * @param context The {@link TaskInputOutputContext} for this job. * @throws IOException If the writes to HDFS fail. *///from w w w . ja v a 2s .co m private void serializeToFile(TaskInputOutputContext context) throws IOException { Path parentPath = new Path(context.getWorkingDirectory(), STATS_DIR); FileSystem fs = FileSystem.get(context.getConfiguration()); fs.mkdirs(parentPath); Path path = new Path(parentPath, context.getTaskAttemptID().toString()); OutputStreamWriter out = new OutputStreamWriter(fs.create(path, true), "UTF-8"); try { out.write("Job Name, Job ID, Task Attempt, Function Signature, Aggregate Time (nanoseconds), " + "Number of Invocations, Time per call (nanoseconds)\n"); ConcurrentHashMap<String, LoggingInfo> signatureTimeMap = mLogTimerAspect.getSignatureTimeMap(); for (Map.Entry<String, LoggingInfo> entrySet : signatureTimeMap.entrySet()) { writeProfileInformation(out, context, entrySet.getKey(), entrySet.getValue()); } signatureTimeMap = mMRLogTimerAspect.getSignatureTimeMap(); for (Map.Entry<String, LoggingInfo> entrySet : signatureTimeMap.entrySet()) { writeProfileInformation(out, context, entrySet.getKey(), entrySet.getValue()); } } finally { out.close(); } }