List of usage examples for org.apache.hadoop.mapreduce TaskInputOutputContext getCounter
public Counter getCounter(Enum<?> counterName);
counterName. From source file:be.ugent.intec.halvade.utils.HalvadeFileUtils.java
License:Open Source License
/** * @return returns 0 if successfull, -1 if filesize is incorrect and -2 if an exception occurred *//*from w ww . java 2s . c om*/ protected static int privateDownloadFileFromHDFS(TaskInputOutputContext context, FileSystem fs, String from, String to) { try { // check if file is present on local scratch File f = new File(to); if (!f.exists()) { Logger.DEBUG("attempting download of \"" + to + "\""); fs.copyToLocalFile(new Path(from), new Path(to)); context.getCounter(HalvadeCounters.FIN_FROM_HDFS) .increment(fs.getFileStatus(new Path(from)).getLen()); } else { // check if filesize is correct if (fs.getFileStatus(new Path(from)).getLen() != f.length()) { // incorrect filesize, remove and download again Logger.DEBUG("incorrect filesize: " + f.length() + " =/= " + fs.getFileStatus(new Path(from)).getLen()); f.delete(); fs.copyToLocalFile(new Path(from), new Path(to)); context.getCounter(HalvadeCounters.FIN_FROM_HDFS) .increment(fs.getFileStatus(new Path(from)).getLen()); } else { Logger.DEBUG("file \"" + to + "\" exists"); } } if (fs.getFileStatus(new Path(from)).getLen() != f.length()) return -1; else return 0; } catch (IOException ex) { Logger.DEBUG("failed to download " + from + " from HDFS: " + ex.getLocalizedMessage()); Logger.EXCEPTION(ex); return -2; } }
From source file:be.ugent.intec.halvade.utils.HalvadeFileUtils.java
License:Open Source License
/** * @return returns 0 if successfull, -1 if filesize is incorrect and -2 if an exception occurred *///w w w. j ava 2 s .c o m protected static int privateUploadFileToHDFS(TaskInputOutputContext context, FileSystem fs, String from, String to) { try { // check if file is present on HDFS Path toPath = new Path(to); Path fromPath = new Path(from); File f = new File(from); if (!fs.exists(toPath)) { fs.copyFromLocalFile(fromPath, toPath); context.getCounter(HalvadeCounters.FOUT_TO_HDFS).increment(f.length()); } else { // check if filesize is correct if (fs.getFileStatus(toPath).getLen() != f.length()) { // incorrect filesize, remove and download again fs.delete(toPath, false); fs.copyFromLocalFile(fromPath, toPath); context.getCounter(HalvadeCounters.FOUT_TO_HDFS).increment(f.length()); } } if (fs.getFileStatus(toPath).getLen() != f.length()) return -1; else return 0; } catch (IOException ex) { Logger.DEBUG("failed to upload " + from + " to HDFS: " + ex.getLocalizedMessage()); Logger.EXCEPTION(ex); return -2; } }
From source file:be.ugent.intec.halvade.utils.HalvadeFileUtils.java
License:Open Source License
public static boolean removeLocalFile(boolean keep, String filename, TaskInputOutputContext context, HalvadeCounters counter) {/*from w w w . ja v a 2 s .c om*/ if (keep) return false; File f = new File(filename); if (f.exists()) context.getCounter(counter).increment(f.length()); return f.exists() && f.delete(); }
From source file:be.ugent.intec.halvade.utils.HalvadeFileUtils.java
License:Open Source License
public static boolean removeLocalDir(boolean keep, String filename, TaskInputOutputContext context, HalvadeCounters counter) {//from w w w . j a va 2s .com if (keep) return false; File f = new File(filename); if (f.exists()) context.getCounter(counter).increment(f.length()); return f.exists() && deleteDir(f); // f.delete(); }
From source file:com.asakusafw.runtime.compatibility.hadoop1.JobCompatibilityHadoop1.java
License:Apache License
@Override public Counter getTaskOutputRecordCounter(TaskInputOutputContext<?, ?, ?, ?> context) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ }/*from ww w. j a va2 s .co m*/ if (context.getTaskAttemptID().isMap()) { return context.getCounter(Task.Counter.MAP_OUTPUT_RECORDS); } else { return context.getCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); } }
From source file:com.asakusafw.runtime.compatibility.hadoop2.JobCompatibilityHadoop2.java
License:Apache License
@Override public Counter getTaskOutputRecordCounter(TaskInputOutputContext<?, ?, ?, ?> context) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ }/*w ww. j a v a 2 s . c o m*/ if (context.getTaskAttemptID().getTaskType() == TaskType.MAP) { return context.getCounter(TaskCounter.MAP_OUTPUT_RECORDS); } else { return context.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS); } }
From source file:org.apache.crunch.test.CrunchTestSupport.java
License:Apache License
/** * The method creates a {@linkplain TaskInputOutputContext} which can be used * in unit tests. The context serves very limited purpose. You can only * operate with counters, taskAttempId, status and configuration while using * this context.// w w w. jav a 2s. c o m */ public static <KI, VI, KO, VO> TaskInputOutputContext<KI, VI, KO, VO> getTestContext( final Configuration config) { TaskInputOutputContext<KI, VI, KO, VO> context = Mockito.mock(TaskInputOutputContext.class); TestCounters.clearCounters(); final StateHolder holder = new StateHolder(); Mockito.when(context.getCounter(Mockito.any(Enum.class))).then(new Answer<Counter>() { @Override public Counter answer(InvocationOnMock invocation) throws Throwable { Enum<?> counter = (Enum<?>) invocation.getArguments()[0]; return TestCounters.getCounter(counter); } }); Mockito.when(context.getCounter(Mockito.anyString(), Mockito.anyString())).then(new Answer<Counter>() { @Override public Counter answer(InvocationOnMock invocation) throws Throwable { String group = (String) invocation.getArguments()[0]; String name = (String) invocation.getArguments()[1]; return TestCounters.getCounter(group, name); } }); Mockito.when(context.getConfiguration()).thenReturn(config); Mockito.when(context.getTaskAttemptID()).thenReturn(new TaskAttemptID()); Mockito.when(context.getStatus()).then(new Answer<String>() { @Override public String answer(InvocationOnMock invocation) throws Throwable { return holder.internalStatus; } }); Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { holder.internalStatus = (String) invocation.getArguments()[0]; return null; } }).when(context).setStatus(Mockito.anyString()); return context; }
From source file:org.apache.crunch.test.CrunchTestSupportTest.java
License:Apache License
@Test public void testContext() { Configuration sampleConfig = new Configuration(); String status = "test"; TaskInputOutputContext<?, ?, ?, ?> testContext = CrunchTestSupport.getTestContext(sampleConfig); assertEquals(sampleConfig, testContext.getConfiguration()); TaskAttemptID taskAttemptID = testContext.getTaskAttemptID(); assertEquals(taskAttemptID, testContext.getTaskAttemptID()); assertNotNull(taskAttemptID);// w w w.j a v a2 s . co m assertNull(testContext.getStatus()); testContext.setStatus(status); assertEquals(status, testContext.getStatus()); assertEquals(0, testContext.getCounter(CT.ONE).getValue()); testContext.getCounter(CT.ONE).increment(1); assertEquals(1, testContext.getCounter(CT.ONE).getValue()); testContext.getCounter(CT.ONE).increment(4); assertEquals(5, testContext.getCounter(CT.ONE).getValue()); }