List of usage examples for org.apache.hadoop.mapreduce TaskAttemptID toString
@Override
public String toString()
From source file:it.crs4.pydoop.mapreduce.pipes.TestPipeApplication.java
License:Apache License
/** * test org.apache.hadoop.mapreduce.pipes.Application * test a internal functions: /*from www . j a v a 2s .com*/ * MessageType.REGISTER_COUNTER, INCREMENT_COUNTER, STATUS, PROGRESS... * * @throws Throwable */ @Test public void testApplication() throws Throwable { System.err.println("testApplication"); File[] psw = cleanTokenPasswordFile(); try { JobID jobId = new JobID("201408272347", 0); TaskID taskId = new TaskID(jobId, TaskType.MAP, 0); TaskAttemptID taskAttemptid = new TaskAttemptID(taskId, 0); Job job = new Job(new Configuration()); job.setJobID(jobId); Configuration conf = job.getConfiguration(); conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptid.toString()); FileSystem fs = new RawLocalFileSystem(); fs.setConf(conf); File fCommand = getFileCommand("it.crs4.pydoop.mapreduce.pipes.PipeApplicationStub"); //getFileCommand("it.crs4.pydoop.mapreduce.pipes.PipeApplicationRunnableStub"); conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); System.err.println("fCommand" + fCommand.getAbsolutePath()); Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>("user".getBytes(), "password".getBytes(), new Text("kind"), new Text("service")); TokenCache.setJobToken(token, job.getCredentials()); conf.setBoolean(MRJobConfig.SKIP_RECORDS, true); TestReporter reporter = new TestReporter(); DummyInputFormat input_format = new DummyInputFormat(); List<InputSplit> isplits = input_format.getSplits(job); InputSplit isplit = isplits.get(0); TaskAttemptContextImpl tcontext = new TaskAttemptContextImpl(conf, taskAttemptid); DummyRecordReader reader = (DummyRecordReader) input_format.createRecordReader(isplit, tcontext); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); RecordWriter<IntWritable, Text> writer = new TestRecordWriter( new FileOutputStream(workSpace.getAbsolutePath() + File.separator + "outfile")); MapContextImpl<IntWritable, Text, IntWritable, Text> context = new MapContextImpl<IntWritable, Text, IntWritable, Text>( conf, taskAttemptid, null, writer, null, reporter, null); System.err.println("ready to launch application"); Application<IntWritable, Text, IntWritable, Text> application = new Application<IntWritable, Text, IntWritable, Text>( context, reader); System.err.println("done"); application.getDownlink().flush(); application.getDownlink().mapItem(new IntWritable(3), new Text("txt")); application.getDownlink().flush(); application.waitForFinish(); // test getDownlink().mapItem(); String stdOut = readStdOut(conf); assertTrue(stdOut.contains("key:3")); assertTrue(stdOut.contains("value:txt")); assertEquals(0.0, context.getProgress(), 0.01); assertNotNull(context.getCounter("group", "name")); // test status MessageType.STATUS assertEquals(context.getStatus(), "PROGRESS"); // check MessageType.PROGRESS assertEquals(0.55f, reader.getProgress(), 0.001); application.getDownlink().close(); // test MessageType.OUTPUT stdOut = readFile(new File(workSpace.getAbsolutePath() + File.separator + "outfile")); assertTrue(stdOut.contains("key:123")); assertTrue(stdOut.contains("value:value")); try { // try to abort application.abort(new Throwable()); fail(); } catch (IOException e) { // abort works ? assertEquals("pipe child exception", e.getMessage()); } } finally { if (psw != null) { // remove password files for (File file : psw) { file.deleteOnExit(); } } } }
From source file:it.crs4.pydoop.mapreduce.pipes.TestPipeApplication.java
License:Apache License
/** * test org.apache.hadoop.mapreduce.pipes.PipesReducer * test the transfer of data: key and value * * @throws Exception/*from www .ja va2s .com*/ */ @Test public void testPipesReducer() throws Exception { System.err.println("testPipesReducer"); File[] psw = cleanTokenPasswordFile(); try { JobID jobId = new JobID("201408272347", 0); TaskID taskId = new TaskID(jobId, TaskType.MAP, 0); TaskAttemptID taskAttemptid = new TaskAttemptID(taskId, 0); Job job = new Job(new Configuration()); job.setJobID(jobId); Configuration conf = job.getConfiguration(); conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptid.toString()); FileSystem fs = new RawLocalFileSystem(); fs.setConf(conf); File fCommand = getFileCommand("it.crs4.pydoop.mapreduce.pipes.PipeReducerStub"); conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); System.err.println("fCommand" + fCommand.getAbsolutePath()); Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>("user".getBytes(), "password".getBytes(), new Text("kind"), new Text("service")); TokenCache.setJobToken(token, job.getCredentials()); conf.setBoolean(MRJobConfig.SKIP_RECORDS, true); TestReporter reporter = new TestReporter(); DummyInputFormat input_format = new DummyInputFormat(); List<InputSplit> isplits = input_format.getSplits(job); InputSplit isplit = isplits.get(0); TaskAttemptContextImpl tcontext = new TaskAttemptContextImpl(conf, taskAttemptid); RecordWriter<IntWritable, Text> writer = new TestRecordWriter( new FileOutputStream(workSpace.getAbsolutePath() + File.separator + "outfile")); BooleanWritable bw = new BooleanWritable(true); List<Text> texts = new ArrayList<Text>(); texts.add(new Text("first")); texts.add(new Text("second")); texts.add(new Text("third")); DummyRawKeyValueIterator kvit = new DummyRawKeyValueIterator(); ReduceContextImpl<BooleanWritable, Text, IntWritable, Text> context = new ReduceContextImpl<BooleanWritable, Text, IntWritable, Text>( conf, taskAttemptid, kvit, null, null, writer, null, null, null, BooleanWritable.class, Text.class); PipesReducer<BooleanWritable, Text, IntWritable, Text> reducer = new PipesReducer<BooleanWritable, Text, IntWritable, Text>(); reducer.setup(context); initStdOut(conf); reducer.reduce(bw, texts, context); reducer.cleanup(context); String stdOut = readStdOut(conf); // test data: key assertTrue(stdOut.contains("reducer key :true")); // and values assertTrue(stdOut.contains("reduce value :first")); assertTrue(stdOut.contains("reduce value :second")); assertTrue(stdOut.contains("reduce value :third")); } finally { if (psw != null) { // remove password files for (File file : psw) { file.deleteOnExit(); } } } }
From source file:org.apache.blur.mapreduce.lib.BlurOutputFormat.java
License:Apache License
@Override public RecordWriter<Text, BlurMutate> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { int id = context.getTaskAttemptID().getTaskID().getId(); TaskAttemptID taskAttemptID = context.getTaskAttemptID(); final GenericBlurRecordWriter writer = new GenericBlurRecordWriter(context.getConfiguration(), id, taskAttemptID.toString() + ".tmp"); return new RecordWriter<Text, BlurMutate>() { @Override//from w ww. jav a 2 s .c o m public void write(Text key, BlurMutate value) throws IOException, InterruptedException { writer.write(key, value); } @Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { writer.close(); } }; }
From source file:org.apache.flink.api.java.hadoop.mapreduce.HadoopOutputFormatBase.java
License:Apache License
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException/*from w w w .j a v a 2s.c o m*/ */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } this.taskNumber = taskNumber + 1; // for hadoop 2.2 this.configuration.set("mapreduce.output.basename", "tmp"); TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String .format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.configuration.set("mapred.task.id", taskAttemptID.toString()); this.configuration.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.configuration.setInt("mapreduce.task.partition", taskNumber + 1); try { this.context = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID); this.outputCommitter = this.mapreduceOutputFormat.getOutputCommitter(this.context); this.outputCommitter.setupJob(HadoopUtils.instantiateJobContext(this.configuration, new JobID())); } catch (Exception e) { throw new RuntimeException(e); } this.context.getCredentials().addAll(this.credentials); Credentials currentUserCreds = getCredentialsFromUGI(UserGroupInformation.getCurrentUser()); if (currentUserCreds != null) { this.context.getCredentials().addAll(currentUserCreds); } // compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1 if (outputCommitter instanceof FileOutputCommitter) { this.configuration.set("mapreduce.task.output.dir", ((FileOutputCommitter) this.outputCommitter).getWorkPath().toString()); } try { this.recordWriter = this.mapreduceOutputFormat.getRecordWriter(this.context); } catch (InterruptedException e) { throw new IOException("Could not create RecordWriter.", e); } } }
From source file:org.apache.flink.hadoopcompatibility.mapreduce.HadoopOutputFormat.java
License:Apache License
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws IOException/*from w w w. j a v a 2s . co m*/ */ @Override public void open(int taskNumber, int numTasks) throws IOException { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } // for hadoop 2.2 this.configuration.set("mapreduce.output.basename", "tmp"); TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.configuration.set("mapred.task.id", taskAttemptID.toString()); this.configuration.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.configuration.setInt("mapreduce.task.partition", taskNumber + 1); try { this.context = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID); } catch (Exception e) { throw new RuntimeException(e); } this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")), context); try { this.fileOutputCommitter.setupJob(HadoopUtils.instantiateJobContext(this.configuration, new JobID())); } catch (Exception e) { throw new RuntimeException(e); } // compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1 this.configuration.set("mapreduce.task.output.dir", this.fileOutputCommitter.getWorkPath().toString()); try { this.recordWriter = this.mapreduceOutputFormat.getRecordWriter(this.context); } catch (InterruptedException e) { throw new IOException("Could not create RecordWriter.", e); } }
From source file:org.apache.hive.hcatalog.pig.TestE2EScenarios.java
License:Apache License
private TaskAttemptContext createTaskAttemptContext(Configuration tconf) { Configuration conf = (tconf == null) ? (new Configuration()) : tconf; TaskAttemptID taskId = HCatMapRedUtil.createTaskAttemptID(new JobID("200908190029", 1), false, 1, 1); conf.setInt("mapred.task.partition", taskId.getId()); conf.set("mapred.task.id", taskId.toString()); TaskAttemptContext rtaskContext = HCatMapRedUtil.createTaskAttemptContext(conf, taskId); return rtaskContext; }
From source file:org.apache.pig.data.TestSchemaTuple.java
License:Apache License
/** * This tests that InterStorage will work as expected with the Tuples generated by * the given TupleFactory./*from www . ja va 2s . co m*/ * @param tf * @throws Exception */ public void testInterStorageSerDe(SchemaTupleFactory tf) throws Exception { int sz = 4096; List<Tuple> written = new ArrayList<Tuple>(sz); File temp = File.createTempFile("tmp", "tmp"); temp.deleteOnExit(); FileOutputStream fos = new FileOutputStream(temp); DataOutputStream dos = new DataOutputStream(fos); InterRecordWriter writer = new InterRecordWriter(dos); // We add these lines because a part of the InterStorage logic // is the ability to seek to the next Tuple based on a magic set // of bytes. This emulates the random byes that will be present // at the beginning of a split. dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); for (int i = 0; i < sz; i++) { SchemaTuple<?> st = (SchemaTuple<?>) tf.newTuple(); fillWithData(st); writer.write(null, st); written.add(st); dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); dos.writeByte(r.nextInt()); } writer.close(null); Configuration conf = new Configuration(); conf.set("fs.default.name", "file:///"); TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 1, true, 1, 1); conf.set(MRConfiguration.TASK_ID, taskId.toString()); InputSplit is = new FileSplit(new Path(temp.getAbsolutePath()), 0, temp.length(), null); InterRecordReader reader = new InterRecordReader(); reader.initialize(is, HadoopShims.createTaskAttemptContext(conf, taskId)); for (int i = 0; i < sz; i++) { assertTrue(reader.nextKeyValue()); SchemaTuple<?> st = (SchemaTuple<?>) reader.getCurrentValue(); assertEquals(written.get(i), st); } reader.close(); }
From source file:org.apache.pig.piggybank.test.storage.TestIndexedStorage.java
License:Apache License
@Test /**/*from w ww.j ava 2 s . co m*/ * Tests getNext called repeatedly returns items in sorted order. */ public void testGetNext() throws IOException, InterruptedException { IndexedStorage storage = new IndexedStorage("\t", "0,1"); Configuration conf = new Configuration(); conf.set("fs.default.name", "file:///"); LocalFileSystem fs = FileSystem.getLocal(conf); TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 1, true, 1, 1); conf.set(MRConfiguration.TASK_ID, taskId.toString()); conf.set(MRConfiguration.INPUT_DIR, Util.encodeEscape(outputDir.getAbsolutePath())); storage.initialize(conf); Integer key; int[][] correctValues = { { 2, 2 }, { 3, 2 }, { 3, 3 }, { 4, 3 }, { 4, 4 }, { 5, 5 }, { 5, 5 }, { 6, 6 }, { 7, 7 }, { 8, 8 }, { 9, 9 }, { 10, 10 }, { 11, 11 }, { 13, 13 } }; for (int[] outer : correctValues) { System.out.println("Testing: (" + outer[0] + "," + outer[1] + ")"); Tuple read = storage.getNext(); System.out.println("Read: " + read); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value. Received: " + read, key.equals(outer[0])); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value. Received: " + read, key.equals(outer[1])); } Tuple read = storage.getNext(); assertTrue("GetNext did not return the correct value", (read == null)); }
From source file:org.apache.pig.piggybank.test.storage.TestIndexedStorage.java
License:Apache License
@Test /**//from w ww . ja v a 2 s .c om * Tests seekNear and getNext work correctly. */ public void testSeek() throws IOException, InterruptedException { IndexedStorage storage = new IndexedStorage("\t", "0,1"); Configuration conf = new Configuration(); conf.set("fs.default.name", "file:///"); LocalFileSystem fs = FileSystem.getLocal(conf); TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 2, true, 2, 2); conf.set(MRConfiguration.TASK_ID, taskId.toString()); conf.set(MRConfiguration.INPUT_DIR, Util.encodeEscape(outputDir.getAbsolutePath())); storage.initialize(conf); TupleFactory tupleFactory = TupleFactory.getInstance(); Tuple seek = tupleFactory.newTuple(2); Integer key; Tuple read; //Seek to 1,1 (not in index). getNext should return 2 seek.set(0, new Integer(1)); seek.set(1, new Integer(1)); storage.seekNear(seek); read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(2)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(2)); //Seek to 3,2. getNext should return 3,2 seek.set(0, new Integer(3)); seek.set(1, new Integer(2)); storage.seekNear(seek); read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(3)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(2)); //getNext should return 3,3 read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(3)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(3)); //Seek to 6,6. getNext should return 6,6 seek.set(0, new Integer(6)); seek.set(1, new Integer(6)); storage.seekNear(seek); read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(6)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(6)); //getNext should return 7,7 read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(7)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(7)); //Seek to 9,9. getNext should return 9,9 seek.set(0, new Integer(9)); seek.set(1, new Integer(9)); storage.seekNear(seek); read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(9)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(9)); //getNext should return 10,10 read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(10)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(10)); //Seek to 13,12 (Not in index). getNext should return 13,13 seek.set(0, new Integer(13)); seek.set(1, new Integer(12)); storage.seekNear(seek); read = storage.getNext(); key = Integer.decode(((DataByteArray) read.get(0)).toString()); assertTrue("GetNext did not return the correct value", key.equals(13)); key = Integer.decode(((DataByteArray) read.get(1)).toString()); assertTrue("GetNext did not return the correct value", key.equals(13)); //Seek to 20 (Not in index). getNext should return null seek.set(0, new Integer(20)); seek.set(1, new Integer(20)); storage.seekNear(seek); read = storage.getNext(); assertTrue("GetNext did not return the correct value", (read == null)); }