List of usage examples for org.apache.hadoop.mapreduce JobID JobID
public JobID(String jtIdentifier, int id)
From source file:it.crs4.pydoop.mapreduce.pipes.TestPipesNonJavaInputFormat.java
License:Apache License
/** * test PipesNonJavaInputFormat//from w w w . j av a2s . c om */ @Test public void testFormat() throws IOException, InterruptedException { JobID jobId = new JobID("201408272347", 0); TaskID taskId = new TaskID(jobId, TaskType.MAP, 0); TaskAttemptID taskAttemptid = new TaskAttemptID(taskId, 0); Job job = new Job(new Configuration()); job.setJobID(jobId); Configuration conf = job.getConfiguration(); TaskAttemptContextImpl tcontext = new TaskAttemptContextImpl(conf, taskAttemptid); PipesNonJavaInputFormat input_format = new PipesNonJavaInputFormat(); DummyRecordReader reader = (DummyRecordReader) input_format.createRecordReader(new FileSplit(), tcontext); assertEquals(0.0f, reader.getProgress(), 0.001); // input and output files File input1 = new File(workSpace + File.separator + "input1"); if (!input1.getParentFile().exists()) { Assert.assertTrue(input1.getParentFile().mkdirs()); } if (!input1.exists()) { Assert.assertTrue(input1.createNewFile()); } File input2 = new File(workSpace + File.separator + "input2"); if (!input2.exists()) { Assert.assertTrue(input2.createNewFile()); } // THIS fill fail without hdfs support. // // set data for splits // conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR, // StringUtils.escapeString(input1.getAbsolutePath()) + "," // + StringUtils.escapeString(input2.getAbsolutePath())); // List<InputSplit> splits = input_format.getSplits(job); // assertTrue(splits.size() >= 2); PipesNonJavaInputFormat.PipesDummyRecordReader dummyRecordReader = new PipesNonJavaInputFormat.PipesDummyRecordReader( new FileSplit(), tcontext); // empty dummyRecordReader assertEquals(0.0, dummyRecordReader.getProgress(), 0.001); // test method next assertTrue(dummyRecordReader.next(new FloatWritable(2.0f), NullWritable.get())); assertEquals(2.0, dummyRecordReader.getProgress(), 0.001); dummyRecordReader.close(); }
From source file:org.apache.ambari.TestJobHistoryParsing.java
License:Apache License
public void test(String workflowId, String workflowName, String workflowNodeName, Map<String, String[]> adjacencies) { Configuration conf = new Configuration(); setProperties(conf, workflowId, workflowName, workflowNodeName, adjacencies); String log = log("JOB", new String[] { ID, NAME, NODE, ADJ }, new String[] { conf.get(ID_PROP), conf.get(NAME_PROP), conf.get(NODE_PROP), JobHistory.JobInfo.getWorkflowAdjacencies(conf) }); ParsedLine line = new ParsedLine(log); JobID jobid = new JobID("id", 1); JobSubmittedEvent event = new JobSubmittedEvent(jobid, workflowName, "", 0l, "", null, "", line.get(ID), line.get(NAME), line.get(NODE), line.get(ADJ)); WorkflowContext context = MapReduceJobHistoryUpdater.buildWorkflowContext(event); String resultingWorkflowId = workflowId; if (workflowId.isEmpty()) resultingWorkflowId = jobid.toString().replace("job_", "mr_"); assertEquals("Didn't recover workflowId", resultingWorkflowId, context.getWorkflowId()); assertEquals("Didn't recover workflowName", workflowName, context.getWorkflowName()); assertEquals("Didn't recover workflowNodeName", workflowNodeName, context.getWorkflowEntityName()); Map<String, String[]> resultingAdjacencies = adjacencies; if (resultingAdjacencies.size() == 0) { resultingAdjacencies = new HashMap<String, String[]>(); resultingAdjacencies.put(workflowNodeName, new String[] {}); }//w ww . j a v a2 s . c o m assertEquals("Got incorrect number of adjacencies", resultingAdjacencies.size(), context.getWorkflowDag().getEntries().size()); for (WorkflowDagEntry entry : context.getWorkflowDag().getEntries()) { String[] sTargets = resultingAdjacencies.get(entry.getSource()); assertNotNull("No original targets for " + entry.getSource(), sTargets); List<String> dTargets = entry.getTargets(); assertEquals("Got incorrect number of targets for " + entry.getSource(), sTargets.length, dTargets.size()); for (int i = 0; i < sTargets.length; i++) { assertEquals("Got incorrect target for " + entry.getSource(), sTargets[i], dTargets.get(i)); } } }
From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOSequenceFileTest.java
License:Apache License
private Stream<KV<Text, LongWritable>> extractResultsFromFile(String fileName) { try (SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { Path path = new Path(fileName); TaskAttemptContext taskContext = HadoopFormats.createTaskAttemptContext(new Configuration(), new JobID("readJob", 0), 0); reader.initialize(new FileSplit(path, 0L, Long.MAX_VALUE, new String[] { "localhost" }), taskContext); List<KV<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add(KV.of(new Text(reader.getCurrentKey().toString()), new LongWritable(reader.getCurrentValue().get()))); }//from w ww .j a v a 2 s . c o m return result.stream(); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormats.java
License:Apache License
/** * Creates {@link JobID} with random jtIdentifier and default job number. * * @return new {@link JobID}//from w w w .ja v a 2s . co m */ public static JobID createJobId() { return new JobID(UUID.randomUUID().toString(), DEFAULT_JOB_NUMBER); }
From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormats.java
License:Apache License
/** * Creates {@link JobID} with specified jtIdentifier and default job number. * * @param jtIdentifier jtIdentifier to specify * @return new {@link JobID}/* w w w.ja va 2s . c o m*/ */ public static JobID createJobId(String jtIdentifier) { return new JobID(jtIdentifier, DEFAULT_JOB_NUMBER); }
From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormats.java
License:Apache License
/** * Creates {@link JobID} with {@code jtIdentifier} specified in hadoop {@link Configuration} under * {@link MRJobConfig#ID} key./*w ww . j ava 2 s . c o m*/ * * @param conf hadoop {@link Configuration} * @return JobID created from {@link Configuration} */ static JobID getJobId(Configuration conf) { String jobJtIdentifier = Preconditions.checkNotNull(conf.get(MRJobConfig.ID), "Configuration must contain jobID under key \"%s\".", HadoopFormatIO.JOB_ID); return new JobID(jobJtIdentifier, DEFAULT_JOB_NUMBER); }
From source file:org.apache.carbondata.sdk.file.AvroCarbonWriter.java
License:Apache License
AvroCarbonWriter(CarbonLoadModel loadModel, Configuration hadoopConf) throws IOException { CarbonTableOutputFormat.setLoadModel(hadoopConf, loadModel); CarbonTableOutputFormat format = new CarbonTableOutputFormat(); JobID jobId = new JobID(UUID.randomUUID().toString(), 0); Random random = new Random(); TaskID task = new TaskID(jobId, TaskType.MAP, random.nextInt()); TaskAttemptID attemptID = new TaskAttemptID(task, random.nextInt()); TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, attemptID); this.recordWriter = format.getRecordWriter(context); this.context = context; this.writable = new ObjectArrayWritable(); }
From source file:org.apache.carbondata.sdk.file.CSVCarbonWriter.java
License:Apache License
CSVCarbonWriter(CarbonLoadModel loadModel, Configuration hadoopConf) throws IOException { CarbonTableOutputFormat.setLoadModel(hadoopConf, loadModel); CarbonTableOutputFormat format = new CarbonTableOutputFormat(); JobID jobId = new JobID(UUID.randomUUID().toString(), 0); Random random = new Random(); TaskID task = new TaskID(jobId, TaskType.MAP, random.nextInt()); TaskAttemptID attemptID = new TaskAttemptID(task, random.nextInt()); TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, attemptID); this.recordWriter = format.getRecordWriter(context); this.context = context; this.writable = new ObjectArrayWritable(); }
From source file:org.apache.carbondata.sdk.file.JsonCarbonWriter.java
License:Apache License
JsonCarbonWriter(CarbonLoadModel loadModel, Configuration configuration) throws IOException { CarbonTableOutputFormat.setLoadModel(configuration, loadModel); CarbonTableOutputFormat outputFormat = new CarbonTableOutputFormat(); JobID jobId = new JobID(UUID.randomUUID().toString(), 0); Random random = new Random(); TaskID task = new TaskID(jobId, TaskType.MAP, random.nextInt()); TaskAttemptID attemptID = new TaskAttemptID(task, random.nextInt()); TaskAttemptContextImpl context = new TaskAttemptContextImpl(configuration, attemptID); this.recordWriter = outputFormat.getRecordWriter(context); this.context = context; this.writable = new ObjectArrayWritable(); }
From source file:org.apache.falcon.logging.v2.TaskLogRetrieverYarnTest.java
License:Apache License
@DataProvider(name = "testData") public Object[][] testData() throws IOException, InterruptedException { int samples = getRandomValueInRange(10) + 1; Object[][] resultSet = new Object[samples][2]; for (int count = 0; count < samples; count++) { List<String> expectedResult = new ArrayList<String>(); Cluster cluster = getCluster(getConf()); String jobId = new JobID("job", RANDOM.nextInt(1000)).toString(); boolean success = RANDOM.nextBoolean(); JobID jobID = JobID.forName(jobId); int numEvents = getRandomValueInRange(10) + 1; TaskCompletionEvent[] events = getTaskCompletionEvents(numEvents, jobID); Job job = mock(Job.class); when(cluster.getJob(jobID)).thenReturn(job); when(job.getTaskCompletionEvents(0)).thenReturn(events); for (TaskCompletionEvent event : events) { if (success) { LogParams params = getLogParams(); when(cluster.getLogParams(jobID, event.getTaskAttemptId())).thenReturn(params); String url = SCHEME + getConf().get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp() + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/" + params.getOwner() + "?start=0"; expectedResult.add(url); } else { when(cluster.getJob(jobID)).thenReturn(null); expectedResult = null;//from ww w. j a v a 2 s. c o m } resultSet[count] = new Object[] { jobId, expectedResult }; } } return resultSet; }