List of usage examples for org.apache.hadoop.mapreduce JobContext getJobID
public JobID getJobID();
From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter.java
License:Apache License
static public JobContext setUpContext(JobContext context, POStore store) throws IOException { // make a copy of the context so that the actions after this call // do not end up updating the same context JobContext contextCopy = HadoopShims.createJobContext(context.getConfiguration(), context.getJobID()); MapRedUtil.setupUDFContext(context.getConfiguration()); // call setLocation() on the storeFunc so that if there are any // side effects like setting map.output.dir on the Configuration // in the Context are needed by the OutputCommitter, those actions // will be done before the committer is created. Also the String // version of StoreFunc for the specific store need // to be set up in the context in case the committer needs them PigOutputFormat.setLocation(contextCopy, store); return contextCopy; }
From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat.java
License:Apache License
private void checkOutputSpecsHelper(List<POStore> stores, JobContext jobcontext) throws IOException, InterruptedException { for (POStore store : stores) { // make a copy of the original JobContext so that // each OutputFormat get a different copy JobContext jobContextCopy = HadoopShims.createJobContext(jobcontext.getConfiguration(), jobcontext.getJobID()); // set output location PigOutputFormat.setLocation(jobContextCopy, store); StoreFuncInterface sFunc = store.getStoreFunc(); OutputFormat of = sFunc.getOutputFormat(); // The above call should have update the conf in the JobContext // to have the output location - now call checkOutputSpecs() try {/* w ww. j a va 2s .c om*/ of.checkOutputSpecs(jobContextCopy); } catch (IOException ioe) { boolean shouldThrowException = true; if (sFunc instanceof OverwritableStoreFunc) { if (((OverwritableStoreFunc) sFunc).shouldOverwrite()) { if (ioe instanceof FileAlreadyExistsException || ioe instanceof org.apache.hadoop.fs.FileAlreadyExistsException) { shouldThrowException = false; } } } if (shouldThrowException) throw ioe; } } }
From source file:org.goldenorb.io.input.VertexInput.java
License:Apache License
/** * *///w w w . j a v a 2 s . c om @SuppressWarnings("unchecked") public void initialize() { // rebuild the input split org.apache.hadoop.mapreduce.InputSplit split = null; DataInputBuffer splitBuffer = new DataInputBuffer(); splitBuffer.reset(rawSplit.getBytes(), 0, rawSplit.getLength()); SerializationFactory factory = new SerializationFactory(orbConf); Deserializer<? extends org.apache.hadoop.mapreduce.InputSplit> deserializer; try { deserializer = (Deserializer<? extends org.apache.hadoop.mapreduce.InputSplit>) factory .getDeserializer(orbConf.getClassByName(splitClass)); deserializer.open(splitBuffer); split = deserializer.deserialize(null); JobConf job = new JobConf(orbConf); JobContext jobContext = new JobContext(job, new JobID(getOrbConf().getJobNumber(), 0)); InputFormat<INPUT_KEY, INPUT_VALUE> inputFormat; inputFormat = (InputFormat<INPUT_KEY, INPUT_VALUE>) ReflectionUtils .newInstance(jobContext.getInputFormatClass(), orbConf); TaskAttemptContext tao = new TaskAttemptContext(job, new TaskAttemptID(new TaskID(jobContext.getJobID(), true, partitionID), 0)); recordReader = inputFormat.createRecordReader(split, tao); recordReader.initialize(split, tao); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } }
From source file:org.goldenorb.OrbPartition.java
License:Apache License
private void dumpData() { Configuration conf = new Configuration(); Job job = null;//w w w.j a v a 2 s.co m JobContext jobContext = null; TaskAttemptContext tao = null; RecordWriter rw; VertexWriter vw; FileOutputFormat outputFormat; boolean tryAgain = true; int count = 0; while (tryAgain && count < 15) try { count++; tryAgain = false; if (job == null) { job = new Job(conf); job.setOutputFormatClass(TextOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(new String(getOrbConf().getNameNode() + getOrbConf().getFileOutputPath()))); } if (jobContext == null) { jobContext = new JobContext(job.getConfiguration(), new JobID()); } System.out.println(jobContext.getConfiguration().get("mapred.output.dir")); tao = new TaskAttemptContext(jobContext.getConfiguration(), new TaskAttemptID(new TaskID(jobContext.getJobID(), true, getPartitionID()), 0)); outputFormat = (FileOutputFormat) tao.getOutputFormatClass().newInstance(); rw = outputFormat.getRecordWriter(tao); vw = (VertexWriter) getOrbConf().getVertexOutputFormatClass().newInstance(); for (Vertex v : vertices.values()) { OrbContext oc = vw.vertexWrite(v); rw.write(oc.getKey(), oc.getValue()); // orbLogger.info("Partition: " + Integer.toString(partitionId) + "writing: " + // oc.getKey().toString() + ", " + oc.getValue().toString()); } rw.close(tao); FileOutputCommitter cm = (FileOutputCommitter) outputFormat.getOutputCommitter(tao); if (cm.needsTaskCommit(tao)) { cm.commitTask(tao); cm.cleanupJob(jobContext); } else { cm.cleanupJob(jobContext); tryAgain = true; } } catch (IOException e) { tryAgain = true; e.printStackTrace(); } catch (InstantiationException e) { tryAgain = true; e.printStackTrace(); } catch (IllegalAccessException e) { tryAgain = true; e.printStackTrace(); } catch (ClassNotFoundException e) { tryAgain = true; e.printStackTrace(); } catch (InterruptedException e) { tryAgain = true; e.printStackTrace(); } if (tryAgain) { synchronized (this) { try { wait(1000); } catch (InterruptedException e) { e.printStackTrace(); } } } }
From source file:org.kitesdk.data.mapreduce.DatasetKeyOutputFormat.java
License:Apache License
private static String getJobDatasetName(JobContext jobContext) { return Hadoop.JobContext.getJobID.invoke(jobContext).toString(); }