List of usage examples for org.apache.hadoop.mapreduce RecordWriter close
public abstract void close(TaskAttemptContext context) throws IOException, InterruptedException;
RecordWriter to future operations. From source file:org.apache.parquet.pig.PerfTest2.java
License:Apache License
public static void write(String out) throws IOException, ParserException, InterruptedException, ExecException { {/*w w w. j a va 2 s . c om*/ StringBuilder schemaString = new StringBuilder("a0: chararray"); for (int i = 1; i < COLUMN_COUNT; i++) { schemaString.append(", a" + i + ": chararray"); } String location = out; String schema = schemaString.toString(); StoreFuncInterface storer = new ParquetStorer(); Job job = new Job(conf); storer.setStoreFuncUDFContextSignature("sig"); String absPath = storer.relToAbsPathForStoreLocation(location, new Path(new File(".").getAbsoluteFile().toURI())); storer.setStoreLocation(absPath, job); storer.checkSchema(new ResourceSchema(Utils.getSchemaFromString(schema))); @SuppressWarnings("unchecked") // that's how the base class is defined OutputFormat<Void, Tuple> outputFormat = storer.getOutputFormat(); // it's ContextUtil.getConfiguration(job) and not just conf ! JobContext jobContext = ContextUtil.newJobContext(ContextUtil.getConfiguration(job), new JobID("jt", jobid++)); outputFormat.checkOutputSpecs(jobContext); if (schema != null) { ResourceSchema resourceSchema = new ResourceSchema(Utils.getSchemaFromString(schema)); storer.checkSchema(resourceSchema); if (storer instanceof StoreMetadata) { ((StoreMetadata) storer).storeSchema(resourceSchema, absPath, job); } } TaskAttemptContext taskAttemptContext = ContextUtil.newTaskAttemptContext( ContextUtil.getConfiguration(job), new TaskAttemptID("jt", jobid, true, 1, 0)); RecordWriter<Void, Tuple> recordWriter = outputFormat.getRecordWriter(taskAttemptContext); storer.prepareToWrite(recordWriter); for (int i = 0; i < ROW_COUNT; i++) { Tuple tuple = TupleFactory.getInstance().newTuple(COLUMN_COUNT); for (int j = 0; j < COLUMN_COUNT; j++) { tuple.set(j, "a" + i + "_" + j); } storer.putNext(tuple); } recordWriter.close(taskAttemptContext); OutputCommitter outputCommitter = outputFormat.getOutputCommitter(taskAttemptContext); outputCommitter.commitTask(taskAttemptContext); outputCommitter.commitJob(jobContext); } }
From source file:org.apache.pig.impl.io.PigFile.java
License:Apache License
public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException { Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties()); // create a simulated JobContext JobContext jc = HadoopShims.createJobContext(conf, new JobID()); StoreFuncInterface sfunc = (StoreFuncInterface) PigContext.instantiateFuncFromSpec(storeFuncSpec); OutputFormat<?, ?> of = sfunc.getOutputFormat(); POStore store = new POStore(new OperatorKey()); store.setSFile(new FileSpec(file, storeFuncSpec)); PigOutputFormat.setLocation(jc, store); OutputCommitter oc;/* w w w . j av a2s . c o m*/ // create a simulated TaskAttemptContext TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, HadoopShims.getNewTaskAttemptID()); PigOutputFormat.setLocation(tac, store); RecordWriter<?, ?> rw; try { of.checkOutputSpecs(jc); oc = of.getOutputCommitter(tac); oc.setupJob(jc); oc.setupTask(tac); rw = of.getRecordWriter(tac); sfunc.prepareToWrite(rw); for (Iterator<Tuple> it = data.iterator(); it.hasNext();) { Tuple row = it.next(); sfunc.putNext(row); } rw.close(tac); } catch (InterruptedException e) { throw new IOException(e); } if (oc.needsTaskCommit(tac)) { oc.commitTask(tac); } HadoopShims.commitOrCleanup(oc, jc); }
From source file:org.apache.rya.accumulo.mr.RyaOutputFormatTest.java
License:Apache License
private void write(final RyaStatement... input) throws IOException, InterruptedException { final RecordWriter<Writable, RyaStatementWritable> writer = new RyaOutputFormat.RyaRecordWriter( job.getConfiguration());/*from w w w .j a v a 2 s.c om*/ for (final RyaStatement rstmt : input) { final RyaStatementWritable rsw = new RyaStatementWritable(rstmt, ryaContext); writer.write(new Text("unused"), rsw); } writer.close(null); }
From source file:org.apache.sqoop.job.mr.TestSqoopLoader.java
License:Apache License
@Test public void testSuccessfulContinuousLoader() throws Throwable { conf.set(MRJobConstants.JOB_ETL_LOADER, GoodContinuousLoader.class.getName()); SqoopLoader executor = new SqoopLoader(jobContextMock, GoodContinuousLoader.class.getName(), getIDF(), getMatcher());//from www .j ava 2s .c o m RecordWriter<Text, NullWritable> writer = executor.getRecordWriter(); IntermediateDataFormat<?> dataFormat = MRJobTestUtil.getTestIDF(); for (int i = 0; i < 10; i++) { StringBuilder builder = new StringBuilder(); for (int count = 0; count < 100; count++) { builder.append(String.valueOf(count)); if (count != 99) { builder.append(","); } } dataFormat.setCSVTextData(builder.toString()); writer.write(new Text(dataFormat.getCSVTextData()), null); } writer.close(null); verify(jobContextMock, times(1)).getConfiguration(); verify(jobContextMock, times(1)).getCounter(SqoopCounters.ROWS_WRITTEN); }
From source file:org.apache.sqoop.job.mr.TestSqoopLoader.java
License:Apache License
@Test(expectedExceptions = SqoopException.class) public void testSuccessfulLoader() throws Throwable { SqoopLoader executor = new SqoopLoader(jobContextMock, GoodLoader.class.getName(), getIDF(), getMatcher()); RecordWriter<Text, NullWritable> writer = executor.getRecordWriter(); IntermediateDataFormat<?> dataFormat = MRJobTestUtil.getTestIDF(); StringBuilder builder = new StringBuilder(); for (int count = 0; count < 100; count++) { builder.append(String.valueOf(count)); if (count != 99) { builder.append(","); }// ww w. j av a 2 s . c o m } dataFormat.setCSVTextData(builder.toString()); writer.write(new Text(dataFormat.getCSVTextData()), null); // Allow writer to complete. TimeUnit.SECONDS.sleep(5); writer.close(null); verify(jobContextMock, times(1)).getConfiguration(); verify(jobContextMock, times(1)).getCounter(SqoopCounters.ROWS_WRITTEN); }
From source file:org.apache.sqoop.job.mr.TestSqoopLoader.java
License:Apache License
@Test(expectedExceptions = ConcurrentModificationException.class) public void testThrowingContinuousLoader() throws Throwable { conf.set(MRJobConstants.JOB_ETL_LOADER, ThrowingContinuousLoader.class.getName()); SqoopLoader executor = new SqoopLoader(jobContextMock, ThrowingContinuousLoader.class.getName(), getIDF(), getMatcher());//from w w w.java2 s .c o m RecordWriter<Text, NullWritable> writer = executor.getRecordWriter(); IntermediateDataFormat<?> dataFormat = MRJobTestUtil.getTestIDF(); try { for (int i = 0; i < 10; i++) { StringBuilder builder = new StringBuilder(); for (int count = 0; count < 100; count++) { builder.append(String.valueOf(count)); if (count != 99) { builder.append(","); } } dataFormat.setCSVTextData(builder.toString()); writer.write(new Text(dataFormat.getCSVTextData()), null); } writer.close(null); } catch (SqoopException ex) { throw ex.getCause(); } }
From source file:org.apache.sqoop.job.mr.TestSqoopOutputFormatLoadExecutor.java
License:Apache License
@Test public void testSuccessfulContinuousLoader() throws Throwable { conf.set(MRJobConstants.JOB_ETL_LOADER, GoodContinuousLoader.class.getName()); SqoopOutputFormatLoadExecutor executor = new SqoopOutputFormatLoadExecutor(true, GoodContinuousLoader.class.getName()); RecordWriter<SqoopWritable, NullWritable> writer = executor.getRecordWriter(); IntermediateDataFormat data = new CSVIntermediateDataFormat(); SqoopWritable writable = new SqoopWritable(); for (int i = 0; i < 10; i++) { StringBuilder builder = new StringBuilder(); for (int count = 0; count < 100; count++) { builder.append(String.valueOf(count)); if (count != 99) { builder.append(","); }/*from w w w . jav a 2 s . co m*/ } data.setTextData(builder.toString()); writable.setString(data.getTextData()); writer.write(writable, null); } writer.close(null); }
From source file:org.apache.sqoop.job.mr.TestSqoopOutputFormatLoadExecutor.java
License:Apache License
@Test(expected = SqoopException.class) public void testSuccessfulLoader() throws Throwable { SqoopOutputFormatLoadExecutor executor = new SqoopOutputFormatLoadExecutor(true, GoodLoader.class.getName()); RecordWriter<SqoopWritable, NullWritable> writer = executor.getRecordWriter(); IntermediateDataFormat data = new CSVIntermediateDataFormat(); SqoopWritable writable = new SqoopWritable(); StringBuilder builder = new StringBuilder(); for (int count = 0; count < 100; count++) { builder.append(String.valueOf(count)); if (count != 99) { builder.append(","); }//w ww . j a v a 2s .c o m } data.setTextData(builder.toString()); writable.setString(data.getTextData()); writer.write(writable, null); //Allow writer to complete. TimeUnit.SECONDS.sleep(5); writer.close(null); }
From source file:org.apache.sqoop.job.mr.TestSqoopOutputFormatLoadExecutor.java
License:Apache License
@Test(expected = ConcurrentModificationException.class) public void testThrowingContinuousLoader() throws Throwable { conf.set(MRJobConstants.JOB_ETL_LOADER, ThrowingContinuousLoader.class.getName()); SqoopOutputFormatLoadExecutor executor = new SqoopOutputFormatLoadExecutor(true, ThrowingContinuousLoader.class.getName()); RecordWriter<SqoopWritable, NullWritable> writer = executor.getRecordWriter(); IntermediateDataFormat data = new CSVIntermediateDataFormat(); SqoopWritable writable = new SqoopWritable(); try {//from w w w. ja v a2s . c o m for (int i = 0; i < 10; i++) { StringBuilder builder = new StringBuilder(); for (int count = 0; count < 100; count++) { builder.append(String.valueOf(count)); if (count != 99) { builder.append(","); } } data.setTextData(builder.toString()); writable.setString(data.getTextData()); writer.write(writable, null); } writer.close(null); } catch (SqoopException ex) { throw ex.getCause(); } }
From source file:org.apache.tez.mapreduce.combine.MRCombiner.java
License:Apache License
private void runNewCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws InterruptedException, IOException { RecordWriter recordWriter = new RecordWriter() { @Override/*from w w w . j ava 2 s.c o m*/ public void write(Object key, Object value) throws IOException, InterruptedException { writer.append(key, value); } @Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { // Will be closed by whoever invokes the combiner. } }; Class<? extends org.apache.hadoop.mapreduce.Reducer> reducerClazz = (Class<? extends org.apache.hadoop.mapreduce.Reducer>) conf .getClass(MRJobConfig.COMBINE_CLASS_ATTR, null, org.apache.hadoop.mapreduce.Reducer.class); org.apache.hadoop.mapreduce.Reducer reducer = ReflectionUtils.newInstance(reducerClazz, conf); org.apache.hadoop.mapreduce.Reducer.Context reducerContext = createReduceContext(conf, mrTaskAttemptID, rawIter, new MRCounters.MRCounter(combineInputKeyCounter), new MRCounters.MRCounter(combineInputValueCounter), recordWriter, reporter, (RawComparator) comparator, keyClass, valClass); reducer.run(reducerContext); recordWriter.close(reducerContext); }