List of usage examples for org.apache.hadoop.io Text readFields
@Override public void readFields(DataInput in) throws IOException
From source file:org.apache.accumulo.hadoopImpl.mapreduce.InputTableConfig.java
License:Apache License
@Override public void readFields(DataInput dataInput) throws IOException { // load iterators long iterSize = dataInput.readInt(); if (iterSize > 0) iterators = new LinkedHashMap<>(); for (int i = 0; i < iterSize; i++) { IteratorSetting newIter = new IteratorSetting(dataInput); iterators.put(newIter.getName(), newIter); }/* w w w.j a v a 2s . co m*/ // load ranges long rangeSize = dataInput.readInt(); if (rangeSize > 0) ranges = new ArrayList<>(); for (int i = 0; i < rangeSize; i++) { Range range = new Range(); range.readFields(dataInput); ranges.add(range); } // load columns long columnSize = dataInput.readInt(); if (columnSize > 0) columns = new HashSet<>(); for (int i = 0; i < columnSize; i++) { long numPairs = dataInput.readInt(); Text colFam = new Text(); colFam.readFields(dataInput); if (numPairs == 1) { columns.add(new IteratorSetting.Column(colFam, null)); } else if (numPairs == 2) { Text colQual = new Text(); colQual.readFields(dataInput); columns.add(new IteratorSetting.Column(colFam, colQual)); } } autoAdjustRanges = dataInput.readBoolean(); useLocalIterators = dataInput.readBoolean(); useIsolatedScanners = dataInput.readBoolean(); offlineScan = dataInput.readBoolean(); if (dataInput.readBoolean()) { samplerConfig = new SamplerConfigurationImpl(dataInput).toSamplerConfiguration(); } executionHints = new HashMap<>(); int numHints = dataInput.readInt(); for (int i = 0; i < numHints; i++) { String k = dataInput.readUTF(); String v = dataInput.readUTF(); executionHints.put(k, v); } }
From source file:org.apache.gobblin.runtime.TaskState.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { Text text = new Text(); text.readFields(in); this.jobId = text.toString().intern(); text.readFields(in);// ww w.j av a 2 s .c om this.taskId = text.toString().intern(); this.taskAttemptId = Optional.absent(); this.setId(this.taskId); this.startTime = in.readLong(); this.endTime = in.readLong(); this.duration = in.readLong(); super.readFields(in); }
From source file:org.apache.hama.bsp.message.TestMessageIO.java
License:Apache License
public void testSpillInputStream() throws Exception { File f = null;/* w w w.j a v a2s .co m*/ try { String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar + "testSpillInputStream.txt"; Configuration conf = new HamaConfiguration(); SpilledDataProcessor processor = new WriteSpilledDataProcessor(fileName); processor.init(conf); SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer(2, 1024, 1024, true, processor); Text text = new Text("Testing the spillage of spilling buffer"); for (int i = 0; i < 100; ++i) { text.write(outputBuffer); outputBuffer.markRecordEnd(); } assertTrue(outputBuffer != null); assertTrue(outputBuffer.size() == 4000); assertTrue(outputBuffer.hasSpilled()); f = new File(fileName); assertTrue(f.exists()); outputBuffer.close(); assertTrue(f.length() == 4000);// + (4000 / 1024 + 1) * 4)); SpilledDataInputBuffer inputBuffer = outputBuffer.getInputStreamToRead(fileName); for (int i = 0; i < 100; ++i) { text.readFields(inputBuffer); assertTrue("Testing the spillage of spilling buffer".equals(text.toString())); text.clear(); } try { text.readFields(inputBuffer); assertTrue(false); } catch (EOFException eof) { assertTrue(true); } inputBuffer.close(); inputBuffer.completeReading(false); assertTrue(f.exists()); inputBuffer.completeReading(true); assertFalse(f.exists()); } finally { if (f != null) { if (f.exists()) { f.delete(); } } } }
From source file:org.apache.hama.bsp.TestCheckpoint.java
License:Apache License
@SuppressWarnings("rawtypes") public void testCheckpoint() throws Exception { Configuration config = new Configuration(); config.set(SyncServiceFactory.SYNC_CLIENT_CLASS, TempSyncClient.class.getName()); config.setBoolean(Constants.FAULT_TOLERANCE_FLAG, true); config.set(Constants.FAULT_TOLERANCE_CLASS, AsyncRcvdMsgCheckpointImpl.class.getName()); config.setBoolean(Constants.CHECKPOINT_ENABLED, true); int port = BSPNetUtils.getFreePort(12502); LOG.info("Got port = " + port); config.set(Constants.PEER_HOST, Constants.DEFAULT_PEER_HOST); config.setInt(Constants.PEER_PORT, port); config.set("bsp.output.dir", "/tmp/hama-test_out"); config.set("bsp.local.dir", "/tmp/hama-test"); FileSystem dfs = FileSystem.get(config); BSPJob job = new BSPJob(new BSPJobID("checkpttest", 1), "/tmp"); TaskAttemptID taskId = new TaskAttemptID(new TaskID(job.getJobID(), 1), 1); TestMessageManager messenger = new TestMessageManager(); PeerSyncClient syncClient = SyncServiceFactory.getPeerSyncClient(config); BSPPeer bspTask = new TestBSPPeer(job, config, taskId, new Counters(), -1L, (BSPPeerSyncClient) syncClient, messenger, TaskStatus.State.RUNNING); assertNotNull("BSPPeerImpl should not be null.", bspTask); LOG.info("Created bsp peer and other parameters"); @SuppressWarnings("unused") FaultTolerantPeerService<Text> service = null; bspTask.sync();/*from w w w . jav a 2s.c om*/ LOG.info("Completed first sync."); checkSuperstepMsgCount(syncClient, bspTask, job, 1L, 0L); Text txtMessage = new Text("data"); messenger.addMessage(txtMessage); bspTask.sync(); LOG.info("Completed second sync."); checkSuperstepMsgCount(syncClient, bspTask, job, 2L, 1L); // Checking the messages for superstep 2 and peer id 1 String expectedPath = "checkpoint/job_checkpttest_0001/2/1"; FSDataInputStream in = dfs.open(new Path(expectedPath)); String className = in.readUTF(); Text message = (Text) ReflectionUtils.newInstance(Class.forName(className), config); message.readFields(in); assertEquals("data", message.toString()); dfs.delete(new Path("checkpoint"), true); }
From source file:org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.java
License:Apache License
@Override public void readFields(DataInput dataInput) throws IOException { this.typeId = dataInput.readByte(); Text inputKey = new Text(); inputKey.readFields(dataInput); init(inputKey, typeId);// www . j a v a 2 s. c om }
From source file:org.apache.mahout.common.StringTuple.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { int len = in.readInt(); tuple = Lists.newArrayListWithCapacity(len); Text value = new Text(); for (int i = 0; i < len; i++) { value.readFields(in); tuple.add(value.toString());/*from w w w . java 2s.c o m*/ } }
From source file:org.apache.tez.mapreduce.processor.map.TestMapProcessor.java
License:Apache License
@Test(timeout = 5000) public void testMapProcessor() throws Exception { String dagName = "mrdag0"; String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName(); JobConf jobConf = new JobConf(defaultConf); setUpJobConf(jobConf);/* w w w . j a v a2 s . co m*/ MRHelpers.translateMRConfToTez(jobConf); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); Path mapInput = new Path(workDir, "map0"); MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(MRRuntimeProtos.MRInputUserPayloadProto .newBuilder().setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)) .build().toByteArray()))), 1); OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); LogicalIOProcessorRuntimeTask task = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0, new Path(workDir, "map0"), new TestUmbilical(), dagName, vertexName, Collections.singletonList(mapInputSpec), Collections.singletonList(mapOutputSpec)); task.initialize(); task.run(); task.close(); OutputContext outputContext = task.getOutputContexts().iterator().next(); TezTaskOutput mapOutputs = new TezTaskOutputFiles(jobConf, outputContext.getUniqueIdentifier()); // TODO NEWTEZ FIXME OutputCommitter verification // MRTask mrTask = (MRTask)t.getProcessor(); // Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask // .getCommitter().getClass().getName()); // t.close(); Path mapOutputFile = getMapOutputFile(jobConf, outputContext); LOG.info("mapOutputFile = " + mapOutputFile); IFile.Reader reader = new IFile.Reader(localFs, mapOutputFile, null, null, null, false, 0, -1); LongWritable key = new LongWritable(); Text value = new Text(); DataInputBuffer keyBuf = new DataInputBuffer(); DataInputBuffer valueBuf = new DataInputBuffer(); long prev = Long.MIN_VALUE; while (reader.nextRawKey(keyBuf)) { reader.nextRawValue(valueBuf); key.readFields(keyBuf); value.readFields(valueBuf); if (prev != Long.MIN_VALUE) { assert (prev <= key.get()); prev = key.get(); } LOG.info("key = " + key.get() + "; value = " + value); } reader.close(); }
From source file:org.apache.tez.runtime.library.common.sort.impl.TestTezMerger.java
License:Apache License
/** * Verify if the records are as per the expected data set * * @param records//from ww w . j av a 2s . co m * @param expectedResult * @throws IOException */ private void verify(TezRawKeyValueIterator records, String[][] expectedResult) throws IOException { //Iterate through merged dataset (shouldn't throw any exceptions) int i = 0; while (records.next()) { DataInputBuffer key = records.getKey(); DataInputBuffer value = records.getValue(); Text k = new Text(); k.readFields(key); Text v = new Text(); v.readFields(value); assertTrue(k.toString().equals(expectedResult[i][0])); String correctResult = expectedResult[i][1]; if (records.isSameKey()) { assertTrue("Expected " + correctResult, correctResult.equalsIgnoreCase(SAME_KEY)); LOG.info("\tSame Key : key=" + k + ", val=" + v); } else { assertTrue("Expected " + correctResult, correctResult.equalsIgnoreCase(DIFF_KEY)); LOG.info("key=" + k + ", val=" + v); } i++; } }
From source file:org.apache.tez.runtime.library.common.writers.TestUnorderedPartitionedKVWriter.java
License:Apache License
public void textTest(int numRegularRecords, int numPartitions, long availableMemory, int numLargeKeys, int numLargevalues, int numLargeKvPairs) throws IOException, InterruptedException { Partitioner partitioner = new HashPartitioner(); ApplicationId appId = ApplicationId.newInstance(10000, 1); TezCounters counters = new TezCounters(); String uniqueId = UUID.randomUUID().toString(); OutputContext outputContext = createMockOutputContext(counters, appId, uniqueId); Random random = new Random(); Configuration conf = createConfiguration(outputContext, Text.class, Text.class, shouldCompress, -1, HashPartitioner.class); CompressionCodec codec = null;//from ww w. j a v a 2s .co m if (shouldCompress) { codec = new DefaultCodec(); ((Configurable) codec).setConf(conf); } int numRecordsWritten = 0; Map<Integer, Multimap<String, String>> expectedValues = new HashMap<Integer, Multimap<String, String>>(); for (int i = 0; i < numPartitions; i++) { expectedValues.put(i, LinkedListMultimap.<String, String>create()); } UnorderedPartitionedKVWriter kvWriter = new UnorderedPartitionedKVWriterForTest(outputContext, conf, numPartitions, availableMemory); int sizePerBuffer = kvWriter.sizePerBuffer; BitSet partitionsWithData = new BitSet(numPartitions); Text keyText = new Text(); Text valText = new Text(); for (int i = 0; i < numRegularRecords; i++) { String key = createRandomString(Math.abs(random.nextInt(10))); String val = createRandomString(Math.abs(random.nextInt(20))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } // Write Large key records for (int i = 0; i < numLargeKeys; i++) { String key = createRandomString(sizePerBuffer + Math.abs(random.nextInt(100))); String val = createRandomString(Math.abs(random.nextInt(20))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } // Write Large val records for (int i = 0; i < numLargevalues; i++) { String key = createRandomString(Math.abs(random.nextInt(10))); String val = createRandomString(sizePerBuffer + Math.abs(random.nextInt(100))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } // Write records where key + val are large (but both can fit in the buffer individually) for (int i = 0; i < numLargeKvPairs; i++) { String key = createRandomString(sizePerBuffer / 2 + Math.abs(random.nextInt(100))); String val = createRandomString(sizePerBuffer / 2 + Math.abs(random.nextInt(100))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } List<Event> events = kvWriter.close(); verify(outputContext, never()).fatalError(any(Throwable.class), any(String.class)); TezCounter outputLargeRecordsCounter = counters.findCounter(TaskCounter.OUTPUT_LARGE_RECORDS); assertEquals(numLargeKeys + numLargevalues + numLargeKvPairs, outputLargeRecordsCounter.getValue()); // Validate the event assertEquals(1, events.size()); assertTrue(events.get(0) instanceof CompositeDataMovementEvent); CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) events.get(0); assertEquals(0, cdme.getSourceIndexStart()); assertEquals(numPartitions, cdme.getCount()); DataMovementEventPayloadProto eventProto = DataMovementEventPayloadProto .parseFrom(ByteString.copyFrom(cdme.getUserPayload())); assertFalse(eventProto.hasData()); BitSet emptyPartitionBits = null; if (partitionsWithData.cardinality() != numPartitions) { assertTrue(eventProto.hasEmptyPartitions()); byte[] emptyPartitions = TezCommonUtils .decompressByteStringToByteArray(eventProto.getEmptyPartitions()); emptyPartitionBits = TezUtilsInternal.fromByteArray(emptyPartitions); assertEquals(numPartitions - partitionsWithData.cardinality(), emptyPartitionBits.cardinality()); } else { assertFalse(eventProto.hasEmptyPartitions()); emptyPartitionBits = new BitSet(numPartitions); } assertEquals(HOST_STRING, eventProto.getHost()); assertEquals(SHUFFLE_PORT, eventProto.getPort()); assertEquals(uniqueId, eventProto.getPathComponent()); // Verify the data // Verify the actual data TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId); Path outputFilePath = kvWriter.finalOutPath; Path spillFilePath = kvWriter.finalIndexPath; if (numRecordsWritten > 0) { assertTrue(localFs.exists(outputFilePath)); assertTrue(localFs.exists(spillFilePath)); } else { return; } // Special case for 0 records. TezSpillRecord spillRecord = new TezSpillRecord(spillFilePath, conf); DataInputBuffer keyBuffer = new DataInputBuffer(); DataInputBuffer valBuffer = new DataInputBuffer(); Text keyDeser = new Text(); Text valDeser = new Text(); for (int i = 0; i < numPartitions; i++) { if (emptyPartitionBits.get(i)) { continue; } TezIndexRecord indexRecord = spillRecord.getIndex(i); FSDataInputStream inStream = FileSystem.getLocal(conf).open(outputFilePath); inStream.seek(indexRecord.getStartOffset()); IFile.Reader reader = new IFile.Reader(inStream, indexRecord.getPartLength(), codec, null, null, false, 0, -1); while (reader.nextRawKey(keyBuffer)) { reader.nextRawValue(valBuffer); keyDeser.readFields(keyBuffer); valDeser.readFields(valBuffer); int partition = partitioner.getPartition(keyDeser, valDeser, numPartitions); assertTrue(expectedValues.get(partition).remove(keyDeser.toString(), valDeser.toString())); } inStream.close(); } for (int i = 0; i < numPartitions; i++) { assertEquals(0, expectedValues.get(i).size()); expectedValues.remove(i); } assertEquals(0, expectedValues.size()); }
From source file:org.cstor.cproc.container.ListWritable.java
License:Apache License
public void readFields(DataInput in) throws IOException { values = new Text[in.readInt()]; // construct values for (int i = 0; i < values.length; i++) { Text value = new Text(); value.readFields(in); // read a value values[i] = value; // store it in values }/*from ww w . j a va 2s . c o m*/ }