List of usage examples for org.apache.hadoop.io LongWritable readFields
@Override
public void readFields(DataInput in) throws IOException
From source file:HistogramBucket.java
License:Apache License
@Override public void readFields(DataInput di) throws IOException { attribute.readFields(di);/* ww w. java 2s . c om*/ LongWritable arraySize = new LongWritable(); arraySize.readFields(di); splits = new ArrayList<DoubleWritable>(); for (int i = 0; i < Integer.parseInt(arraySize.toString()); i++) { DoubleWritable d = new DoubleWritable(); d.readFields(di); splits.add(d); } }
From source file:com.twitter.elephanttwin.io.ListLongWritable.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { int numElements = in.readInt(); list = new ArrayList<LongWritable>(numElements); for (int cnt = 0; cnt < numElements; cnt++) { LongWritable l = new LongWritable(); l.readFields(in); list.add(l);//from w ww.j a va 2 s . c o m } }
From source file:com.twitter.elephanttwin.io.ListLongWritable.java
License:Apache License
public static List<LongWritable> readLongList(DataInput in) throws IOException { int numElements = in.readInt(); List<LongWritable> retlist = new ArrayList<LongWritable>(numElements); for (int cnt = 0; cnt < numElements; cnt++) { LongWritable l = new LongWritable(); l.readFields(in); retlist.add(l);//from ww w. j a v a 2 s . co m } return retlist; }
From source file:com.twitter.hraven.FlowKey.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); LongWritable lw = new LongWritable(); lw.readFields(in); this.runId = lw.get(); }
From source file:com.twitter.hraven.JobId.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { LongWritable lw = new LongWritable(); lw.readFields(in); this.jobEpoch = lw.get(); lw.readFields(in);// w ww .j a va2 s . co m this.jobSequence = lw.get(); }
From source file:it.uniroma1.bdc.tesi.piccioli.giraphstandalone.ksimplecycle.TextValueAndSetPerSuperstep.java
@Override public void readFields(DataInput in) throws IOException { int size;//from w w w . ja va2 s. co m // value.readFields(in); size = in.readInt();//Leggo numero di key da inserire nella MAP for (int i = 0; i < size; i++) { LongWritable key = new LongWritable(); key.readFields(in);//Leggo Chiave DoubleWritable valueh = new DoubleWritable(); valueh.readFields(in);//Leggo Chiave this.setPerSuperstep.put(key, valueh); } // }
From source file:org.apache.giraph.comm.requests.SendWorkerAggregatorsRequest.java
License:Apache License
@Override public void doRequest(ServerData serverData) { DataInput input = getDataInput(); OwnerAggregatorServerData aggregatorData = serverData.getOwnerAggregatorData(); try {/*from ww w. j ava 2s .com*/ int num = input.readInt(); for (int i = 0; i < num; i++) { String name = input.readUTF(); GlobalCommType type = GlobalCommType.values()[input.readByte()]; if (type == GlobalCommType.SPECIAL_COUNT) { LongWritable value = new LongWritable(); value.readFields(input); aggregatorData.receivedRequestCountFromWorker(value.get(), getSenderTaskId()); } else if (type == GlobalCommType.REDUCED_VALUE) { Writable value = aggregatorData.createInitialValue(name); value.readFields(input); aggregatorData.reduce(name, value); } else { throw new IllegalStateException("SendWorkerAggregatorsRequest received " + type); } } } catch (IOException e) { throw new IllegalStateException("doRequest: " + "IOException occurred while processing request", e); } aggregatorData.receivedRequestFromWorker(); }
From source file:org.apache.tez.mapreduce.processor.map.TestMapProcessor.java
License:Apache License
@Test(timeout = 5000) public void testMapProcessor() throws Exception { String dagName = "mrdag0"; String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName(); JobConf jobConf = new JobConf(defaultConf); setUpJobConf(jobConf);/*from ww w . j a va2 s . c o m*/ MRHelpers.translateMRConfToTez(jobConf); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); Path mapInput = new Path(workDir, "map0"); MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(MRRuntimeProtos.MRInputUserPayloadProto .newBuilder().setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)) .build().toByteArray()))), 1); OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); LogicalIOProcessorRuntimeTask task = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0, new Path(workDir, "map0"), new TestUmbilical(), dagName, vertexName, Collections.singletonList(mapInputSpec), Collections.singletonList(mapOutputSpec)); task.initialize(); task.run(); task.close(); OutputContext outputContext = task.getOutputContexts().iterator().next(); TezTaskOutput mapOutputs = new TezTaskOutputFiles(jobConf, outputContext.getUniqueIdentifier()); // TODO NEWTEZ FIXME OutputCommitter verification // MRTask mrTask = (MRTask)t.getProcessor(); // Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask // .getCommitter().getClass().getName()); // t.close(); Path mapOutputFile = getMapOutputFile(jobConf, outputContext); LOG.info("mapOutputFile = " + mapOutputFile); IFile.Reader reader = new IFile.Reader(localFs, mapOutputFile, null, null, null, false, 0, -1); LongWritable key = new LongWritable(); Text value = new Text(); DataInputBuffer keyBuf = new DataInputBuffer(); DataInputBuffer valueBuf = new DataInputBuffer(); long prev = Long.MIN_VALUE; while (reader.nextRawKey(keyBuf)) { reader.nextRawValue(valueBuf); key.readFields(keyBuf); value.readFields(valueBuf); if (prev != Long.MIN_VALUE) { assert (prev <= key.get()); prev = key.get(); } LOG.info("key = " + key.get() + "; value = " + value); } reader.close(); }
From source file:org.apache.tez.runtime.library.common.sort.impl.TestTezMerger.java
License:Apache License
private void merge(List<Path> pathList, int mergeFactor, RawComparator rc) throws Exception { //Merge datasets TezMerger merger = new TezMerger(); TezRawKeyValueIterator records = merger.merge(defaultConf, localFs, IntWritable.class, LongWritable.class, null, false, 0, 1024, pathList.toArray(new Path[pathList.size()]), true, mergeFactor, new Path(workDir, "tmp_" + System.nanoTime()), ((rc == null) ? comparator : rc), new Reporter(), null, null, null, new Progress()); //Verify the merged data is correct Map<Integer, Integer> dataMap = Maps.newHashMap(); int pk = -1;/*from ww w . j a v a 2 s .c om*/ while (records.next()) { DataInputBuffer key = records.getKey(); DataInputBuffer value = records.getValue(); IntWritable k = new IntWritable(); k.readFields(key); LongWritable v = new LongWritable(); v.readFields(value); if (records.isSameKey()) { LOG.info("\tSame Key : key=" + k.get() + ", val=" + v.get()); //More than one key should be present in the source data assertTrue(verificationDataSet.get(k.get()).size() > 1); //Ensure this is same as the previous key we saw assertTrue("previousKey=" + pk + ", current=" + k.get(), pk == k.get()); } else { LOG.info("key=" + k.get() + ", val=" + v.get()); } pk = k.get(); int keyCount = (dataMap.containsKey(k.get())) ? (dataMap.get(k.get()) + 1) : 1; dataMap.put(k.get(), keyCount); } //Verify if the number of distinct entries is the same in source and the test assertTrue( "dataMap=" + dataMap.keySet().size() + ", verificationSet=" + verificationDataSet.keySet().size(), dataMap.keySet().size() == verificationDataSet.keySet().size()); //Verify with source data for (Integer key : verificationDataSet.keySet()) { assertTrue( "Data size for " + key + " not matching with source; dataSize:" + dataMap.get(key).intValue() + ", source:" + verificationDataSet.get(key).size(), dataMap.get(key).intValue() == verificationDataSet.get(key).size()); } //Verify if every key has the same number of repeated items in the source dataset as well for (Map.Entry<Integer, Integer> entry : dataMap.entrySet()) { assertTrue(entry.getKey() + "", verificationDataSet.get(entry.getKey()).size() == entry.getValue()); } LOG.info("******************"); verificationDataSet.clear(); }
From source file:org.apache.tez.runtime.library.common.writers.TestUnorderedPartitionedKVWriter.java
License:Apache License
private void baseTest(int numRecords, int numPartitions, Set<Integer> skippedPartitions, boolean shouldCompress) throws IOException, InterruptedException { PartitionerForTest partitioner = new PartitionerForTest(); ApplicationId appId = ApplicationId.newInstance(10000, 1); TezCounters counters = new TezCounters(); String uniqueId = UUID.randomUUID().toString(); OutputContext outputContext = createMockOutputContext(counters, appId, uniqueId); Configuration conf = createConfiguration(outputContext, IntWritable.class, LongWritable.class, shouldCompress, -1);//from w w w . jav a2s. c o m CompressionCodec codec = null; if (shouldCompress) { codec = new DefaultCodec(); ((Configurable) codec).setConf(conf); } int numOutputs = numPartitions; long availableMemory = 2048; int numRecordsWritten = 0; Map<Integer, Multimap<Integer, Long>> expectedValues = new HashMap<Integer, Multimap<Integer, Long>>(); for (int i = 0; i < numOutputs; i++) { expectedValues.put(i, LinkedListMultimap.<Integer, Long>create()); } UnorderedPartitionedKVWriter kvWriter = new UnorderedPartitionedKVWriterForTest(outputContext, conf, numOutputs, availableMemory); int sizePerBuffer = kvWriter.sizePerBuffer; int sizePerRecord = 4 + 8; // IntW + LongW int sizePerRecordWithOverhead = sizePerRecord + 12; // Record + META_OVERHEAD IntWritable intWritable = new IntWritable(); LongWritable longWritable = new LongWritable(); for (int i = 0; i < numRecords; i++) { intWritable.set(i); longWritable.set(i); int partition = partitioner.getPartition(intWritable, longWritable, numOutputs); if (skippedPartitions != null && skippedPartitions.contains(partition)) { continue; } expectedValues.get(partition).put(intWritable.get(), longWritable.get()); kvWriter.write(intWritable, longWritable); numRecordsWritten++; } List<Event> events = kvWriter.close(); int recordsPerBuffer = sizePerBuffer / sizePerRecordWithOverhead; int numExpectedSpills = numRecordsWritten / recordsPerBuffer; verify(outputContext, never()).fatalError(any(Throwable.class), any(String.class)); // Verify the status of the buffers if (numExpectedSpills == 0) { assertEquals(1, kvWriter.numInitializedBuffers); } else { assertTrue(kvWriter.numInitializedBuffers > 1); } assertNull(kvWriter.currentBuffer); assertEquals(0, kvWriter.availableBuffers.size()); // Verify the counters TezCounter outputRecordBytesCounter = counters.findCounter(TaskCounter.OUTPUT_BYTES); TezCounter outputRecordsCounter = counters.findCounter(TaskCounter.OUTPUT_RECORDS); TezCounter outputBytesWithOverheadCounter = counters.findCounter(TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD); TezCounter fileOutputBytesCounter = counters.findCounter(TaskCounter.OUTPUT_BYTES_PHYSICAL); TezCounter spilledRecordsCounter = counters.findCounter(TaskCounter.SPILLED_RECORDS); TezCounter additionalSpillBytesWritternCounter = counters .findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN); TezCounter additionalSpillBytesReadCounter = counters.findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ); TezCounter numAdditionalSpillsCounter = counters.findCounter(TaskCounter.ADDITIONAL_SPILL_COUNT); assertEquals(numRecordsWritten * sizePerRecord, outputRecordBytesCounter.getValue()); assertEquals(numRecordsWritten, outputRecordsCounter.getValue()); assertEquals(numRecordsWritten * sizePerRecordWithOverhead, outputBytesWithOverheadCounter.getValue()); long fileOutputBytes = fileOutputBytesCounter.getValue(); if (numRecordsWritten > 0) { assertTrue(fileOutputBytes > 0); if (!shouldCompress) { assertTrue(fileOutputBytes > outputRecordBytesCounter.getValue()); } } else { assertEquals(0, fileOutputBytes); } assertEquals(recordsPerBuffer * numExpectedSpills, spilledRecordsCounter.getValue()); long additionalSpillBytesWritten = additionalSpillBytesWritternCounter.getValue(); long additionalSpillBytesRead = additionalSpillBytesReadCounter.getValue(); if (numExpectedSpills == 0) { assertEquals(0, additionalSpillBytesWritten); assertEquals(0, additionalSpillBytesRead); } else { assertTrue(additionalSpillBytesWritten > 0); assertTrue(additionalSpillBytesRead > 0); if (!shouldCompress) { assertTrue(additionalSpillBytesWritten > (recordsPerBuffer * numExpectedSpills * sizePerRecord)); assertTrue(additionalSpillBytesRead > (recordsPerBuffer * numExpectedSpills * sizePerRecord)); } } assertTrue(additionalSpillBytesWritten == additionalSpillBytesRead); assertEquals(numExpectedSpills, numAdditionalSpillsCounter.getValue()); BitSet emptyPartitionBits = null; // Verify the event returned assertEquals(1, events.size()); assertTrue(events.get(0) instanceof CompositeDataMovementEvent); CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) events.get(0); assertEquals(0, cdme.getSourceIndexStart()); assertEquals(numOutputs, cdme.getCount()); DataMovementEventPayloadProto eventProto = DataMovementEventPayloadProto .parseFrom(ByteString.copyFrom(cdme.getUserPayload())); assertFalse(eventProto.hasData()); if (skippedPartitions == null && numRecordsWritten > 0) { assertFalse(eventProto.hasEmptyPartitions()); emptyPartitionBits = new BitSet(numPartitions); } else { assertTrue(eventProto.hasEmptyPartitions()); byte[] emptyPartitions = TezCommonUtils .decompressByteStringToByteArray(eventProto.getEmptyPartitions()); emptyPartitionBits = TezUtilsInternal.fromByteArray(emptyPartitions); if (numRecordsWritten == 0) { assertEquals(numPartitions, emptyPartitionBits.cardinality()); } else { for (Integer e : skippedPartitions) { assertTrue(emptyPartitionBits.get(e)); } assertEquals(skippedPartitions.size(), emptyPartitionBits.cardinality()); } } if (emptyPartitionBits.cardinality() != numPartitions) { assertEquals(HOST_STRING, eventProto.getHost()); assertEquals(SHUFFLE_PORT, eventProto.getPort()); assertEquals(uniqueId, eventProto.getPathComponent()); } else { assertFalse(eventProto.hasHost()); assertFalse(eventProto.hasPort()); assertFalse(eventProto.hasPathComponent()); } // Verify the actual data TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId); Path outputFilePath = kvWriter.finalOutPath; Path spillFilePath = kvWriter.finalIndexPath; if (numRecordsWritten > 0) { assertTrue(localFs.exists(outputFilePath)); assertTrue(localFs.exists(spillFilePath)); } else { return; } // Special case for 0 records. TezSpillRecord spillRecord = new TezSpillRecord(spillFilePath, conf); DataInputBuffer keyBuffer = new DataInputBuffer(); DataInputBuffer valBuffer = new DataInputBuffer(); IntWritable keyDeser = new IntWritable(); LongWritable valDeser = new LongWritable(); for (int i = 0; i < numOutputs; i++) { if (skippedPartitions != null && skippedPartitions.contains(i)) { continue; } TezIndexRecord indexRecord = spillRecord.getIndex(i); FSDataInputStream inStream = FileSystem.getLocal(conf).open(outputFilePath); inStream.seek(indexRecord.getStartOffset()); IFile.Reader reader = new IFile.Reader(inStream, indexRecord.getPartLength(), codec, null, null, false, 0, -1); while (reader.nextRawKey(keyBuffer)) { reader.nextRawValue(valBuffer); keyDeser.readFields(keyBuffer); valDeser.readFields(valBuffer); int partition = partitioner.getPartition(keyDeser, valDeser, numOutputs); assertTrue(expectedValues.get(partition).remove(keyDeser.get(), valDeser.get())); } inStream.close(); } for (int i = 0; i < numOutputs; i++) { assertEquals(0, expectedValues.get(i).size()); expectedValues.remove(i); } assertEquals(0, expectedValues.size()); }