List of usage examples for org.apache.hadoop.io FloatWritable get
public float get()
From source file:com.facebook.presto.hive.DwrfHiveRecordCursor.java
License:Apache License
private void parseDoubleColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true;/* w w w .j a v a 2 s. c o m*/ Object object = getMaterializedValue(column); if (object == null) { nulls[column] = true; } else { nulls[column] = false; HiveType type = hiveTypes[column]; if (hiveTypes[column].equals(HIVE_FLOAT)) { FloatWritable floatWritable = checkWritable(object, FloatWritable.class); doubles[column] = floatWritable.get(); } else if (hiveTypes[column].equals(HIVE_DOUBLE)) { DoubleWritable doubleWritable = checkWritable(object, DoubleWritable.class); doubles[column] = doubleWritable.get(); } else { throw new RuntimeException(String.format("%s is not a valid DOUBLE type", type)); } } }
From source file:com.facebook.presto.hive.orc.OrcHiveRecordCursor.java
License:Apache License
private void parseDoubleColumn(int column) { // don't include column number in message because it causes boxing which is expensive here checkArgument(!isPartitionColumn[column], "Column is a partition key"); loaded[column] = true;// ww w. j ava 2s . c om Object object = getFieldValue(row, hiveColumnIndexes[column]); if (object == null) { nulls[column] = true; } else { nulls[column] = false; HiveType type = hiveTypes[column]; if (hiveTypes[column].equals(HIVE_FLOAT)) { FloatWritable floatWritable = (FloatWritable) object; doubles[column] = floatWritable.get(); } else if (hiveTypes[column].equals(HIVE_DOUBLE)) { DoubleWritable doubleWritable = (DoubleWritable) object; doubles[column] = doubleWritable.get(); } else { throw new RuntimeException(String.format("%s is not a valid DOUBLE type", type)); } } }
From source file:com.github.ygf.pagerank.PageRankTopNReducer.java
License:Apache License
@Override protected void reduce(FloatWritable inKey, Iterable<IntWritable> inValues, Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); int topResults = Integer.parseInt(conf.get("pagerank.top_results")); for (IntWritable inValue : inValues) { int page = inValue.get(); float pageRank = inKey.get(); // The elements in the queue are sorted (in non-decreasing order) by // PageRank. The queue is filled up until it contains topResults // elements. Then, a new element will be added only if its PageRank // is greater than the lowest PageRank in the queue. If the queue is // full and a new element is added, the one with the lowest PageRank // is removed from the queue. if (topN.size() < topResults || pageRank >= topN.peek().getKey()) { topN.add(new AbstractMap.SimpleEntry<Float, Integer>(pageRank, page)); if (topN.size() > topResults) { topN.poll();//ww w .jav a2 s .c o m } } } }
From source file:com.gotometrics.orderly.FloatRowKey.java
License:Apache License
@Override public Object deserialize(ImmutableBytesWritable w) throws IOException { FloatWritable fw = (FloatWritable) super.deserialize(w); if (fw == null) return fw; return Float.valueOf(fw.get()); }
From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java
License:Apache License
public void writeRawFloat(FloatWritable fw) throws IOException { out.writeRawFloat(fw.get()); }
From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java
License:Apache License
public void writeFloat(FloatWritable fw) throws IOException { out.writeFloat(fw.get()); }
From source file:com.linkedin.cubert.io.CompactWritablesDeserializer.java
License:Open Source License
@SuppressWarnings("unchecked") @Override//from www.j a v a2 s . c om public K deserialize(K object) throws IOException { if (in.available() == 0) throw new IOException(); Tuple t = (Tuple) object; if (t == null) { t = TupleFactory.getInstance().newTuple(datatypes.length); } for (int i = 0; i < datatypes.length; i++) { Object field = null; switch (datatypes[i]) { case BOOLEAN: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { ((BooleanWritable) writables[i]).set(iw.get() == 1); field = writables[i]; } break; } case BYTE: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { ((ByteWritable) writables[i]).set((byte) iw.get()); field = writables[i]; } break; } case DOUBLE: { DoubleWritable dw = VariableLengthEncoder.decodeDouble(in); if (dw != null) { ((DoubleWritable) writables[i]).set(dw.get()); field = writables[i]; } break; } case FLOAT: { FloatWritable fw = VariableLengthEncoder.decodeFloat(in); if (fw != null) { ((FloatWritable) writables[i]).set(fw.get()); field = writables[i]; } break; } case INT: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { ((IntWritable) writables[i]).set(iw.get()); field = writables[i]; } break; } case LONG: { LongWritable lw = VariableLengthEncoder.decodeLong(in); if (lw != null) { ((LongWritable) writables[i]).set(lw.get()); field = writables[i]; } break; } case STRING: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { int length = iw.get(); if (length > buffer.length) buffer = new byte[2 * buffer.length]; in.read(buffer, 0, length); ((Text) writables[i]).set(new String(buffer, 0, length)); field = writables[i]; } break; } default: throw new RuntimeException("Cannot deserialize column of type " + datatypes[i]); } t.set(i, field); } return (K) t; }
From source file:com.ML_Hadoop.MultipleLinearRegression.MultipleLinearRegressionReduce.java
@Override // necessary otherwise it runs default reduce() public void reduce(LongWritable key, Iterable<FloatWritable> values, Context context) throws IOException, InterruptedException { // decode keys, key=0 is the cost and rest are values for theta if ((int) key.get() == 0) { //key==new LongWritable(0)) is also correct Float cost = 0.0f;/* w ww . ja va2s. c o m*/ for (FloatWritable val : values) cost += val.get(); prediction_error = cost; context.write(key, new FloatWritable(cost)); } else { // extracts theta Float cost = 0.0f; for (FloatWritable val : values) cost += val.get(); // update theta System.out.println("cost for key: " + cost); System.out.println("cost " + cost * alpha / input_data_size); int key_index = (int) key.get() - 1; System.out.println("key_index: " + key_index); theta[key_index] -= cost * alpha / input_data_size; context.write(key, new FloatWritable(cost)); } }
From source file:com.moz.fiji.hive.io.FijiCellWritable.java
License:Apache License
/** * Reads and converts data according to the specified schema. * * @param in DataInput to deserialize this object from. * @param schema Schema to be used for deserializing this data. * @return the data read and converted according to the schema. * @throws IOException if there was an error reading. */// www . j a va 2s .c o m private static Object readData(DataInput in, Schema schema) throws IOException { switch (schema.getType()) { case INT: Integer intData = WritableUtils.readVInt(in); return intData; case LONG: Long longData = WritableUtils.readVLong(in); return longData; case DOUBLE: DoubleWritable doubleWritable = (DoubleWritable) WritableFactories.newInstance(DoubleWritable.class); doubleWritable.readFields(in); return doubleWritable.get(); case ENUM: case STRING: String stringData = WritableUtils.readString(in); return stringData; case FLOAT: FloatWritable floatWritable = (FloatWritable) WritableFactories.newInstance(FloatWritable.class); floatWritable.readFields(in); return floatWritable.get(); case ARRAY: List<Object> listData = Lists.newArrayList(); Integer numElements = WritableUtils.readVInt(in); for (int c = 0; c < numElements; c++) { Object listElement = readData(in, schema.getElementType()); listData.add(listElement); } return listData; case RECORD: GenericRecord recordData = new GenericData.Record(schema); Integer numFields = WritableUtils.readVInt(in); for (int c = 0; c < numFields; c++) { String fieldName = WritableUtils.readString(in); Object fieldData = readData(in, schema.getField(fieldName).schema()); recordData.put(fieldName, fieldData); } return recordData; case MAP: Map<String, Object> mapData = Maps.newHashMap(); Integer numEntries = WritableUtils.readVInt(in); for (int c = 0; c < numEntries; c++) { String key = WritableUtils.readString(in); Object value = readData(in, schema.getValueType()); mapData.put(key, value); } return mapData; case UNION: Integer tag = WritableUtils.readVInt(in); Schema unionSubSchema = schema.getTypes().get(tag); Object unionData = readData(in, unionSubSchema); return unionData; case BYTES: byte[] bytesData = WritableUtils.readCompressedByteArray(in); return bytesData; case BOOLEAN: BooleanWritable booleanWritable = (BooleanWritable) WritableFactories .newInstance(BooleanWritable.class); booleanWritable.readFields(in); return booleanWritable.get(); case NULL: return null; default: throw new UnsupportedOperationException("Unsupported type: " + schema.getType()); } }
From source file:com.uber.hoodie.hadoop.realtime.HoodieRealtimeRecordReaderTest.java
License:Apache License
@Test public void testReaderWithNestedAndComplexSchema() throws Exception { // initial commit Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getComplexEvolvedSchema()); HoodieTestUtils.initTableType(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); String commitTime = "100"; int numberOfRecords = 100; int numberOfLogRecords = numberOfRecords / 2; File partitionDir = InputFormatTestUtil.prepareParquetDataset(basePath, schema, 1, numberOfRecords, commitTime);/*w w w .ja va2s . c o m*/ InputFormatTestUtil.commit(basePath, commitTime); // Add the paths FileInputFormat.setInputPaths(jobConf, partitionDir.getPath()); // update files or generate new log file String newCommitTime = "101"; HoodieLogFormat.Writer writer = writeLogFile(partitionDir, schema, "fileid0", commitTime, newCommitTime, numberOfLogRecords); long size = writer.getCurrentSize(); writer.close(); assertTrue("block - size should be > 0", size > 0); InputFormatTestUtil.deltaCommit(basePath, newCommitTime); //create a split with baseFile (parquet file written earlier) and new log file(s) String logFilePath = writer.getLogFile().getPath().toString(); HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit( new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + commitTime + ".parquet"), 0, 1, jobConf), basePath.getRoot().getPath(), Arrays.asList(logFilePath), newCommitTime); //create a RecordReader to be used by HoodieRealtimeRecordReader RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader( new FileSplit(split.getPath(), 0, fs.getLength(split.getPath()), (String[]) null), jobConf, null); JobConf jobConf = new JobConf(); List<Schema.Field> fields = schema.getFields(); String names = fields.stream().map(f -> f.name()).collect(Collectors.joining(",")); String positions = fields.stream().map(f -> String.valueOf(f.pos())).collect(Collectors.joining(",")); jobConf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, names); jobConf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, positions); jobConf.set("partition_columns", "datestr"); // validate record reader compaction HoodieRealtimeRecordReader recordReader = new HoodieRealtimeRecordReader(split, jobConf, reader); // use reader to read base Parquet File and log file, merge in flight and return latest commit // here the first 50 records should be updated, see above NullWritable key = recordReader.createKey(); ArrayWritable value = recordReader.createValue(); int numRecordsRead = 0; while (recordReader.next(key, value)) { int currentRecordNo = numRecordsRead; ++numRecordsRead; Writable[] values = value.get(); String recordCommitTime; //check if the record written is with latest commit, here "101" if (numRecordsRead > numberOfLogRecords) { recordCommitTime = commitTime; } else { recordCommitTime = newCommitTime; } String recordCommitTimeSuffix = "@" + recordCommitTime; Assert.assertEquals(values[0].toString(), recordCommitTime); key = recordReader.createKey(); value = recordReader.createValue(); // Assert type STRING Assert.assertEquals("test value for field: field1", values[5].toString(), "field" + currentRecordNo); Assert.assertEquals("test value for field: field2", values[6].toString(), "field" + currentRecordNo + recordCommitTimeSuffix); Assert.assertEquals("test value for field: name", values[7].toString(), "name" + currentRecordNo); // Assert type INT IntWritable intWritable = (IntWritable) values[8]; Assert.assertEquals("test value for field: favoriteIntNumber", intWritable.get(), currentRecordNo + recordCommitTime.hashCode()); // Assert type LONG LongWritable longWritable = (LongWritable) values[9]; Assert.assertEquals("test value for field: favoriteNumber", longWritable.get(), currentRecordNo + recordCommitTime.hashCode()); // Assert type FLOAT FloatWritable floatWritable = (FloatWritable) values[10]; Assert.assertEquals("test value for field: favoriteFloatNumber", floatWritable.get(), (float) ((currentRecordNo + recordCommitTime.hashCode()) / 1024.0), 0); // Assert type DOUBLE DoubleWritable doubleWritable = (DoubleWritable) values[11]; Assert.assertEquals("test value for field: favoriteDoubleNumber", doubleWritable.get(), (currentRecordNo + recordCommitTime.hashCode()) / 1024.0, 0); // Assert type MAP ArrayWritable mapItem = (ArrayWritable) values[12]; Writable mapItemValue1 = mapItem.get()[0]; Writable mapItemValue2 = mapItem.get()[1]; Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get()[0].toString(), "mapItem1"); Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get()[0].toString(), "mapItem2"); Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get().length, 2); Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get().length, 2); Writable mapItemValue1value = ((ArrayWritable) mapItemValue1).get()[1]; Writable mapItemValue2value = ((ArrayWritable) mapItemValue2).get()[1]; Assert.assertEquals("test value for field: tags[\"mapItem1\"].item1", ((ArrayWritable) mapItemValue1value).get()[0].toString(), "item" + currentRecordNo); Assert.assertEquals("test value for field: tags[\"mapItem2\"].item1", ((ArrayWritable) mapItemValue2value).get()[0].toString(), "item2" + currentRecordNo); Assert.assertEquals("test value for field: tags[\"mapItem1\"].item2", ((ArrayWritable) mapItemValue1value).get()[1].toString(), "item" + currentRecordNo + recordCommitTimeSuffix); Assert.assertEquals("test value for field: tags[\"mapItem2\"].item2", ((ArrayWritable) mapItemValue2value).get()[1].toString(), "item2" + currentRecordNo + recordCommitTimeSuffix); // Assert type RECORD ArrayWritable recordItem = (ArrayWritable) values[13]; Writable[] nestedRecord = recordItem.get(); Assert.assertEquals("test value for field: testNestedRecord.isAdmin", ((BooleanWritable) nestedRecord[0]).get(), false); Assert.assertEquals("test value for field: testNestedRecord.userId", nestedRecord[1].toString(), "UserId" + currentRecordNo + recordCommitTimeSuffix); // Assert type ARRAY ArrayWritable arrayValue = (ArrayWritable) values[14]; Writable[] arrayValues = arrayValue.get(); for (int i = 0; i < arrayValues.length; i++) { Assert.assertEquals("test value for field: stringArray", "stringArray" + i + recordCommitTimeSuffix, arrayValues[i].toString()); } } }