List of usage examples for org.apache.hadoop.io ArrayWritable set
public void set(Writable[] values)
From source file:com.csiro.hadoop.WritableTest.java
public static void main(String[] args) { System.out.println("*** Primitive Writable ***"); BooleanWritable bool1 = new BooleanWritable(true); ByteWritable byte1 = new ByteWritable((byte) 3); System.out.printf("Boolean:%s Byte:%d\n", bool1, byte1.get()); IntWritable int1 = new IntWritable(5); IntWritable int2 = new IntWritable(17); System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get()); int1.set(int2.get()); System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get()); Integer int3 = new Integer(23); int1.set(int3); System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get()); System.out.println("*** Array Writable ***"); ArrayWritable a = new ArrayWritable(IntWritable.class); a.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) }); IntWritable[] values = (IntWritable[]) a.get(); for (IntWritable i : values) { System.out.println(i);/*from www . j av a 2 s .c o m*/ } IntArrayWritable ia = new IntArrayWritable(); ia.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) }); IntWritable[] ivalues = (IntWritable[]) ia.get(); ia.set((new LongWritable[] { new LongWritable(10001) })); System.out.println("*** Map Writables ***"); MapWritable m = new MapWritable(); IntWritable key1 = new IntWritable(5); NullWritable value1 = NullWritable.get(); m.put(key1, value1); System.out.println(m.containsKey(key1)); System.out.println(m.get(key1)); m.put(new LongWritable(100000000), key1); Set<Writable> keys = m.keySet(); for (Writable k : keys) System.out.println(k.getClass()); }
From source file:com.ikanow.aleph2.search_service.elasticsearch.utils.TestJsonNodeWritableUtils.java
License:Apache License
@SuppressWarnings("deprecation") @Test/*from ww w. j a v a 2s.co m*/ public void test_mapWritableWrapper() { final ObjectMapper mapper = BeanTemplateUtils.configureMapper(Optional.empty()); final MapWritable m1 = new MapWritable(); m1.put(new Text("test1"), new BooleanWritable(true)); final MapWritable m2 = new MapWritable(); m2.put(new Text("nested"), m1); m2.put(new Text("test2"), new Text("test2")); final ArrayWritable a1 = new ArrayWritable(IntWritable.class); a1.set(new Writable[] { new IntWritable(4), new IntWritable(5) }); final ArrayWritable a2 = new ArrayWritable(MapWritable.class); a2.set(new Writable[] { m1, m1 }); m2.put(new Text("array"), a2); m1.put(new Text("array"), a1); final JsonNode j2 = JsonNodeWritableUtils.from(m2); assertEquals(3, j2.size()); // Check j's contents assertEquals(Stream.of("nested", "test2", "array").sorted().collect(Collectors.toList()), Optionals.streamOf(j2.fieldNames(), false).sorted().collect(Collectors.toList())); assertEquals("test2", j2.get("test2").asText()); final JsonNode j1 = j2.get("nested"); assertEquals(2, j1.size()); final JsonNode j1b = JsonNodeWritableUtils.from(m1); assertTrue("{\"test1\":true,\"array\":[4,5]}".equals(j1b.toString()) || "{\"array\":[4,5],\"test1\":true}".equals(j1b.toString())); //(tests entrySet) final ArrayNode an = mapper.createArrayNode(); an.add(mapper.convertValue(4, JsonNode.class)); an.add(mapper.convertValue(5, JsonNode.class)); assertEquals(Arrays.asList(mapper.convertValue(true, JsonNode.class), an), Optionals.streamOf(((ObjectNode) j1).elements(), false).collect(Collectors.toList())); // OK, now test adding: assertEquals(2, j1.size()); final ObjectNode o1 = (ObjectNode) j1; o1.put("added", "added_this"); final ObjectNodeWrapper o1c = (ObjectNodeWrapper) o1; assertFalse(o1c.containsKey("not_present")); assertTrue(o1c.containsKey("added")); assertTrue(o1c.containsKey("test1")); assertEquals(Stream.of("test1", "array", "added").sorted().collect(Collectors.toList()), Optionals.streamOf(j1.fieldNames(), false).sorted().collect(Collectors.toList())); assertEquals( Arrays.asList(mapper.convertValue(true, JsonNode.class), an, mapper.convertValue("added_this", JsonNode.class)), Optionals.streamOf(((ObjectNode) j1).elements(), false).collect(Collectors.toList())); assertTrue(j1.toString().contains("added_this")); assertTrue(j1.toString().contains("4,5")); assertEquals(mapper.convertValue("added_this", JsonNode.class), j1.get("added")); assertEquals(3, j1.size()); // OK now test removing: assertEquals(null, o1.remove("not_present")); assertEquals(mapper.convertValue(true, JsonNode.class), o1.remove("test1")); assertEquals(2, o1.size()); ObjectNode o1b = o1.remove(Arrays.asList("added", "array")); assertEquals(0, o1.size()); assertEquals(0, o1b.size()); o1.putAll(JsonNodeWritableUtils.from(m1)); // will be minus one object assertEquals(2, o1.size()); assertTrue(o1c.containsValue(mapper.convertValue(true, JsonNode.class))); assertFalse(o1c.containsValue("banana")); final ObjectNodeWrapper o2 = (ObjectNodeWrapper) JsonNodeWritableUtils.from(m2); assertFalse(o2.isEmpty()); assertTrue(o2.containsKey("array")); assertFalse(o2.containsValue("array")); assertTrue(o2.containsValue(mapper.convertValue("test2", JsonNode.class))); assertEquals(TextNode.class, o2.remove("test2").getClass()); assertEquals(2, o2.size()); o2.removeAll(); assertEquals(0, o2.size()); }
From source file:com.jfolson.hive.serde.RTypedBytesWritableInput.java
License:Apache License
public ArrayWritable readVector(ArrayWritable aw) throws IOException { if (aw == null) { aw = new ArrayWritable(Writable.class); } /* else if (!aw.getValueClass().equals(clazz)) { throw new RuntimeException("value class has to be "+clazz.getCanonicalName()); }*///from www . j a va 2 s. c om int length = in.readVectorHeader(); Writable[] writables = new Writable[length]; for (int i = 0; i < length; i++) { Writable w = null; if (aw.getValueClass() == TypedBytesWritable.class) { w = readRaw(); } else { w = this.read(); if (w instanceof NullWritable) { w = null; } } writables[i] = w; } aw.set(writables); return aw; }
From source file:com.ricemap.spateDB.mapred.SpatialRecordReader.java
License:Apache License
/** * Reads all shapes left in the current block in one shot. * @param shapes/*from w ww. j ava 2 s .co m*/ * @return * @throws IOException */ protected boolean nextShapes(ArrayWritable shapes) throws IOException { // Prepare a vector that will hold all objects in this Vector<Shape> vshapes = new Vector<Shape>(); try { Shape stockObject = (Shape) shapes.getValueClass().newInstance(); // Reached the end of this split if (getPos() >= end) return false; long initialReadPos = getPos(); long readBytes = 0; // Read all shapes in this block while (vshapes.size() < maxShapesInOneRead && readBytes < maxBytesInOneRead && nextShape(stockObject)) { vshapes.add(stockObject.clone()); readBytes = getPos() - initialReadPos; } // Store them in the return value shapes.set(vshapes.toArray(new Shape[vshapes.size()])); return vshapes.size() > 0; } catch (InstantiationException e1) { e1.printStackTrace(); } catch (IllegalAccessException e1) { e1.printStackTrace(); } catch (OutOfMemoryError e) { LOG.error("Error reading shapes. Stopped with " + vshapes.size() + " shapes"); throw e; } return false; }
From source file:com.uber.hoodie.hadoop.realtime.RealtimeCompactedRecordReader.java
License:Apache License
@Override public boolean next(NullWritable aVoid, ArrayWritable arrayWritable) throws IOException { // Call the underlying parquetReader.next - which may replace the passed in ArrayWritable // with a new block of values boolean result = this.parquetReader.next(aVoid, arrayWritable); if (!result) { // if the result is false, then there are no more records return false; } else {/* ww w.j ava2s . c o m*/ // TODO(VC): Right now, we assume all records in log, have a matching base record. (which // would be true until we have a way to index logs too) // return from delta records map if we have some match. String key = arrayWritable.get()[HoodieRealtimeInputFormat.HOODIE_RECORD_KEY_COL_POS].toString(); if (deltaRecordMap.containsKey(key)) { // TODO(NA): Invoke preCombine here by converting arrayWritable to Avro. This is required since the // deltaRecord may not be a full record and needs values of columns from the parquet Optional<GenericRecord> rec; if (usesCustomPayload) { rec = deltaRecordMap.get(key).getData().getInsertValue(getWriterSchema()); } else { rec = deltaRecordMap.get(key).getData().getInsertValue(getReaderSchema()); } if (!rec.isPresent()) { // If the record is not present, this is a delete record using an empty payload so skip this base record // and move to the next record return next(aVoid, arrayWritable); } GenericRecord recordToReturn = rec.get(); if (usesCustomPayload) { // If using a custom payload, return only the projection fields recordToReturn = HoodieAvroUtils.rewriteRecordWithOnlyNewSchemaFields(rec.get(), getReaderSchema()); } // we assume, a later safe record in the log, is newer than what we have in the map & // replace it. ArrayWritable aWritable = (ArrayWritable) avroToArrayWritable(recordToReturn, getWriterSchema()); Writable[] replaceValue = aWritable.get(); if (LOG.isDebugEnabled()) { LOG.debug(String.format("key %s, base values: %s, log values: %s", key, arrayWritableToString(arrayWritable), arrayWritableToString(aWritable))); } Writable[] originalValue = arrayWritable.get(); try { System.arraycopy(replaceValue, 0, originalValue, 0, originalValue.length); arrayWritable.set(originalValue); } catch (RuntimeException re) { LOG.error("Got exception when doing array copy", re); LOG.error("Base record :" + arrayWritableToString(arrayWritable)); LOG.error("Log record :" + arrayWritableToString(aWritable)); throw re; } } return true; } }
From source file:com.uber.hoodie.hadoop.realtime.RealtimeUnmergedRecordReader.java
License:Apache License
@Override public boolean next(NullWritable key, ArrayWritable value) throws IOException { if (!iterator.hasNext()) { return false; }/*ww w. ja va 2 s. com*/ // Copy from buffer iterator and set to passed writable value.set(iterator.next().get()); return true; }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void test() throws IOException {
// vv ArrayWritableTest
ArrayWritable writable = new ArrayWritable(Text.class);
// ^^ ArrayWritableTest
writable.set(new Text[] { new Text("cat"), new Text("dog") });
TextArrayWritable dest = new TextArrayWritable();
WritableUtils.cloneInto(dest, writable);
assertThat(dest.get().length, is(2));
// TODO: fix cast, also use single assert
assertThat((Text) dest.get()[0], is(new Text("cat")));
assertThat((Text) dest.get()[1], is(new Text("dog")));
Text[] copy = (Text[]) dest.toArray();
assertThat(copy[0], is(new Text("cat")));
assertThat(copy[1], is(new Text("dog")));
}//from w w w.ja v a 2 s . c o m
From source file:edu.indiana.d2i.htrc.corpus.CorpusProcessingUtils.java
License:Apache License
/** * Cleans raw volume content//from w ww. jav a 2 s. c o m * * @param volume * volume content represented as a list of pages */ public static void cleanVolume(ArrayWritable volume) { Text[] pages = (Text[]) volume.get(); for (int i = 0; i < pages.length; i++) { // TODO: apply clean logic to each page String pageContent = pages[i].toString(); /** * Logic to clean the pageContent */ // Logic goes here /** * set the cleaned content back, pageContent is page content after * cleaning */ pages[i].set(pageContent); } // set cleaned volume content back volume.set(pages); }
From source file:edu.indiana.d2i.htrc.corpus.CorpusProcessingUtils.java
License:Apache License
/** * Transform cleaned volume of form raw text to indices of words in full * word set/*from ww w . j av a 2 s . co m*/ * * @param volume * volume content represented as a list of pages * @param wordSet * set of interested words (full word set) */ public static void transformVolume(ArrayWritable volume, List<String> wordSet) { Text[] pages = (Text[]) volume.get(); for (int i = 0; i < pages.length; i++) { /** * Need change the RegExp when we are also interested in general * symbols other than words */ String[] tokens = pages[i].toString().toLowerCase().split("\\W+"); StringBuilder transformedPage = new StringBuilder(); int idx = -1; for (String token : tokens) { idx = wordSet.indexOf(token); /** * omit tokens that don't appear in word set */ if (idx != -1) { transformedPage.append(idx + " "); } } // remove ' ' at tail pages[i].set(transformedPage.toString().trim()); } // set volume volume.set(pages); }
From source file:edu.indiana.d2i.htrc.corpus.CorpusProcessingUtils.java
License:Apache License
/** * convert volume content to indices in sub word set * /*from ww w.j a v a 2s . co m*/ * @param volume * volume content represented as indices to full word set * @param mappingIndices */ public static void fullWordSet2SubWordSet(ArrayWritable volume, List<Integer> mappingIndices) { Text[] pages = (Text[]) volume.get(); for (int i = 0; i < pages.length; i++) { String[] indices = pages[i].toString().split(" "); StringBuilder pg = new StringBuilder(); for (int j = 0; j < indices.length; j++) { int idx = mappingIndices.indexOf(indices[j]); /** * omit words not in the sub set */ if (idx != -1) { pg.append(idx + " "); } } // remove ' ' at tail pages[i].set(pg.toString().trim()); } // set volume volume.set(pages); }