List of usage examples for org.apache.hadoop.io SortedMapWritable SortedMapWritable
public SortedMapWritable()
From source file:com.jfolson.hive.serde.RTypedBytesWritableInput.java
License:Apache License
public SortedMapWritable readSortedMap(SortedMapWritable mw) throws IOException { if (mw == null) { mw = new SortedMapWritable(); }/*from w w w . j a v a 2 s. com*/ int length = in.readMapHeader(); for (int i = 0; i < length; i++) { WritableComparable key = (WritableComparable) read(); Writable value = read(); mw.put(key, value); } return mw; }
From source file:org.elasticsearch.hadoop.util.WritableUtils.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
public static Writable toWritable(Object object) {
if (object instanceof Writable) {
return (Writable) object;
}/*from www .j ava 2 s. c om*/
if (object == null) {
return NullWritable.get();
}
if (object instanceof String) {
return new Text((String) object);
}
if (object instanceof Long) {
return new VLongWritable((Long) object);
}
if (object instanceof Integer) {
return new VIntWritable((Integer) object);
}
if (object instanceof Byte) {
return new ByteWritable((Byte) object);
}
if (object instanceof Short) {
return WritableCompatUtil.availableShortWritable((Short) object);
}
if (object instanceof Double) {
return new DoubleWritable((Double) object);
}
if (object instanceof Float) {
return new FloatWritable((Float) object);
}
if (object instanceof Boolean) {
return new BooleanWritable((Boolean) object);
}
if (object instanceof byte[]) {
return new BytesWritable((byte[]) object);
}
if (object instanceof List) {
List<Object> list = (List<Object>) object;
if (!list.isEmpty()) {
Object first = list.get(0);
Writable[] content = new Writable[list.size()];
for (int i = 0; i < content.length; i++) {
content[i] = toWritable(list.get(i));
}
return new ArrayWritable(toWritable(first).getClass(), content);
}
return new ArrayWritable(NullWritable.class, new Writable[0]);
}
if (object instanceof SortedSet) {
SortedMapWritable smap = new SortedMapWritable();
SortedSet<Object> set = (SortedSet) object;
for (Object obj : set) {
smap.put((WritableComparable) toWritable(obj), NullWritable.get());
}
return smap;
}
if (object instanceof Set) {
MapWritable map = new MapWritable();
Set<Object> set = (Set) object;
for (Object obj : set) {
map.put(toWritable(obj), NullWritable.get());
}
return map;
}
if (object instanceof SortedMap) {
SortedMapWritable smap = new SortedMapWritable();
Map<Object, Object> map = (Map) object;
for (Map.Entry<?, ?> entry : map.entrySet()) {
smap.put((WritableComparable) toWritable(entry.getKey()), toWritable(entry.getValue()));
}
return smap;
}
if (object instanceof Map) {
MapWritable result = new MapWritable();
Map<Object, Object> map = (Map) object;
for (Map.Entry<?, ?> entry : map.entrySet()) {
result.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
}
return result;
}
// fall-back to bytearray
return new BytesWritable(object.toString().getBytes(StringUtils.UTF_8));
}
From source file:org.eobjects.hadoopdatacleaner.datastores.RowUtils.java
License:Open Source License
public static SortedMapWritable inputRowToSortedMapWritable(InputRow inputRow) { SortedMapWritable rowWritable = new SortedMapWritable(); for (InputColumn<?> inputColumn : inputRow.getInputColumns()) { String columnName = inputColumn.getName(); Object value = inputRow.getValue(inputColumn); if (value != null) rowWritable.put(new Text(columnName), new Text(value.toString())); else/*ww w . j a v a2 s. co m*/ rowWritable.put(new Text(columnName), NullWritable.get()); } return rowWritable; }
From source file:org.eobjects.hadoopdatacleaner.mapreduce.flatfile.FlatFileMapperReducerTest.java
License:Open Source License
@Test public void testMapper() throws IOException { SortedMapWritable expectedPoland = new SortedMapWritable(); expectedPoland.put(new Text("Country name"), new Text("Poland")); expectedPoland.put(new Text("ISO 3166-2"), new Text("PL")); expectedPoland.put(new Text("ISO 3166-3"), new Text("POL")); expectedPoland.put(new Text("ISO Numeric"), new Text("616")); mapDriver.withInput(new LongWritable(0), new Text( "Country name;ISO 3166-2;ISO 3166-3;ISO Numeric;Linked to country;Synonym1;Synonym2;Synonym3")) .withInput(new LongWritable(44), new Text("Poland;PL;POL;616;")); List<Pair<Text, SortedMapWritable>> actualOutputs = mapDriver.run(); Assert.assertEquals(2, actualOutputs.size()); Pair<Text, SortedMapWritable> actualOutputPoland = actualOutputs.get(0); actualOutputPoland.getSecond().containsValue("Poland"); }
From source file:org.eobjects.hadoopdatacleaner.mapreduce.flatfile.FlatFileMapperReducerTest.java
License:Open Source License
@Test public void testReducerHeader() throws IOException { List<SortedMapWritable> rows = new ArrayList<SortedMapWritable>(); SortedMapWritable header = new SortedMapWritable(); header.put(new Text("ISO 3166-2_ISO 3166-3"), new Text("ISO 3166-2_ISO 3166-3")); header.put(new Text("Country name"), new Text("Country name")); header.put(new Text("ISO 3166-2"), new Text("ISO 3166-2")); header.put(new Text("ISO 3166-3"), new Text("ISO 3166-3")); header.put(new Text("ISO Numeric"), new Text("ISO Numeric")); header.put(new Text("Linked to country"), new Text("Linked to country")); header.put(new Text("Synonym1"), new Text("Synonym1")); header.put(new Text("Synonym2"), new Text("Synonym2")); header.put(new Text("Synonym3"), new Text("Synonym3")); rows.add(header);/*from w w w . j a va2s .c o m*/ reduceDriver.withInput(new Text("Value distribution (Country name)"), rows); reduceDriver.withOutput(NullWritable.get(), new Text( "Country name;ISO 3166-2;ISO 3166-2_ISO 3166-3;ISO 3166-3;ISO Numeric;Linked to country;Synonym1;Synonym2;Synonym3")); reduceDriver.runTest(); }
From source file:org.eobjects.hadoopdatacleaner.mapreduce.flatfile.FlatFileMapperReducerTest.java
License:Open Source License
@Test public void testReducerPoland() throws IOException { List<SortedMapWritable> rows = new ArrayList<SortedMapWritable>(); SortedMapWritable poland = new SortedMapWritable(); poland.put(new Text("Country name"), new Text("Poland")); poland.put(new Text("ISO 3166-2"), new Text("PL")); poland.put(new Text("ISO 3166-3"), new Text("POL")); rows.add(poland);// w ww. j a va 2 s .c o m reduceDriver.withInput(new Text("Value distribution (Country name)"), rows); reduceDriver.withOutput(NullWritable.get(), new Text("Poland;PL;POL")); reduceDriver.runTest(); }
From source file:org.eobjects.hadoopdatacleaner.mapreduce.hbase.HBaseTableMapperTest.java
License:Open Source License
@Test public void testDenmark() throws IOException { ImmutableBytesWritable inputKey = new ImmutableBytesWritable(Bytes.toBytes("Denmark")); List<Cell> cells = new ArrayList<Cell>(); Cell cell = new KeyValue(Bytes.toBytes("Denmark"), Bytes.toBytes("mainFamily"), Bytes.toBytes("country_name"), Bytes.toBytes("Denmark")); cells.add(cell);/* ww w .j ava 2 s. co m*/ cell = new KeyValue(Bytes.toBytes("Denmark"), Bytes.toBytes("mainFamily"), Bytes.toBytes("iso2"), Bytes.toBytes("DK")); cells.add(cell); cell = new KeyValue(Bytes.toBytes("Denmark"), Bytes.toBytes("mainFamily"), Bytes.toBytes("iso3"), Bytes.toBytes("DNK")); cells.add(cell); Result inputResult = Result.create(cells); SortedMapWritable expectedOutput = new SortedMapWritable(); expectedOutput.put(new Text("mainFamily:country_name"), new Text("Denmark")); expectedOutput.put(new Text("mainFamily:iso2"), new Text("DK")); expectedOutput.put(new Text("mainFamily:iso2_iso3"), new Text("DK_DNK")); expectedOutput.put(new Text("mainFamily:iso3"), new Text("DNK")); String expectedAnalyzerKey1 = "Value distribution (mainFamily:country_name)"; String expectedAnalyzerKey2 = "Value distribution (mainFamily:iso2)"; mapDriver.withInput(inputKey, inputResult); List<Pair<Text, SortedMapWritable>> actualOutputs = mapDriver.run(); Assert.assertEquals(actualOutputs.size(), 2); Pair<Text, SortedMapWritable> actualOutput1 = actualOutputs.get(0); Pair<Text, SortedMapWritable> actualOutput2 = actualOutputs.get(1); Assert.assertEquals(expectedAnalyzerKey1, actualOutput1.getFirst().toString()); Assert.assertEquals(expectedAnalyzerKey2, actualOutput2.getFirst().toString()); for (@SuppressWarnings("rawtypes") Map.Entry<WritableComparable, Writable> mapEntry : expectedOutput.entrySet()) { Text expectedColumnName = (Text) mapEntry.getKey(); Text expectedColumnValue = (Text) mapEntry.getValue(); Assert.assertTrue(actualOutput1.getSecond().containsKey(expectedColumnName)); Assert.assertEquals(expectedColumnValue, actualOutput1.getSecond().get(expectedColumnName)); Assert.assertTrue(actualOutput2.getSecond().containsKey(expectedColumnName)); Assert.assertEquals(expectedColumnValue, actualOutput2.getSecond().get(expectedColumnName)); } }
From source file:org.eobjects.hadoopdatacleaner.mapreduce.hbase.HBaseTableReducerTest.java
License:Open Source License
@Test public void testReducer() throws IOException { List<SortedMapWritable> inputRows = new ArrayList<SortedMapWritable>(); SortedMapWritable inputRow = new SortedMapWritable(); inputRow.put(new Text("mainFamily:country_name"), new Text("Denmark")); inputRow.put(new Text("mainFamily:iso2"), new Text("DK")); inputRow.put(new Text("mainFamily:iso2_iso3"), new Text("DK_DNK")); inputRow.put(new Text("mainFamily:iso3"), new Text("DNK")); inputRows.add(inputRow);//from ww w. j av a2 s . co m inputRow = new SortedMapWritable(); inputRow.put(new Text("mainFamily:country_name"), new Text("Poland")); inputRow.put(new Text("mainFamily:iso2"), new Text("PL")); inputRow.put(new Text("mainFamily:iso2_iso3"), new Text("PL_POL")); inputRow.put(new Text("mainFamily:iso3"), new Text("POL")); inputRows.add(inputRow); String inputAnalyzerKey1 = "Value distribution (mainFamily:country_name)"; reduceDriver.withInput(new Text(inputAnalyzerKey1), inputRows); List<Pair<NullWritable, Mutation>> actualOutputs = reduceDriver.run(); Assert.assertEquals(2, actualOutputs.size()); Pair<NullWritable, Mutation> actualOutput1 = actualOutputs.get(0); Put actualPut1 = (Put) actualOutput1.getSecond(); List<Cell> keyValues = actualPut1.get(Bytes.toBytes("mainFamily"), Bytes.toBytes("country_name")); Assert.assertEquals(1, keyValues.size()); Cell cell = keyValues.get(0); System.out.println("Value: " + Bytes.toString(CellUtil.cloneValue(cell))); Assert.assertEquals("Denmark", Bytes.toString(CellUtil.cloneValue(cell))); Pair<NullWritable, Mutation> actualOutput2 = actualOutputs.get(1); Put actualPut2 = (Put) actualOutput2.getSecond(); keyValues = actualPut2.get(Bytes.toBytes("mainFamily"), Bytes.toBytes("country_name")); Assert.assertEquals(1, keyValues.size()); cell = keyValues.get(0); System.out.println("Value: " + Bytes.toString(CellUtil.cloneValue(cell))); Assert.assertEquals("Poland", Bytes.toString(CellUtil.cloneValue(cell))); }
From source file:org.huahinframework.core.io.Key.java
License:Apache License
/** * Returns the Key for sorting.//from ww w. ja v a 2 s .c o m * @return value of sort */ @SuppressWarnings("rawtypes") public SortedMapWritable sort() { SortedMapWritable smw = new SortedMapWritable(); for (Entry<WritableComparable, Writable> entry : writableMap.entrySet()) { KeyDetail kd = (KeyDetail) entry.getKey(); if (kd.getSort() != Record.SORT_NON) { kd.setKey((WritableComparable) entry.getValue()); smw.put(new IntWritable(kd.getSortPriority()), kd); } } return smw; }
From source file:org.huahinframework.core.io.Key.java
License:Apache License
/** * Returns a value that is used for sort * @return sort/*from w w w . j a v a 2 s . co m*/ */ @SuppressWarnings("rawtypes") public SortedMapWritable getSort() { SortedMapWritable smw = new SortedMapWritable(); for (Entry<WritableComparable, Writable> entry : writableMap.entrySet()) { KeyDetail kd = (KeyDetail) entry.getKey(); if (kd.getSort() != Record.SORT_NON) { smw.put(kd, entry.getValue()); } } return smw; }