List of usage examples for org.apache.hadoop.io MapWritable entrySet
@Override
public Set<Map.Entry<Writable, Writable>> entrySet()
From source file:org.elasticsearch.hadoop.util.WritableUtils.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
public static Writable toWritable(Object object) {
if (object instanceof Writable) {
return (Writable) object;
}//from w w w.ja v a2 s . com
if (object == null) {
return NullWritable.get();
}
if (object instanceof String) {
return new Text((String) object);
}
if (object instanceof Long) {
return new VLongWritable((Long) object);
}
if (object instanceof Integer) {
return new VIntWritable((Integer) object);
}
if (object instanceof Byte) {
return new ByteWritable((Byte) object);
}
if (object instanceof Short) {
return WritableCompatUtil.availableShortWritable((Short) object);
}
if (object instanceof Double) {
return new DoubleWritable((Double) object);
}
if (object instanceof Float) {
return new FloatWritable((Float) object);
}
if (object instanceof Boolean) {
return new BooleanWritable((Boolean) object);
}
if (object instanceof byte[]) {
return new BytesWritable((byte[]) object);
}
if (object instanceof List) {
List<Object> list = (List<Object>) object;
if (!list.isEmpty()) {
Object first = list.get(0);
Writable[] content = new Writable[list.size()];
for (int i = 0; i < content.length; i++) {
content[i] = toWritable(list.get(i));
}
return new ArrayWritable(toWritable(first).getClass(), content);
}
return new ArrayWritable(NullWritable.class, new Writable[0]);
}
if (object instanceof SortedSet) {
SortedMapWritable smap = new SortedMapWritable();
SortedSet<Object> set = (SortedSet) object;
for (Object obj : set) {
smap.put((WritableComparable) toWritable(obj), NullWritable.get());
}
return smap;
}
if (object instanceof Set) {
MapWritable map = new MapWritable();
Set<Object> set = (Set) object;
for (Object obj : set) {
map.put(toWritable(obj), NullWritable.get());
}
return map;
}
if (object instanceof SortedMap) {
SortedMapWritable smap = new SortedMapWritable();
Map<Object, Object> map = (Map) object;
for (Map.Entry<?, ?> entry : map.entrySet()) {
smap.put((WritableComparable) toWritable(entry.getKey()), toWritable(entry.getValue()));
}
return smap;
}
if (object instanceof Map) {
MapWritable result = new MapWritable();
Map<Object, Object> map = (Map) object;
for (Map.Entry<?, ?> entry : map.entrySet()) {
result.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
}
return result;
}
// fall-back to bytearray
return new BytesWritable(object.toString().getBytes(StringUtils.UTF_8));
}
From source file:org.huahinframework.core.io.Value.java
License:Apache License
/** * {@inheritDoc}/* ww w . j av a 2 s .c o m*/ */ @Override public String toString() { StringBuilder sb = new StringBuilder(); for (Writable w : writableMap.values()) { if (w instanceof ArrayWritable) { ArrayWritable aw = (ArrayWritable) w; for (Writable x : aw.get()) { sb.append(x.toString()).append(StringUtil.TAB); } } else if (w instanceof MapWritable) { MapWritable mw = (MapWritable) w; for (Entry<Writable, Writable> entry : mw.entrySet()) { sb.append(entry.getKey().toString().toString()).append(StringUtil.TAB) .append(entry.getValue().toString().toString()).append(StringUtil.TAB); } } else { sb.append(w.toString()).append("\t"); } } return sb.toString().substring(0, sb.toString().length() - 1); }
From source file:org.huahinframework.core.util.ObjectUtil.java
License:Apache License
/** * Convert the PrimitiveObject from Hadoop {@link Writable}. * @param object/*w ww. j a v a 2s .c om*/ * @return PrimitiveObject */ public static PrimitiveObject hadoop2Primitive(Writable object) { if (object instanceof NullWritable) { return new PrimitiveObject(NULL, null); } if (object instanceof ByteWritable) { return new PrimitiveObject(BYTE, ((ByteWritable) object).get()); } else if (object instanceof IntWritable) { return new PrimitiveObject(INTEGER, ((IntWritable) object).get()); } else if (object instanceof LongWritable) { return new PrimitiveObject(LONG, ((LongWritable) object).get()); } else if (object instanceof DoubleWritable) { return new PrimitiveObject(DOUBLE, ((DoubleWritable) object).get()); } else if (object instanceof FloatWritable) { return new PrimitiveObject(FLOAT, ((FloatWritable) object).get()); } else if (object instanceof BooleanWritable) { return new PrimitiveObject(BOOLEAN, ((BooleanWritable) object).get()); } else if (object instanceof Text) { return new PrimitiveObject(STRING, ((Text) object).toString()); } else if (object instanceof ArrayWritable) { ArrayWritable aw = (ArrayWritable) object; if (aw.get().length == 0) { return new PrimitiveObject(ARRAY, true, STRING, new ArrayList<String>()); } int type = NULL; List<Object> l = new ArrayList<Object>(); for (Writable w : aw.get()) { PrimitiveObject no = hadoop2Primitive(w); type = no.getType(); l.add(no.getObject()); } return new PrimitiveObject(ARRAY, true, type, l); } else if (object instanceof MapWritable) { MapWritable mw = (MapWritable) object; if (mw.size() == 0) { return new PrimitiveObject(ARRAY, true, STRING, STRING, new HashMap<String, String>()); } int keyType = NULL; int valueType = NULL; Map<Object, Object> m = new HashMap<Object, Object>(); for (Entry<Writable, Writable> entry : mw.entrySet()) { PrimitiveObject keyNo = hadoop2Primitive(entry.getKey()); PrimitiveObject valueNo = hadoop2Primitive(entry.getValue()); keyType = keyNo.getType(); valueType = valueNo.getType(); m.put(keyNo.getObject(), valueNo.getObject()); } return new PrimitiveObject(MAP, true, keyType, valueType, m); } throw new ClassCastException("cast object not found"); }
From source file:org.huahinframework.core.util.ObjectUtilTest.java
License:Apache License
@Test public void testPrimitive2HadoopIOMap() { Map<String, Integer> o = new HashMap<String, Integer>(); MapWritable m = new MapWritable(); o.put("0", 0); m.put(new Text("0"), new IntWritable(0)); o.put("1", 1); m.put(new Text("1"), new IntWritable(1)); HadoopObject ho = ObjectUtil.primitive2Hadoop(o); assertEquals(ObjectUtil.MAP, ho.getType()); assertEquals(MapWritable.class, ho.getObject().getClass()); MapWritable mw = (MapWritable) ho.getObject(); if (mw.size() != m.size()) { fail("map not equals size: " + mw.size() + " != " + m.size()); }//from w ww . java 2 s .co m for (Entry<Writable, Writable> entry : m.entrySet()) { if (mw.get(entry.getKey()) == null) { fail("map key not found"); } assertEquals(mw.get(entry.getKey()), entry.getValue()); } }
From source file:org.hxx.hadoop.GeneratorHbase.java
License:Apache License
private static Put createPutByDatum(byte[] url, CrawlDatum value) { MapWritable meta = value.getMetaData(); Text key = null;/*from w w w.j a va 2s . c om*/ for (Entry<Writable, Writable> e : meta.entrySet()) { if ("urlid".equals(((Text) e.getKey()).toString())) { key = (Text) e.getValue(); break; } } Put put = new Put(Bytes.toBytes(key.toString())); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("url"), url); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("Score"), Bytes.toBytes(value.getScore())); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("Status"), new byte[] { value.getStatus() }); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("Fetchtime"), Bytes.toBytes(value.getFetchTime())); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("Retries"), new byte[] { value.getRetriesSinceFetch() }); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("FetchInterval"), Bytes.toBytes(value.getFetchInterval())); put.add(Bytes.toBytes("cf1"), Bytes.toBytes("Modifiedtime"), Bytes.toBytes(value.getModifiedTime())); if (value.getSignature() != null && value.getSignature().length != 0) put.add(Bytes.toBytes("cf1"), Bytes.toBytes("Signature"), value.getSignature()); for (Entry<Writable, Writable> e : meta.entrySet()) { if (!"urlid".equals(((Text) e.getKey()).toString())) put.add(Bytes.toBytes("cf1"), Bytes.toBytes(e.getKey().toString()), Bytes.toBytes(e.getValue().toString())); } return put; }
From source file:org.schedoscope.export.redis.outputformat.RedisHashWritable.java
License:Apache License
private Map<String, String> fromMapWritable(MapWritable value) { Map<String, String> mapValue = new HashMap<String, String>(); for (Entry<Writable, Writable> e : value.entrySet()) { mapValue.put(e.getKey().toString(), e.getValue().toString()); }/*from w ww. j ava2 s . c om*/ return mapValue; }
From source file:org.vroyer.hive.solr.SolrWriter.java
License:Open Source License
@Override public void write(Writable w) throws IOException { MapWritable map = (MapWritable) w; SolrInputDocument doc = new SolrInputDocument(); for (final Map.Entry<Writable, Writable> entry : map.entrySet()) { String key = entry.getKey().toString(); if (entry.getValue() instanceof TimestampWritable) { Timestamp t = ((TimestampWritable) entry.getValue()).getTimestamp(); doc.setField(key, dateFormat.format(new Date(t.getTime()))); } else if (entry.getValue() instanceof ShortWritable) { doc.setField(key, ((ShortWritable) entry.getValue()).get()); } else {/* www. j av a2 s . c om*/ doc.setField(key, entry.getValue().toString()); } } log.debug("doc:" + doc.toString()); table.save(doc); }
From source file:selfy.JSONOutput.java
@Override public RecordWriter<Text, MapWritable> getRecordWriter(final TaskAttemptContext tac) throws IOException, InterruptedException { Path p = FileOutputFormat.getOutputPath(tac); FileSystem fileSystem = p.getFileSystem(tac.getConfiguration()); final int n = 1024; final FSDataOutputStream[] streams = new FSDataOutputStream[n]; final boolean[] first = new boolean[n]; for (int i = 0; i < n; i++) { first[i] = true;/*from w ww . j a v a 2 s . c o m*/ Path output = new Path(p, i + ".json"); streams[i] = fileSystem.create(output); streams[i].write("{\n".getBytes()); } return new RecordWriter<Text, MapWritable>() { @Override public void write(Text k, MapWritable v) throws IOException, InterruptedException { long d = k.toString().hashCode(); d = d - Integer.MIN_VALUE; int indexNumber = (int) (d % n); FSDataOutputStream stream = streams[indexNumber]; if (!first[indexNumber]) stream.write(",\n".getBytes()); else first[indexNumber] = false; stream.write(("\"" + k + "\" : {\n").getBytes()); stream.write("\t\"documentCount\" : ".getBytes()); stream.write(("" + v.size() + ",\n").getBytes()); stream.write("\t\"documents\": [\n".getBytes()); int i = 0; for (Map.Entry<Writable, Writable> a : v.entrySet()) { String id = ((Text) a.getKey()).toString(); int score = ((IntWritable) a.getValue()).get(); stream.write("\t\t{\n".getBytes()); stream.write(("\t\t\t\"document\" : \"" + id + "\",\n").getBytes()); stream.write(("\t\t\t\"score\" : " + (float) score + "\n").getBytes()); stream.write("\t\t}".getBytes()); if (i + 1 < v.size()) stream.write(",\n".getBytes()); else stream.write("\n".getBytes()); i++; } stream.write("\t]\n".getBytes()); stream.write("}".getBytes()); } @Override public void close(TaskAttemptContext tac) throws IOException, InterruptedException { for (int i = 0; i < n; i++) { streams[i].write("}\n".getBytes()); streams[i].close(); } } }; }