Example usage for org.apache.hadoop.io MapWritable put

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable put.

Prototype

@Override
    public Writable put(Writable key, Writable value) 

Source Link

Usage

From source file:org.apache.taverna.platform.execution.impl.hadoop.TavernaRecordReader.java

License:Apache License

@Override
public MapWritable getCurrentValue() throws IOException, InterruptedException {
    MapWritable mapWritable = new MapWritable();
    mapWritable.put(new Text("tag"), new Text(datalinks.get(recordName)));
    mapWritable.put(new Text("record"), new Text(files[index].getPath().toString()));
    return mapWritable;
}

---

From source file:org.elasticsearch.hadoop.serialization.FieldExtractorTests.java

License:Apache License

@Test
public void testMapFieldExtractorNested() throws Exception {
    ConstantFieldExtractor cfe = new MapWritableFieldExtractor();
    Map<Writable, Writable> m = new MapWritable();
    MapWritable nested = new MapWritable();
    nested.put(new Text("bar"), new Text("found"));
    m.put(new Text("foo"), nested);
    assertEquals(new Text("found"), extract(cfe, "foo.bar", m));
}

---

From source file:org.elasticsearch.hadoop.serialization.handler.write.impl.SerializationEventConverterTest.java

License:Apache License

@Test
public void generateEventWritable() throws Exception {
    MapWritable document = new MapWritable();
    document.put(new Text("field"), new Text("value"));

    SerializationEventConverter eventConverter = new SerializationEventConverter();

    SerializationFailure iaeFailure = new SerializationFailure(new IllegalArgumentException("garbage"),
            document, new ArrayList<String>());

    String rawEvent = eventConverter.getRawEvent(iaeFailure);
    assertThat(rawEvent, Matchers.startsWith("org.apache.hadoop.io.MapWritable@"));
    String timestamp = eventConverter.getTimestamp(iaeFailure);
    assertTrue(StringUtils.hasText(timestamp));
    assertTrue(DateUtils.parseDate(timestamp).getTime().getTime() > 1L);
    String exceptionType = eventConverter.renderExceptionType(iaeFailure);
    assertEquals("illegal_argument_exception", exceptionType);
    String exceptionMessage = eventConverter.renderExceptionMessage(iaeFailure);
    assertEquals("garbage", exceptionMessage);
    String eventMessage = eventConverter.renderEventMessage(iaeFailure);
    assertEquals("Could not construct bulk entry from record", eventMessage);
}

---

From source file:org.elasticsearch.hadoop.util.WritableUtils.java

License:Apache License

@SuppressWarnings({ "unchecked", "rawtypes" })
public static Writable toWritable(Object object) {
    if (object instanceof Writable) {
        return (Writable) object;
    }/*  w  ww  .ja v a  2  s. c o m*/
    if (object == null) {
        return NullWritable.get();
    }
    if (object instanceof String) {
        return new Text((String) object);
    }
    if (object instanceof Long) {
        return new VLongWritable((Long) object);
    }
    if (object instanceof Integer) {
        return new VIntWritable((Integer) object);
    }
    if (object instanceof Byte) {
        return new ByteWritable((Byte) object);
    }
    if (object instanceof Short) {
        return WritableCompatUtil.availableShortWritable((Short) object);
    }
    if (object instanceof Double) {
        return new DoubleWritable((Double) object);
    }
    if (object instanceof Float) {
        return new FloatWritable((Float) object);
    }
    if (object instanceof Boolean) {
        return new BooleanWritable((Boolean) object);
    }
    if (object instanceof byte[]) {
        return new BytesWritable((byte[]) object);
    }
    if (object instanceof List) {
        List<Object> list = (List<Object>) object;
        if (!list.isEmpty()) {
            Object first = list.get(0);
            Writable[] content = new Writable[list.size()];
            for (int i = 0; i < content.length; i++) {
                content[i] = toWritable(list.get(i));
            }
            return new ArrayWritable(toWritable(first).getClass(), content);
        }
        return new ArrayWritable(NullWritable.class, new Writable[0]);
    }
    if (object instanceof SortedSet) {
        SortedMapWritable smap = new SortedMapWritable();
        SortedSet<Object> set = (SortedSet) object;
        for (Object obj : set) {
            smap.put((WritableComparable) toWritable(obj), NullWritable.get());
        }
        return smap;
    }
    if (object instanceof Set) {
        MapWritable map = new MapWritable();
        Set<Object> set = (Set) object;
        for (Object obj : set) {
            map.put(toWritable(obj), NullWritable.get());
        }
        return map;
    }
    if (object instanceof SortedMap) {
        SortedMapWritable smap = new SortedMapWritable();
        Map<Object, Object> map = (Map) object;
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            smap.put((WritableComparable) toWritable(entry.getKey()), toWritable(entry.getValue()));
        }
        return smap;
    }
    if (object instanceof Map) {
        MapWritable result = new MapWritable();
        Map<Object, Object> map = (Map) object;
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            result.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
        }
        return result;
    }
    // fall-back to bytearray
    return new BytesWritable(object.toString().getBytes(StringUtils.UTF_8));
}

---

From source file:org.freeeed.main.FileProcessor.java

License:Apache License

/**
 * Create a map//from w ww  .  j a va 2s. c o m
 *
 * @param metadata Hadoop metadata to insert into map
 * @return Created map
 * @throws IOException
 */
private MapWritable createMapWritable(Metadata metadata, DiscoveryFile discoveryFile) throws IOException {
    String fileName = discoveryFile.getPath().getPath();
    MapWritable mapWritable = new MapWritable();
    String[] names = metadata.names();
    for (String name : names) {
        mapWritable.put(new Text(name), new Text(metadata.get(name)));
    }
    byte[] bytes = Util.getFileContent(fileName);
    mapWritable.put(new Text(ParameterProcessing.NATIVE), new BytesWritable(bytes));

    if (isPdf()) {
        String pdfFileName = fileName + ".pdf";
        if (new File(pdfFileName).exists()) {
            byte[] pdfBytes = Util.getFileContent(pdfFileName);
            mapWritable.put(new Text(ParameterProcessing.NATIVE_AS_PDF), new BytesWritable(pdfBytes));
        }
    }

    createMapWritableForHtml(mapWritable);

    return mapWritable;
}

---

From source file:org.freeeed.main.FileProcessor.java

License:Apache License

/**
 * Collect the html code//www .j a v a 2 s .  c  o  m
 *
 * @param mapWritable
 * @throws IOException
 */
// TODO lots of room to improve html generation
private void createMapWritableForHtml(MapWritable mapWritable) throws IOException {
    File htmlOutputDir = new File(getHtmlOutputDir());
    List<String> htmlFiles = new ArrayList<>();
    //get all generated files
    String[] files = htmlOutputDir.list();
    if (files != null) {
        for (String file : files) {
            String htmlFileName = htmlOutputDir.getPath() + File.separator + file;
            File htmlFile = new File(htmlFileName);
            if (htmlFile.exists()) {
                if ("html".equalsIgnoreCase(Util.getExtension(htmlFile.getName()))) {
                    byte[] htmlBytes = Util.getFileContent(htmlFileName);
                    mapWritable.put(new Text(ParameterProcessing.NATIVE_AS_HTML_NAME),
                            new BytesWritable(htmlBytes));
                } else {
                    byte[] htmlBytes = Util.getFileContent(htmlFileName);
                    String key = ParameterProcessing.NATIVE_AS_HTML + "_" + file;
                    mapWritable.put(new Text(key), new BytesWritable(htmlBytes));
                    htmlFiles.add(file);
                }
            }
        }
    }

    if (htmlFiles.size() > 0) {
        StringBuilder sb = new StringBuilder();
        for (String file : htmlFiles) {
            sb.append(file).append(",");
        }

        mapWritable.put(new Text(ParameterProcessing.NATIVE_AS_HTML), new Text(sb.toString()));
    }
}

---

From source file:org.huahinframework.core.io.RecordTest.java

License:Apache License

@Test
public void testValueTextMapWritable() {
    Record record = new Record();
    MapWritable o = new MapWritable();
    o.put(new Text("String1"), new Text("String1"));
    o.put(new Text("String2"), new Text("String2"));
    o.put(new Text("String3"), new Text("String3"));
    record.addValue("Object", o);
    assertEquals(record.getValueMapWritable("Object"), o);
    assertEquals(record.getValueMapWritable("Object").size(), 3);

    assertEquals(record.getValueMapWritable("Object").get(new Text("String1")), new Text("String1"));
    assertEquals(record.getValueMapWritable("Object").get(new Text("String2")), new Text("String2"));
    assertEquals(record.getValueMapWritable("Object").get(new Text("String3")), new Text("String3"));

    assertEquals(record.getValueList("Object2"), null);

    try {/*from  w  ww.jav a 2 s  .com*/
        record.getValueInteger("Object");
        fail("fail ClassCastException");
    } catch (Exception e) {
        assertTrue(e instanceof ClassCastException);
    }
}

---

From source file:org.huahinframework.core.io.RecordTest.java

License:Apache License

@Test
public void testValueTextIntWritableMapWritable() {
    Record record = new Record();
    MapWritable o = new MapWritable();
    o.put(new Text("String1"), new IntWritable(1));
    o.put(new Text("String2"), new IntWritable(2));
    o.put(new Text("String3"), new IntWritable(3));
    record.addValue("Object", o);
    assertEquals(record.getValueMapWritable("Object"), o);
    assertEquals(record.getValueMapWritable("Object").size(), 3);

    assertEquals(record.getValueMapWritable("Object").get(new Text("String1")), new IntWritable(1));
    assertEquals(record.getValueMapWritable("Object").get(new Text("String2")), new IntWritable(2));
    assertEquals(record.getValueMapWritable("Object").get(new Text("String3")), new IntWritable(3));

    assertEquals(record.getValueList("Object2"), null);

    try {//from www  .ja  va2  s.  c  o  m
        record.getValueInteger("Object");
        fail("fail ClassCastException");
    } catch (Exception e) {
        assertTrue(e instanceof ClassCastException);
    }
}

---

From source file:org.huahinframework.core.util.ObjectUtil.java

License:Apache License

/**
 * Convert the HadoopObject from Java primitive.
 * @param object Java primitive object//from  w  w w .  ja  v a2 s  .c  om
 * @return HadoopObject
 */
public static HadoopObject primitive2Hadoop(Object object) {
    if (object == null) {
        return new HadoopObject(NULL, NullWritable.get());
    }

    if (object instanceof Byte) {
        return new HadoopObject(BYTE, new ByteWritable((Byte) object));
    } else if (object instanceof Integer) {
        return new HadoopObject(INTEGER, new IntWritable((Integer) object));
    } else if (object instanceof Long) {
        return new HadoopObject(LONG, new LongWritable((Long) object));
    } else if (object instanceof Double) {
        return new HadoopObject(DOUBLE, new DoubleWritable((Double) object));
    } else if (object instanceof Float) {
        return new HadoopObject(FLOAT, new FloatWritable((Float) object));
    } else if (object instanceof Boolean) {
        return new HadoopObject(BOOLEAN, new BooleanWritable((Boolean) object));
    } else if (object instanceof String) {
        return new HadoopObject(STRING, new Text((String) object));
    } else if (object.getClass().isArray()) {
        return arrayPrimitive2Hadoop(object);
    } else if (object instanceof Collection<?>) {
        Collection<?> collection = (Collection<?>) object;
        return arrayPrimitive2Hadoop(collection.toArray());
    } else if (object instanceof Map<?, ?>) {
        Map<?, ?> map = (Map<?, ?>) object;
        if (map.size() == 0) {
            throw new ClassCastException("object not found");
        }

        MapWritable mapWritable = new MapWritable();
        for (Entry<?, ?> entry : map.entrySet()) {
            mapWritable.put(primitive2Hadoop(entry.getKey()).getObject(),
                    primitive2Hadoop(entry.getValue()).getObject());
        }

        return new HadoopObject(MAP, mapWritable);
    }

    throw new ClassCastException("cast object not found");
}

---

From source file:org.huahinframework.core.util.ObjectUtilTest.java

License:Apache License

@Test
public void testPrimitive2HadoopIOMap() {
    Map<String, Integer> o = new HashMap<String, Integer>();
    MapWritable m = new MapWritable();

    o.put("0", 0);
    m.put(new Text("0"), new IntWritable(0));

    o.put("1", 1);
    m.put(new Text("1"), new IntWritable(1));

    HadoopObject ho = ObjectUtil.primitive2Hadoop(o);
    assertEquals(ObjectUtil.MAP, ho.getType());
    assertEquals(MapWritable.class, ho.getObject().getClass());

    MapWritable mw = (MapWritable) ho.getObject();
    if (mw.size() != m.size()) {
        fail("map not equals size: " + mw.size() + " != " + m.size());
    }/* ww w .  j av  a 2  s .  c om*/

    for (Entry<Writable, Writable> entry : m.entrySet()) {
        if (mw.get(entry.getKey()) == null) {
            fail("map key not found");
        }

        assertEquals(mw.get(entry.getKey()), entry.getValue());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »