List of usage examples for org.apache.hadoop.io MapWritable put
@Override
public Writable put(Writable key, Writable value)
From source file:com.shmsoft.dmass.main.FileProcessor.java
License:Apache License
/** * Create a map/*from ww w . jav a2s . c o m*/ * * @param metadata Hadoop metadata to insert into map * @param fileName File currently in process * @return Created map * @throws IOException */ private MapWritable createMapWritable(Metadata metadata, String fileName) throws IOException { MapWritable mapWritable = new MapWritable(); String[] names = metadata.names(); for (String name : names) { mapWritable.put(new Text(name), new Text(metadata.get(name))); } byte[] bytes = new File(fileName).length() < ParameterProcessing.ONE_GIG ? Util.getFileContent(fileName) : "File too large".getBytes(); mapWritable.put(new Text(ParameterProcessing.NATIVE), new BytesWritable(bytes)); if (isPdf()) { String pdfFileName = fileName + ".pdf"; if (new File(pdfFileName).exists()) { byte[] pdfBytes = Util.getFileContent(pdfFileName); mapWritable.put(new Text(ParameterProcessing.NATIVE_AS_PDF), new BytesWritable(pdfBytes)); } } return mapWritable; }
From source file:com.shmsoft.dmass.main.ZipFileProcessor.java
License:Apache License
/** * Create a map//w w w . ja va 2 s . co m * * @param metadata Tika class of key/value pairs to place in map * @return MapWritable with key/value pairs added */ private MapWritable createMapWritable(Metadata metadata) { MapWritable mapWritable = new MapWritable(); String[] names = metadata.names(); for (String name : names) { String value = metadata.get(name); // TODO how could value be null? (but it did happen to me) if (value == null) { value = ""; } mapWritable.put(new Text(name), new Text(value)); } return mapWritable; }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void mapWritable() throws IOException {
// vv MapWritableTest
MapWritable src = new MapWritable();
src.put(new IntWritable(1), new Text("cat"));
src.put(new VIntWritable(2), new LongWritable(163));
MapWritable dest = new MapWritable();
WritableUtils.cloneInto(dest, src);
assertThat((Text) dest.get(new IntWritable(1)), is(new Text("cat")));
assertThat((LongWritable) dest.get(new VIntWritable(2)), is(new LongWritable(163)));
// ^^ MapWritableTest
}/*from w w w.j av a 2s .c o m*/
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void setWritableEmulation() throws IOException {
MapWritable src = new MapWritable();
src.put(new IntWritable(1), NullWritable.get());
src.put(new IntWritable(2), NullWritable.get());
MapWritable dest = new MapWritable();
WritableUtils.cloneInto(dest, src);
assertThat(dest.containsKey(new IntWritable(1)), is(true));
}/*from w w w .j a va 2 s .co m*/
From source file:de.averbis.eucases.outlinkmeta.nutch.parse.OutlinkMetaParseFilter.java
License:Open Source License
/** * Creates the metadata for an outlink according to the configured fields. Empty fields are not added. * /*from w w w . j a v a 2 s.c o m*/ * @param metadata * The metadata created by the previous parsers * @return The metadata for the outlink */ private MapWritable createOutlinkAnnotations(Metadata metadata) { MapWritable md = new MapWritable(); for (String field : this.getFields()) { NutchField nutchField = new NutchField(); for (String value : metadata.getValues(field)) { nutchField.add(value); } if (nutchField.getValues().size() > 0) { md.put(new Text(field), nutchField); } } return md; }
From source file:edu.ub.ahstfg.io.WritableConverter.java
License:Open Source License
/** * Converts a HashMap<String, LinkedList<Long>> to MapWritable. * @param input HasMap to convert.//from w w w .j av a2 s . co m * @return Converted MapWritable. */ public static MapWritable hashMapStringLinkedListShort2MapWritable(HashMap<String, LinkedList<Short>> input) { MapWritable ret = new MapWritable(); LinkedList<Short> arl; for (String s : input.keySet()) { arl = input.get(s); ret.put(new Text(s), LinkedListShort2ArrayWritable(arl)); } return ret; }
From source file:ezbake.amino.impl.dataloader.WarehausNumberLoader.java
License:Apache License
/** * Method to extract the data from the RAW info. * * @param rawData The raw data to process * @param outputMap The MapWritable for storing the bucketed values *///from w w w . j a va2 s .c o m @Override protected void extractFromRaw(final byte[] rawData, final MapWritable outputMap) { final Text data = new Text(rawData); outputMap.put(BUCKET1, data); int twoVal = Integer.parseInt(data.toString()); if (twoVal <= number2Max) { outputMap.put(BUCKET2, data); } }
From source file:full_MapReduce.AttributeInfoReducer.java
License:Open Source License
public void reduce(Text key, Iterable<AttributeCounterWritable> values, Context context) throws IOException, InterruptedException { MapWritable res = new MapWritable(); Text value;//from w w w .ja va2 s .co m Text classification; IntWritable count; for (AttributeCounterWritable cur_attribute_counter : values) { value = cur_attribute_counter.getValue(); classification = cur_attribute_counter.getClassification(); count = cur_attribute_counter.getCount(); if (!res.containsKey(value)) { res.put(new Text(value), new MapWritable()); } MapWritable cur_map = (MapWritable) res.get(value); if (!cur_map.containsKey(classification)) { cur_map.put(new Text(classification), new IntWritable(0)); } ((IntWritable) cur_map.get(classification)) .set(((IntWritable) cur_map.get(classification)).get() + count.get()); } context.write(key, res); }
From source file:gaffer.accumulostore.key.core.AbstractCoreKeyAccumuloElementConverter.java
License:Apache License
@Override public Value getValueFromProperties(final Properties properties, final String group) throws AccumuloElementConversionException { final MapWritable map = new MapWritable(); for (final Map.Entry<String, Object> entry : properties.entrySet()) { final String propertyName = entry.getKey(); final StorePropertyDefinition propertyDefinition = storeSchema.getElement(group) .getProperty(propertyName); if (propertyDefinition != null) { if (StorePositions.VALUE.isEqual(propertyDefinition.getPosition())) { try { map.put(new Text(propertyName), new BytesWritable(propertyDefinition.getSerialiser().serialise(entry.getValue()))); } catch (final SerialisationException e) { throw new AccumuloElementConversionException("Failed to serialise property " + propertyName, e);/*from ww w .j ava2s .c om*/ } } } } if (map.isEmpty()) { return new Value(); } return new Value(WritableUtils.toByteArray(map)); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
@SuppressWarnings("unchecked") Vertex<V, E, LongWritable, LongWritable> createVertex(String JSONString) { JSONArray JSONInput = (JSONArray) JSONValue.parse(JSONString); LongWritable sourceID = new LongWritable(Long.valueOf(JSONInput.get(0).toString())); assert (vertexMap.get(sourceID) == null); Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>(sourceID); //fix this/*from w w w . j ava2s . c o m*/ //assumed value of jsonMap= "key1:type1:value1$ key2:type2:value2$....." //type could be Long or String or Double String jsonMap = JSONInput.get(1).toString(); String[] vprop = jsonMap.split(Pattern.quote("$")); //key,value property pairs for a vertex MapWritable vertexMap = new MapWritable(); for (int i = 0; i < vprop.length; i++) { String[] map = vprop[i].split(Pattern.quote(":")); Text key = new Text(map[0]); //FIXME:assuming String values for now Text value = new Text(map[2]); vertexMap.put(key, value); } V vertexValue = (V) vertexMap; vertex.setValue(vertexValue); JSONArray edgeList = (JSONArray) JSONInput.get(2); for (Object edgeInfo : edgeList) { Object edgeValues[] = ((JSONArray) edgeInfo).toArray(); LongWritable sinkID = new LongWritable(Long.valueOf(edgeValues[0].toString())); LongWritable edgeID = new LongWritable(Long.valueOf(edgeValues[1].toString())); //fix this //same format as vertex String[] eprop = edgeValues[2].toString().split(Pattern.quote("$")); MapWritable edgeMap = new MapWritable(); for (int i = 0; i < eprop.length; i++) { String[] map = eprop[i].split(Pattern.quote(":")); Text key = new Text(map[0]); //FIXME:assuming String values for now Text value = new Text(map[2]); edgeMap.put(key, value); } Edge<E, LongWritable, LongWritable> edge = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); E edgeValue = (E) edgeMap; edge.setValue(edgeValue); vertex.addEdge(edge); } return vertex; }