List of usage examples for org.apache.hadoop.io ObjectWritable get
public Object get()
From source file:co.cask.cdap.hive.datasets.DatasetSerDe.java
License:Apache License
@Override public Object deserialize(Writable writable) throws SerDeException { ObjectWritable objectWritable = (ObjectWritable) writable; Object obj = objectWritable.get(); try {/*from w w w . j av a2s . c o m*/ return deserializer.deserialize(obj); } catch (Throwable t) { LOG.error("Unable to deserialize object {}.", obj, t); throw new SerDeException("Unable to deserialize an object.", t); } }
From source file:co.cask.cdap.hive.stream.StreamSerDe.java
License:Apache License
@Override public Object deserialize(Writable writable) throws SerDeException { // The writable should always contains a StreamEvent object provided by the StreamRecordReader ObjectWritable objectWritable = (ObjectWritable) writable; StreamEvent streamEvent = (StreamEvent) objectWritable.get(); // timestamp and headers are always guaranteed to be first. List<Object> event = Lists.newArrayList(); event.add(streamEvent.getTimestamp()); event.add(streamEvent.getHeaders()); try {/* w ww . ja v a2 s . c o m*/ // The format should always format the stream event into a record. event.addAll(deserializer.translateRecord(streamFormat.read(streamEvent))); return event; } catch (Throwable t) { LOG.info("Unable to format the stream body.", t); throw new SerDeException("Unable to format the stream body.", t); } }
From source file:com.bah.culvert.constraints.Constraint.java
License:Apache License
/** * Read the constraint from the stream//ww w .j a v a2 s.c o m * @param in to read the constraint from * @return specified {@link Constraint} * @throws IOException if the constraint could not be created or read */ public static Constraint readFromStream(DataInput in) throws IOException { ObjectWritable ow = new ObjectWritable(); Configuration conf = new Configuration(); ow.setConf(conf); ow.readFields(in); return (Constraint) ow.get(); }
From source file:com.bah.culvert.constraints.IndexRangeConstraint.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { range.readFields(in);//from w ww . ja v a2s . c om ObjectWritable ow = new ObjectWritable(); Configuration conf = new Configuration(); ow.setConf(conf); ow.readFields(in); this.index = (Index) ow.get(); }
From source file:com.bah.culvert.constraints.join.IndexedJoin.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); ObjectWritable ow = new ObjectWritable(); Configuration conf = new Configuration(); ow.setConf(conf);//from ww w .ja va 2 s . c o m ow.readFields(in); this.rightIndex = (Index) ow.get(); }
From source file:com.bah.culvert.constraints.Join.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { ObjectWritable ow = new ObjectWritable(); Configuration conf = new Configuration(); ow.setConf(conf);//from ww w. j a v a 2 s. co m // read in left table ow.readFields(in); this.leftTable = (TableAdapter) ow.get(); // read in left constraint ow.readFields(in); this.left = (Constraint) ow.get(); // read in left column this.leftColumn = new CColumn(); this.leftColumn.readFields(in); // read in right table this.rightTable = Text.readString(in); ow.readFields(in); this.database = (DatabaseAdapter) ow.get(); }
From source file:com.bah.culvert.constraints.RetrieveColumns.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { this.columns = new CColumn[in.readInt()]; for (int i = 0; i < this.columns.length; i++) { CColumn column = new CColumn(); column.readFields(in);//from w ww.j av a 2 s .c o m this.columns[i] = column; } this.subConstraint = Constraint.readFromStream(in); ObjectWritable ow = new ObjectWritable(); Configuration conf = new Configuration(); ow.setConf(conf); ow.readFields(in); this.table = (TableAdapter) ow.get(); }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void test() throws IOException {
ObjectWritable src = new ObjectWritable(Integer.TYPE, 163);
ObjectWritable dest = new ObjectWritable();
WritableUtils.cloneInto(dest, src);
assertThat((Integer) dest.get(), is(163));
}/* w w w. ja va 2 s. com*/
From source file:org.apache.hive.streaming.NiFiRecordSerDe.java
License:Apache License
@Override public Object deserialize(Writable writable) throws SerDeException { ObjectWritable t = (ObjectWritable) writable; Record record = (Record) t.get(); List<Object> r = new ArrayList<>(Collections.nCopies(columnNames.size(), null)); try {//from w w w. j a va2 s. co m RecordSchema recordSchema = record.getSchema(); for (RecordField field : recordSchema.getFields()) { String fieldName = field.getFieldName(); String normalizedFieldName = fieldName.toLowerCase(); // Get column position of field name, and set field value there Integer fpos = fieldPositionMap.get(normalizedFieldName); if (fpos == null || fpos == -1) { // This is either a partition column or not a column in the target table, ignore either way continue; } Object currField = extractCurrentField(record, field, schema.getStructFieldTypeInfo(normalizedFieldName)); r.set(fpos, currField); } stats.setRowCount(stats.getRowCount() + 1); } catch (Exception e) { log.warn("Error [{}] parsing Record [{}].", new Object[] { e.toString(), t }, e); throw new SerDeException(e); } return r; }
From source file:org.apache.nutch.scoring.webgraph.ScoreUpdater.java
License:Apache License
/** * Creates new CrawlDatum objects with the updated score from the NodeDb or * with a cleared score.//from w ww . j a va 2s . c om */ public void reduce(Text key, Iterator<ObjectWritable> values, OutputCollector<Text, CrawlDatum> output, Reporter reporter) throws IOException { String url = key.toString(); Node node = null; CrawlDatum datum = null; // set the node and the crawl datum, should be one of each unless no node // for url in the crawldb while (values.hasNext()) { ObjectWritable next = values.next(); Object value = next.get(); if (value instanceof Node) { node = (Node) value; } else if (value instanceof CrawlDatum) { datum = (CrawlDatum) value; } } // datum should never be null, could happen if somehow the url was // normalized or changed after being pulled from the crawldb if (datum != null) { if (node != null) { // set the inlink score in the nodedb float inlinkScore = node.getInlinkScore(); datum.setScore(inlinkScore); LOG.debug(url + ": setting to score " + inlinkScore); } else { // clear out the score in the crawldb datum.setScore(clearScore); LOG.debug(url + ": setting to clear score of " + clearScore); } output.collect(key, datum); } else { LOG.debug(url + ": no datum"); } }