List of usage examples for org.apache.hadoop.io Writable readFields
void readFields(DataInput in) throws IOException;
in. From source file:net.sf.katta.zk.ZKClient.java
License:Apache License
private Writable readWritable(final Writable writable, byte[] data) throws KattaException { final DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(data, data.length);//from w ww.j av a 2 s . c o m try { writable.readFields(buffer); } catch (final IOException e) { throw new KattaException("unable to read data into Writable", e); } finally { try { buffer.close(); } catch (IOException e) { LOG.warn("could not close data buffer", e); } } return writable; }
From source file:org.apache.accumulo.server.tabletserver.log.MultiReader.java
License:Apache License
private static void copy(Writable src, Writable dest) throws IOException { // not exactly efficient... DataOutputBuffer output = new DataOutputBuffer(); src.write(output);//from w ww. j a v a 2s . co m DataInputBuffer input = new DataInputBuffer(); input.reset(output.getData(), output.getLength()); dest.readFields(input); }
From source file:org.apache.accumulo.test.util.SerializationUtil.java
License:Apache License
public static void deserializeWritable(Writable writable, InputStream inputStream) { Objects.requireNonNull(writable); Objects.requireNonNull(inputStream); DataInputStream in = null;/* w w w .java2 s.c o m*/ try { in = new DataInputStream(inputStream); writable.readFields(in); } catch (IOException ex) { throw new RuntimeException(ex); } finally { if (in != null) try { in.close(); } catch (IOException e) { log.error("cannot close", e); } } }
From source file:org.apache.crunch.impl.spark.serde.WritableSerDe.java
License:Apache License
@Override public Writable fromBytes(byte[] bytes) { Writable inst = ReflectionUtils.newInstance(clazz, null); ByteArrayInputStream bais = new ByteArrayInputStream(bytes); DataInputStream dis = new DataInputStream(bais); try {/*w w w . j ava 2 s.com*/ inst.readFields(dis); } catch (IOException e) { throw new RuntimeException(e); } return inst; }
From source file:org.apache.crunch.types.writable.GenericArrayWritable.java
License:Apache License
public void readFields(DataInput in) throws IOException { values = new Writable[WritableUtils.readVInt(in)]; // construct values if (values.length > 0) { int nulls = WritableUtils.readVInt(in); if (nulls == values.length) { return; }/*from ww w . j a v a 2s .c o m*/ String valueType = Text.readString(in); setValueType(valueType); for (int i = 0; i < values.length - nulls; i++) { Writable value = WritableFactories.newInstance(valueClass); value.readFields(in); // read a value values[i] = value; // store it in values } } }
From source file:org.apache.giraph.graph.BspServiceMaster.java
License:Apache License
/** * Get the aggregator values for a particular superstep and aggregate them. * * @param superstep superstep to check//from w w w . ja v a2s .c om */ private void collectAndProcessAggregatorValues(long superstep) { String workerFinishedPath = getWorkerFinishedPath(getApplicationAttempt(), superstep); List<String> hostnameIdPathList = null; try { hostnameIdPathList = getZkExt().getChildrenExt(workerFinishedPath, false, false, true); } catch (KeeperException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: InterruptedException", e); } for (String hostnameIdPath : hostnameIdPathList) { JSONObject workerFinishedInfoObj = null; JSONArray aggregatorArray = null; try { byte[] zkData = getZkExt().getData(hostnameIdPath, false, null); workerFinishedInfoObj = new JSONObject(new String(zkData)); } catch (KeeperException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: InterruptedException", e); } catch (JSONException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: JSONException", e); } try { aggregatorArray = workerFinishedInfoObj.getJSONArray(JSONOBJ_AGGREGATOR_VALUE_ARRAY_KEY); } catch (JSONException e) { if (LOG.isDebugEnabled()) { LOG.debug("collectAndProcessAggregatorValues: " + "No aggregators" + " for " + hostnameIdPath); } continue; } for (int i = 0; i < aggregatorArray.length(); ++i) { try { if (LOG.isInfoEnabled()) { LOG.info("collectAndProcessAggregatorValues: " + "Getting aggregators from " + aggregatorArray.getJSONObject(i)); } String aggregatorName = aggregatorArray.getJSONObject(i).getString(AGGREGATOR_NAME_KEY); String aggregatorClassName = aggregatorArray.getJSONObject(i) .getString(AGGREGATOR_CLASS_NAME_KEY); @SuppressWarnings("unchecked") Aggregator<Writable> aggregator = (Aggregator<Writable>) getAggregator(aggregatorName); boolean firstTime = false; if (aggregator == null) { @SuppressWarnings("unchecked") Class<? extends Aggregator<Writable>> aggregatorClass = (Class<? extends Aggregator<Writable>>) Class .forName(aggregatorClassName); aggregator = registerAggregator(aggregatorName, aggregatorClass); firstTime = true; } Writable aggregatorValue = aggregator.createAggregatedValue(); InputStream input = new ByteArrayInputStream( Base64.decode(aggregatorArray.getJSONObject(i).getString(AGGREGATOR_VALUE_KEY))); aggregatorValue.readFields(new DataInputStream(input)); if (LOG.isDebugEnabled()) { LOG.debug( "collectAndProcessAggregatorValues: " + "aggregator value size=" + input.available() + " for aggregator=" + aggregatorName + " value=" + aggregatorValue); } if (firstTime) { aggregator.setAggregatedValue(aggregatorValue); } else { aggregator.aggregate(aggregatorValue); } } catch (IOException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: " + "IOException when reading aggregator data " + aggregatorArray, e); } catch (JSONException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: " + "JSONException when reading aggregator data " + aggregatorArray, e); } catch (ClassNotFoundException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: " + "ClassNotFoundException when reading aggregator data " + aggregatorArray, e); } catch (InstantiationException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: " + "InstantiationException when reading aggregator data " + aggregatorArray, e); } catch (IllegalAccessException e) { throw new IllegalStateException("collectAndProcessAggregatorValues: " + "IOException when reading aggregator data " + aggregatorArray, e); } } } }
From source file:org.apache.giraph.graph.BspServiceWorker.java
License:Apache License
/** * Get values of aggregators aggregated by master in previous superstep. * * @param superstep Superstep to get the aggregated values from *///from ww w . j av a 2 s . c o m private void getAggregatorValues(long superstep) { String mergedAggregatorPath = getMergedAggregatorPath(getApplicationAttempt(), superstep - 1); JSONArray aggregatorArray = null; try { byte[] zkData = getZkExt().getData(mergedAggregatorPath, false, null); aggregatorArray = new JSONArray(new String(zkData)); } catch (KeeperException.NoNodeException e) { LOG.info("getAggregatorValues: no aggregators in " + mergedAggregatorPath + " on superstep " + superstep); return; } catch (KeeperException e) { throw new IllegalStateException( "Failed to get data for " + mergedAggregatorPath + " with KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException( "Failed to get data for " + mergedAggregatorPath + " with InterruptedException", e); } catch (JSONException e) { throw new IllegalStateException( "Failed to get data for " + mergedAggregatorPath + " with JSONException", e); } for (int i = 0; i < aggregatorArray.length(); ++i) { try { if (LOG.isDebugEnabled()) { LOG.debug("getAggregatorValues: " + "Getting aggregators from " + aggregatorArray.getJSONObject(i)); } String aggregatorName = aggregatorArray.getJSONObject(i).getString(AGGREGATOR_NAME_KEY); Aggregator<Writable> aggregator = getAggregatorMap().get(aggregatorName); if (aggregator == null) { continue; } Writable aggregatorValue = aggregator.getAggregatedValue(); InputStream input = new ByteArrayInputStream( Base64.decode(aggregatorArray.getJSONObject(i).getString(AGGREGATOR_VALUE_KEY))); aggregatorValue.readFields(new DataInputStream(input)); aggregator.setAggregatedValue(aggregatorValue); if (LOG.isDebugEnabled()) { LOG.debug("getAggregatorValues: " + "Got aggregator=" + aggregatorName + " value=" + aggregatorValue); } } catch (JSONException e) { throw new IllegalStateException("Failed to decode data for index " + i + " with KeeperException", e); } catch (IOException e) { throw new IllegalStateException("Failed to decode data for index " + i + " with KeeperException", e); } } if (LOG.isInfoEnabled()) { LOG.info("getAggregatorValues: Finished loading " + mergedAggregatorPath + " with aggregator values " + aggregatorArray); } }
From source file:org.apache.giraph.master.MasterAggregatorHandler.java
License:Apache License
/** * Accept reduced values sent by worker. Every value will be sent * only once, by its owner.//w w w .jav a 2 s . co m * We don't need to count the number of these requests because global * superstep barrier will happen after workers ensure all requests of this * type have been received and processed by master. * * @param reducedValuesInput Input in which aggregated values are * written in the following format: * numReducers * name_1 REDUCED_VALUE value_1 * name_2 REDUCED_VALUE value_2 * ... * @throws IOException */ public void acceptReducedValues(DataInput reducedValuesInput) throws IOException { int numReducers = reducedValuesInput.readInt(); for (int i = 0; i < numReducers; i++) { String name = reducedValuesInput.readUTF(); GlobalCommType type = GlobalCommType.values()[reducedValuesInput.readByte()]; if (type != GlobalCommType.REDUCED_VALUE) { throw new IllegalStateException("SendReducedToMasterRequest received " + type); } Reducer<Object, Writable> reducer = reducerMap.get(name); if (reducer == null) { throw new IllegalStateException( "acceptReducedValues: " + "Master received reduced value which isn't registered: " + name); } Writable valueToReduce = reducer.createInitialValue(); valueToReduce.readFields(reducedValuesInput); if (reducer.getCurrentValue() != null) { reducer.reducePartial(valueToReduce); } else { reducer.setCurrentValue(valueToReduce); } progressable.progress(); } if (LOG.isDebugEnabled()) { LOG.debug("acceptReducedValues: Accepted one set with " + numReducers + " aggregated values"); } }
From source file:org.apache.giraph.utils.WritableUtils.java
License:Apache License
/** * Read fields from byteArray to a Writeable object. * * @param byteArray Byte array to find the fields in. * @param writableObjects Objects to fill in the fields. *//*w w w. j a v a 2s .c om*/ public static void readFieldsFromByteArray(byte[] byteArray, Writable... writableObjects) { DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(byteArray)); try { for (Writable writableObject : writableObjects) { writableObject.readFields(inputStream); } } catch (IOException e) { throw new IllegalStateException("readFieldsFromByteArray: IOException", e); } }
From source file:org.apache.giraph.utils.WritableUtils.java
License:Apache License
/** * Read fields from byteArray to a Writeable object, skipping the size. * Serialization method is choosable//from w w w . j a va 2 s .c o m * * @param byteArray Byte array to find the fields in. * @param writableObject Object to fill in the fields. * @param unsafe Use unsafe deserialization */ public static void readFieldsFromByteArrayWithSize(byte[] byteArray, Writable writableObject, boolean unsafe) { ExtendedDataInput extendedDataInput; if (unsafe) { extendedDataInput = new UnsafeByteArrayInputStream(byteArray); } else { extendedDataInput = new ExtendedByteArrayDataInput(byteArray); } try { extendedDataInput.readInt(); writableObject.readFields(extendedDataInput); } catch (IOException e) { throw new IllegalStateException("readFieldsFromByteArrayWithSize: IOException", e); } }