List of usage examples for org.apache.hadoop.io WritableUtils readVLong
public static long readVLong(DataInput stream) throws IOException
From source file:Counter.java
License:Apache License
/** * Read the binary representation of the counter *///w w w.j a v a2 s.co m @Override public synchronized void readFields(DataInput in) throws IOException { name = Text.readString(in); if (in.readBoolean()) { displayName = Text.readString(in); } else { displayName = name; } value = WritableUtils.readVLong(in); }
From source file:cascading.tuple.hadoop.io.HadoopTupleInputStream.java
License:Open Source License
public long readVLong() throws IOException { return WritableUtils.readVLong(this); }
From source file:co.cask.cdap.data.stream.StreamInputSplit.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); boolean hasIndex = in.readBoolean(); if (hasIndex) { indexPath = new Path(WritableUtils.readString(in)); }/*from w w w. ja va 2 s .c om*/ startTime = WritableUtils.readVLong(in); endTime = WritableUtils.readVLong(in); }
From source file:co.cask.cdap.examples.purchase.Purchase.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { customer = WritableUtils.readString(in); product = WritableUtils.readString(in); quantity = WritableUtils.readVInt(in); price = WritableUtils.readVInt(in);/*www. j a va 2 s. c o m*/ purchaseTime = WritableUtils.readVLong(in); catalogId = WritableUtils.readString(in); }
From source file:co.cask.cdap.hive.stream.StreamInputSplit.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); eventPath = new Path(WritableUtils.readString(in)); boolean hasIndex = in.readBoolean(); if (hasIndex) { indexPath = new Path(WritableUtils.readString(in)); }//w w w. j a v a2 s . c o m startTime = WritableUtils.readVLong(in); endTime = WritableUtils.readVLong(in); }
From source file:com.asakusafw.bridge.hadoop.directio.Util.java
License:Apache License
static DirectInputFragment readFragment(DataInput in) throws IOException { String path = WritableUtils.readString(in); long offset = WritableUtils.readVLong(in); long length = WritableUtils.readVLong(in); String[] locations = WritableUtils.readStringArray(in); Map<String, String> attributes = readMap(in); return new DirectInputFragment(path, offset, length, Arrays.asList(locations), attributes); }
From source file:com.asakusafw.runtime.stage.input.StageInputDriver.java
License:Apache License
@SuppressWarnings("rawtypes") private static List<StageInput> decode(Configuration conf, String encoded) throws IOException, ClassNotFoundException { assert conf != null; assert encoded != null; ByteArrayInputStream source = new ByteArrayInputStream(encoded.getBytes(ASCII)); DataInputStream input = new DataInputStream(new GZIPInputStream(new Base64InputStream(source))); long version = WritableUtils.readVLong(input); if (version != SERIAL_VERSION) { throw new IOException(MessageFormat.format("Invalid StageInput version: framework={0}, saw={1}", SERIAL_VERSION, version)); }//from ww w .ja v a2 s . com String[] dictionary = WritableUtils.readStringArray(input); int inputListSize = WritableUtils.readVInt(input); List<StageInput> results = new ArrayList<>(); for (int inputListIndex = 0; inputListIndex < inputListSize; inputListIndex++) { String pathString = readEncoded(input, dictionary); String formatName = readEncoded(input, dictionary); String mapperName = readEncoded(input, dictionary); int attributeCount = WritableUtils.readVInt(input); Map<String, String> attributes = new HashMap<>(); for (int attributeIndex = 0; attributeIndex < attributeCount; attributeIndex++) { String keyString = readEncoded(input, dictionary); String valueString = readEncoded(input, dictionary); attributes.put(keyString, valueString); } Class<? extends InputFormat> formatClass = conf.getClassByName(formatName) .asSubclass(InputFormat.class); Class<? extends Mapper> mapperClass = conf.getClassByName(mapperName).asSubclass(Mapper.class); results.add(new StageInput(pathString, formatClass, mapperClass, attributes)); } return results; }
From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java
License:Apache License
private static List<OutputSpec> getSpecs(JobContext context) { assert context != null; String encoded = context.getConfiguration().getRaw(KEY); if (encoded == null) { return Collections.emptyList(); }//ww w. j a v a 2 s . c o m VariableTable table = getVariableTable(context); try { ByteArrayInputStream source = new ByteArrayInputStream(encoded.getBytes(ASCII)); DataInputStream input = new DataInputStream(new GZIPInputStream(new Base64InputStream(source))); long version = WritableUtils.readVLong(input); if (version != SERIAL_VERSION) { throw new IOException(MessageFormat.format("Invalid StageOutput version: framework={0}, saw={1}", SERIAL_VERSION, version)); } List<OutputSpec> results = new ArrayList<>(); int specCount = WritableUtils.readVInt(input); for (int specIndex = 0; specIndex < specCount; specIndex++) { String basePath = WritableUtils.readString(input); try { basePath = table.parse(basePath); } catch (IllegalArgumentException e) { throw new IllegalStateException(MessageFormat.format("Invalid basePath: {0}", basePath), e); } int patternCount = WritableUtils.readVInt(input); List<String> patterns = new ArrayList<>(); for (int patternIndex = 0; patternIndex < patternCount; patternIndex++) { String pattern = WritableUtils.readString(input); try { pattern = table.parse(pattern); } catch (IllegalArgumentException e) { throw new IllegalStateException( MessageFormat.format("Invalid delete pattern: {0}", pattern), e); } patterns.add(pattern); } results.add(new OutputSpec(basePath, patterns, true)); } return results; } catch (IOException e) { throw new IllegalStateException(e); } }
From source file:com.bah.culvert.data.CKeyValue.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { // Get the byte length int rowLength = WritableUtils.readVInt(in); int familyLength = WritableUtils.readVInt(in); int qualifierLength = WritableUtils.readVInt(in); int valueLength = WritableUtils.readVInt(in); this.rowId = new byte[rowLength]; this.family = new byte[familyLength]; this.qualifier = new byte[qualifierLength]; this.value = new byte[valueLength]; // read the data in.readFully(this.rowId); in.readFully(this.family); in.readFully(this.qualifier); in.readFully(this.value); this.timestamp = WritableUtils.readVLong(in); }
From source file:com.cloudera.crunch.type.writable.TupleWritable.java
License:Apache License
/** * {@inheritDoc}// w w w . j a v a2 s .c om */ @SuppressWarnings("unchecked") // No static typeinfo on Tuples public void readFields(DataInput in) throws IOException { int card = WritableUtils.readVInt(in); values = new Writable[card]; written = WritableUtils.readVLong(in); Class<? extends Writable>[] cls = new Class[card]; try { for (int i = 0; i < card; ++i) { if (has(i)) { cls[i] = Class.forName(Text.readString(in)).asSubclass(Writable.class); } } for (int i = 0; i < card; ++i) { if (has(i)) { values[i] = cls[i].newInstance(); values[i].readFields(in); } } } catch (ClassNotFoundException e) { throw (IOException) new IOException("Failed tuple init").initCause(e); } catch (IllegalAccessException e) { throw (IOException) new IOException("Failed tuple init").initCause(e); } catch (InstantiationException e) { throw (IOException) new IOException("Failed tuple init").initCause(e); } }