Example usage for org.apache.hadoop.record Buffer Buffer

Introduction

In this page you can find the example usage for org.apache.hadoop.record Buffer Buffer.

Prototype

public Buffer(byte[] bytes) 

Source Link

Document

Create a Buffer using the byte array as the initial value.

Usage

From source file:com.dappervision.hbase.mapred.TypedBytesTableRecordReader.java

License:Apache License

/**
 * @param key HStoreKey as input key.//from w  w  w  . j a va 2 s .co  m
 * @param value MapWritable as input value
 * @return true if there was more data
 * @throws IOException
 */
public boolean next(TypedBytesWritable key, TypedBytesWritable value) throws IOException {
    ImmutableBytesWritable key0 = new ImmutableBytesWritable();
    Result value0 = new Result();
    boolean out = this.recordReaderImpl.next(key0, value0);
    if (out) {
        TreeMap tm = new TreeMap();
        for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> entry : value0.getNoVersionMap().entrySet()) {
            TreeMap tm_inner = new TreeMap();
            for (Map.Entry<byte[], byte[]> entry0 : entry.getValue().entrySet()) {
                tm_inner.put(new Buffer(entry0.getKey()), new Buffer(entry0.getValue()));
            }
            tm.put(new Buffer(entry.getKey()), tm_inner);
        }
        key.setValue(new Buffer(key0.get()));
        value.setValue(tm);
    }
    return out;

}

---

From source file:com.dappervision.hbase.mapred.TypedBytesTableRecordReaderSingleValue.java

License:Apache License

/**
 * @param key HStoreKey as input key.//from w  w w . j av a  2  s .co m
 * @param value MapWritable as input value
 * @return true if there was more data
 * @throws IOException
 */
public boolean next(TypedBytesWritable key, TypedBytesWritable value) throws IOException {
    ImmutableBytesWritable key0 = new ImmutableBytesWritable();
    Result value0 = new Result();
    boolean out = this.recordReaderImpl.next(key0, value0);
    if (out) {
        byte[] value_byte = value0.value();
        if (value_byte == null) {
            throw new IOException(
                    "SingleValue requires at least one column to be present for each row, this should not be possible!");
        }
        key.setValue(new Buffer(key0.get()));
        value.setValue(new Buffer(value_byte));
    }
    return out;

}

---

From source file:com.dappervision.hbase.mapred.TypedBytesTableReducer.java

License:Apache License

@Override
public void reduce(Text key, Iterator<Text> values,
        OutputCollector<TypedBytesWritable, TypedBytesWritable> outputCollector, Reporter arg3)
        throws IOException {
    byte[] keyBytes = key.getBytes();
    TypedBytesWritable keyWritable = new TypedBytesWritable();
    TypedBytesWritable valueWritable = new TypedBytesWritable();
    keyWritable.setValue(new Buffer(keyBytes));

    //merge the column family and qualifier
    HashMap<String, HashMap<String, String>> cfMap = new HashMap<String, HashMap<String, String>>();
    while (values.hasNext()) {
        Text value = values.next();
        String strVal = value.toString();
        //Separate column family with comma (:)
        //Separate the qualifier and value with equity
        String[] cf_qual_val_parts = strVal.split(":");
        String cf = cf_qual_val_parts[0];
        String qual_val = cf_qual_val_parts[1];
        String[] qual_val_parts = qual_val.split("=");
        String qual = qual_val_parts[0];
        String val = qual_val_parts[1];

        if (cfMap.get(cf) != null) {
            HashMap<String, String> qualMap = cfMap.get(cf);
            if (qualMap == null) {
                qualMap = new HashMap<String, String>();
            }/*from www  . j a v a2s.com*/
            qualMap.put(qual, val); // the duplicated key will be replaced, if using Buffer, we should do it ourselves
        } else {
            HashMap<String, String> qualMap = new HashMap<String, String>();
            qualMap.put(qual, val);
            cfMap.put(cf, qualMap);
        }
    }

    HashMap<Buffer, HashMap<Buffer, Buffer>> bufMap = new HashMap<Buffer, HashMap<Buffer, Buffer>>();
    Set<Entry<String, HashMap<String, String>>> entrySet = cfMap.entrySet();
    for (Entry<String, HashMap<String, String>> entry : entrySet) {
        HashMap<String, String> qualValMap = entry.getValue();

        HashMap<Buffer, Buffer> qualValBufMap = new HashMap<Buffer, Buffer>();
        for (Entry<String, String> qualValEntry : qualValMap.entrySet()) {
            qualValBufMap.put(new Buffer(qualValEntry.getKey().getBytes()),
                    new Buffer(qualValEntry.getValue().getBytes()));
        }

        bufMap.put(new Buffer(entry.getKey().getBytes()), qualValBufMap);
    }
    valueWritable.setValue(bufMap);

    outputCollector.collect(keyWritable, valueWritable);
}

---

From source file:com.jfolson.hive.serde.RTypedBytesInput.java

License:Apache License

public Object read(int code) throws IOException {
    if (code == RType.BYTES.code) {
        return new Buffer(readBytes());
    } else if (code == RType.BYTE.code) {
        return readByte();
    } else if (code == RType.BOOL.code) {
        return readBool();
    } else if (code == RType.INT.code) {
        return readInt();
    } else if (code == RType.SHORT.code) {
        return readShort();
    } else if (code == RType.LONG.code) {
        return readLong();
    } else if (code == RType.FLOAT.code) {
        return readFloat();
    } else if (code == RType.DOUBLE.code) {
        return readDouble();
    } else if (code == RType.STRING.code) {
        return readString();
    } else if (code == RType.VECTOR.code) {
        return readVector();
    } else if (code == RType.LIST.code) {
        return readList();
    } else if (code == RType.MAP.code) {
        return readMap();
    } else if (code == RType.NULL.code) {
        return null;
    } else if (code == RType.RNATIVE.code) {
        return readRNative();
    } else if (code == RType.MARKER.code) {
        return null;
    } else if (50 <= code && code <= 200) { // application-specific typecodes
        byte[] bytes = readRawBytes();
        bytes[0] = (byte) code;
        return new TypedBytesWritable(bytes);
    } else {/*from w ww  .j a v a  2s.  c  o m*/
        throw new RuntimeException("unknown type");
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesInput.java

License:Apache License

/**
 * Reads the raw bytes following a <code>Type.LIST</code> code.
 * /*from   www .  jav a2s  .  c o  m*/
 * @return the obtained bytes sequence
 * @throws IOException
 */
public byte[] readRawList() throws IOException {
    Buffer buffer = new Buffer(new byte[] { (byte) RType.LIST.code });
    byte[] bytes = readRaw();
    while (bytes != null) {
        buffer.append(bytes);
        bytes = readRaw();
    }
    buffer.append(new byte[] { (byte) RType.MARKER.code });
    return buffer.get();
}

---

From source file:com.jfolson.hive.serde.RTypedBytesRecordInput.java

License:Apache License

public Buffer readBuffer(String tag) throws IOException {
    in.skipType();
    return new Buffer(in.readBytes());
}

---

From source file:com.nexr.data.sdp.rolling.hdfs.LogRecord.java

License:Apache License

public void add(String key, String value) {
    synchronized (this) {
        if (this.mapFields == null) {
            this.mapFields = new TreeMap<String, org.apache.hadoop.record.Buffer>();
        }//  ww w  .  ja  v a 2s. c  o  m
    }
    this.mapFields.put(key, new Buffer(value.getBytes()));
}

---

From source file:org.commoncrawl.service.directory.BlockingClient.java

License:Open Source License

public static byte[] loadDataFromPath(InetAddress directoryServiceServer, String itemPath) throws IOException {

    final BlockingClient client = new BlockingClient();

    try {/* w  w  w  .  j  av  a  2  s . c  o  m*/

        client.connect(directoryServiceServer);

        DirectoryServiceQuery query = new DirectoryServiceQuery();
        query.setItemPath(itemPath);

        client._blockingCallSemaphore = new Semaphore(0);
        client._serviceStub.query(query,
                new AsyncRequest.Callback<DirectoryServiceQuery, DirectoryServiceItemList>() {
                    @Override
                    public void requestComplete(
                            AsyncRequest<DirectoryServiceQuery, DirectoryServiceItemList> request) {

                        if (request.getStatus() == AsyncRequest.Status.Success) {

                            DirectoryServiceItem item = request.getOutput().getItems().get(0);
                            client._buffer = new Buffer(item.getItemData().getReadOnlyBytes());
                        } else {
                            LOG.error("Request:" + request.getInput().getItemPath() + " returned NULL result.");
                        }
                        if (client._blockingCallSemaphore != null) {
                            client._blockingCallSemaphore.release();
                        }
                    }
                });
        client._blockingCallSemaphore.acquireUninterruptibly();
        client._blockingCallSemaphore = null;

        if (client._buffer.getCount() == 0) {
            throw new IOException("Failed to retrieve item at path:" + itemPath);
        } else {
            return client._buffer.get();
        }

    } catch (IOException e) {
        throw e;
    }
}

---

From source file:org.commoncrawl.service.directory.DirectoryServiceCmdLineTool.java

License:Open Source License

public void doImport(String directoryPath, File importFileName) throws IOException {

    Log.info("Import: Dir Path:" + directoryPath + " Incoming FilePath:" + importFileName);

    FileInputStream inputStream = new FileInputStream(importFileName);
    byte[] data = new byte[(int) importFileName.length()];
    inputStream.read(data);//from www  . j  ava 2s. c  o  m

    DirectoryServiceItem item = new DirectoryServiceItem();
    item.setItemPath(directoryPath);
    item.setItemData(new Buffer(data));
    _blockingCallSemaphore = new Semaphore(0);
    _serviceStub.publish(item, new AsyncRequest.Callback<DirectoryServiceItem, DirectoryServiceItem>() {
        @Override
        public void requestComplete(AsyncRequest<DirectoryServiceItem, DirectoryServiceItem> request) {
            if (request.getStatus() == AsyncRequest.Status.Success) {
                System.out.println(
                        "Request Succeeded. Version Number is:" + request.getOutput().getVersionNumber());
            } else {
                System.out.println("Request Failed");
            }
            if (_blockingCallSemaphore != null) {
                _blockingCallSemaphore.release();
            }
        }
    });
    _blockingCallSemaphore.acquireUninterruptibly();
    _blockingCallSemaphore = null;
}

---

From source file:org.commoncrawl.service.directory.DirectoryServiceTester.java

License:Open Source License

void publishTestItem(final String path, String testBuffer) throws IOException {
    byte bytes[] = testBuffer.getBytes(Charset.forName("UTF8"));

    DirectoryServiceItem item = new DirectoryServiceItem();
    item.setItemPath(path);/*from w  w w  .  j  ava2  s.c  o  m*/
    item.setItemData(new Buffer(bytes));

    _serviceStub.publish(item, new AsyncRequest.Callback<DirectoryServiceItem, DirectoryServiceItem>() {

        @Override
        public void requestComplete(AsyncRequest<DirectoryServiceItem, DirectoryServiceItem> request) {
            LOG.info("Publish of Item:" + path + " Completed.");
        }
    });
}

---

• «
• 1
• 2
• »