Example usage for org.apache.hadoop.io BytesWritable getLength

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable getLength.

Prototype

@Override
public int getLength() 

Source Link

Document

Get the current size of the buffer.

Usage

From source file:io.warp10.standalone.StandaloneMemoryStore.java

License:Apache License

private void load(String path) throws IOException {

    long nano = System.nanoTime();
    int gts = 0;/* w w  w  .j  a va 2s.c o m*/
    long bytes = 0L;

    Configuration conf = new Configuration();

    conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
    conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();

    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());

    SequenceFile.Reader.Option optPath = SequenceFile.Reader.file(new Path(path));

    SequenceFile.Reader reader = null;

    try {
        reader = new SequenceFile.Reader(conf, optPath);

        System.out.println("Loading '" + path + "' back in memory.");

        while (reader.next(key, value)) {
            gts++;
            GTSWrapper wrapper = new GTSWrapper();
            deserializer.deserialize(wrapper, key.copyBytes());
            GTSEncoder encoder = new GTSEncoder(0L, null, value.copyBytes());
            encoder.setCount(wrapper.getCount());

            bytes += value.getLength() + key.getLength();
            encoder.safeSetMetadata(wrapper.getMetadata());
            store(encoder);
            if (null != this.directoryClient) {
                this.directoryClient.register(wrapper.getMetadata());
            }
        }
    } catch (FileNotFoundException fnfe) {
        System.err.println("File '" + path + "' was not found, skipping.");
        return;
    } catch (IOException ioe) {
        throw ioe;
    } catch (Exception e) {
        throw new IOException(e);
    }

    reader.close();

    nano = System.nanoTime() - nano;

    System.out.println("Loaded " + gts + " GTS (" + bytes + " bytes) in " + (nano / 1000000.0D) + " ms.");
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream./*from w  ww .j a  v a2  s. co  m*/
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else if (obj == null) {
        // write a zero length string
        WritableUtils.writeVInt(stream, 0);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void writeObject(Writable obj, DataOutputStream stream) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    System.err.println("obj: " + obj);

    DataOutputBuffer buffer = new DataOutputBuffer();
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.flush();//  w w  w .j a  v a 2s.  c  om

        stream.write(t.getBytes(), 0, len);
        stream.flush();
        System.err.println("len: " + len);

    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.write(b.getBytes(), 0, len);
        System.err.println("len: " + len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
        System.err.println("len: " + length);
    }
    stream.flush();

}

---

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream./* w w  w .j a  v  a 2s . c  o m*/
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

---

From source file:kafka.etl.KafkaETLUtils.java

License:Apache License

public static byte[] getBytes(BytesWritable val) {

    byte[] buffer = val.getBytes();

    /* FIXME: remove the following part once the below gira is fixed
     * https://issues.apache.org/jira/browse/HADOOP-6298
     *//*from   w w  w  . j a v  a 2 s  . c o m*/
    long len = val.getLength();
    byte[] bytes = buffer;
    if (len < buffer.length) {
        bytes = new byte[(int) len];
        System.arraycopy(buffer, 0, bytes, 0, (int) len);
    }

    return bytes;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFBase64.java

License:Apache License

public Text evaluate(BytesWritable b) {
    if (b == null) {
        return null;
    }//from   w  w  w  .ja v a  2s  .  c  om
    byte[] bytes = new byte[b.getLength()];
    System.arraycopy(b.getBytes(), 0, bytes, 0, b.getLength());
    result.set(Base64.encodeBase64(bytes));
    return result;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFBase64.java

License:Apache License

public Text evaluate(BytesWritable b, Text alphabet) throws Exception {
    if (b == null || alphabet == null) {
        return null;
    }/*from   w  w w  .j  a  v a2  s  .  co m*/

    char[] CA = alphabet.toString().toCharArray();
    if (CA.length != 64) {
        throw new UDFArgumentLengthException("The length of alphabet must be 64.");
    }

    net.mooncloud.hadoop.hive.ql.util.Base64.alphabet(CA);

    byte[] bytes = new byte[b.getLength()];
    System.arraycopy(b.getBytes(), 0, bytes, 0, b.getLength());
    result.set(net.mooncloud.hadoop.hive.ql.util.Base64.encode(bytes));
    return result;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFMd5.java

License:Apache License

/**
 * Convert bytes to md5//from   w w w  . ja  va2  s . co m
 */
public Text evaluate(BytesWritable b) {
    if (b == null) {
        return null;
    }

    digest.reset();
    digest.update(b.getBytes(), 0, b.getLength());
    byte[] md5Bytes = digest.digest();
    String md5Hex = Hex.encodeHexString(md5Bytes);

    result.set(md5Hex);
    return result;
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetBinaryDetails() throws Exception {
    File index = _temporaryFolder.newFolder("indexWithBinaryData");
    String textFieldName = "textField";
    String binaryFieldName = "binaryField";
    String textFieldContent = "sample text";
    byte[] bytesFieldContent = new byte[] { 1, 2, 3 };

    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(index), new StandardAnalyzer(Version.LUCENE_35),
            true, MaxFieldLength.UNLIMITED);
    Document document = new Document();
    document.add(new Field(binaryFieldName, bytesFieldContent, Store.YES));
    document.add(new Field(textFieldName, textFieldContent, Store.NO, Index.ANALYZED));
    indexWriter.addDocument(document);/*ww  w  .j a v  a 2s  .  co  m*/
    indexWriter.optimize();
    indexWriter.close();
    DeployClient deployClient = new DeployClient(_miniCluster.getProtocol());
    IndexState indexState = deployClient.addIndex(index.getName(), index.getParentFile().getAbsolutePath(), 1)
            .joinDeployment();
    assertEquals(IndexState.DEPLOYED, indexState);

    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer())
            .parse(textFieldName + ": " + textFieldContent);
    final Hits hits = client.search(query, new String[] { index.getName() }, 10);
    assertNotNull(hits);
    assertEquals(1, hits.getHits().size());
    final Hit hit = hits.getHits().get(0);
    final MapWritable details = client.getDetails(hit);
    final Set<Writable> keySet = details.keySet();
    assertEquals(1, keySet.size());
    final Writable writable = details.get(new Text(binaryFieldName));
    assertNotNull(writable);
    assertThat(writable, instanceOf(BytesWritable.class));
    BytesWritable bytesWritable = (BytesWritable) writable;
    bytesWritable.setCapacity(bytesWritable.getLength());// getBytes() returns
    // the full array
    assertArrayEquals(bytesFieldContent, bytesWritable.getBytes());
    client.close();
}

---

From source file:org.apache.accumulo.core.file.rfile.bcfile.ByteArray.java

License:Apache License

/**
 * Constructing a ByteArray from a {@link BytesWritable}.
 */// w  w w  . jav  a  2s .  c om
public ByteArray(BytesWritable other) {
    this(other.getBytes(), 0, other.getLength());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• »