Example usage for org.apache.hadoop.io BytesWritable getBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable getBytes.

Prototype

@Override
public byte[] getBytes() 

Source Link

Document

Get the data backing the BytesWritable.

Usage

From source file:kafka.etl.KafkaETLUtils.java

License:Apache License

public static byte[] getBytes(BytesWritable val) {

    byte[] buffer = val.getBytes();

    /* FIXME: remove the following part once the below gira is fixed
     * https://issues.apache.org/jira/browse/HADOOP-6298
     *///from  w w w .ja  va2s.  c  o m
    long len = val.getLength();
    byte[] bytes = buffer;
    if (len < buffer.length) {
        bytes = new byte[(int) len];
        System.arraycopy(buffer, 0, bytes, 0, (int) len);
    }

    return bytes;
}

---

From source file:ml.shifu.shifu.core.autotype.AutoTypeDistinctCountReducer.java

License:Apache License

@Override
protected void reduce(IntWritable key, Iterable<BytesWritable> values, Context context)
        throws IOException, InterruptedException {
    HyperLogLogPlus hyperLogLogPlus = null;
    for (BytesWritable bytesWritable : values) {
        if (hyperLogLogPlus == null) {
            hyperLogLogPlus = HyperLogLogPlus.Builder.build(bytesWritable.getBytes());
        } else {/*from www . j a va  2s . co  m*/
            try {
                hyperLogLogPlus = (HyperLogLogPlus) hyperLogLogPlus
                        .merge(HyperLogLogPlus.Builder.build(bytesWritable.getBytes()));
            } catch (CardinalityMergeException e) {
                throw new RuntimeException(e);
            }
        }
    }
    outputValue.set(hyperLogLogPlus.cardinality());
    context.write(key, outputValue);
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFBase64.java

License:Apache License

public Text evaluate(BytesWritable b) {
    if (b == null) {
        return null;
    }// ww  w. j a v a  2 s.  co m
    byte[] bytes = new byte[b.getLength()];
    System.arraycopy(b.getBytes(), 0, bytes, 0, b.getLength());
    result.set(Base64.encodeBase64(bytes));
    return result;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFBase64.java

License:Apache License

public Text evaluate(BytesWritable b, Text alphabet) throws Exception {
    if (b == null || alphabet == null) {
        return null;
    }//from w  ww . java2s  .  c  o  m

    char[] CA = alphabet.toString().toCharArray();
    if (CA.length != 64) {
        throw new UDFArgumentLengthException("The length of alphabet must be 64.");
    }

    net.mooncloud.hadoop.hive.ql.util.Base64.alphabet(CA);

    byte[] bytes = new byte[b.getLength()];
    System.arraycopy(b.getBytes(), 0, bytes, 0, b.getLength());
    result.set(net.mooncloud.hadoop.hive.ql.util.Base64.encode(bytes));
    return result;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFMd5.java

License:Apache License

/**
 * Convert bytes to md5/*from   w  ww  .  j  av a2s.c  om*/
 */
public Text evaluate(BytesWritable b) {
    if (b == null) {
        return null;
    }

    digest.reset();
    digest.update(b.getBytes(), 0, b.getLength());
    byte[] md5Bytes = digest.digest();
    String md5Hex = Hex.encodeHexString(md5Bytes);

    result.set(md5Hex);
    return result;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFRSASign.java

License:Apache License

public BytesWritable evaluate(BytesWritable b, BytesWritable privateKey) {
    if (b == null || privateKey == null) {
        return null;
    }/*w w w . j  a  v  a 2s  . c o  m*/

    try {
        result = new BytesWritable(RSAUtils.sign(b.getBytes(), privateKey.getBytes()));
    } catch (Exception e) {
        e.printStackTrace();
    }
    return result;
}

---

From source file:net.mooncloud.hadoop.hive.ql.udf.UDFRSAVerify.java

License:Apache License

public BooleanWritable evaluate(BytesWritable b, BytesWritable sign, BytesWritable publicKey) {
    if (b == null || sign == null || publicKey == null) {
        return null;
    }/*from   w w  w .  j  a va2s.co  m*/

    try {
        result = new BooleanWritable(RSAUtils.verify(b.getBytes(), publicKey.getBytes(), sign.getBytes()));
    } catch (Exception e) {
        e.printStackTrace();
    }
    return result;
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetBinaryDetails() throws Exception {
    File index = _temporaryFolder.newFolder("indexWithBinaryData");
    String textFieldName = "textField";
    String binaryFieldName = "binaryField";
    String textFieldContent = "sample text";
    byte[] bytesFieldContent = new byte[] { 1, 2, 3 };

    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(index), new StandardAnalyzer(Version.LUCENE_35),
            true, MaxFieldLength.UNLIMITED);
    Document document = new Document();
    document.add(new Field(binaryFieldName, bytesFieldContent, Store.YES));
    document.add(new Field(textFieldName, textFieldContent, Store.NO, Index.ANALYZED));
    indexWriter.addDocument(document);// ww  w  . j a v  a 2 s .  c o  m
    indexWriter.optimize();
    indexWriter.close();
    DeployClient deployClient = new DeployClient(_miniCluster.getProtocol());
    IndexState indexState = deployClient.addIndex(index.getName(), index.getParentFile().getAbsolutePath(), 1)
            .joinDeployment();
    assertEquals(IndexState.DEPLOYED, indexState);

    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer())
            .parse(textFieldName + ": " + textFieldContent);
    final Hits hits = client.search(query, new String[] { index.getName() }, 10);
    assertNotNull(hits);
    assertEquals(1, hits.getHits().size());
    final Hit hit = hits.getHits().get(0);
    final MapWritable details = client.getDetails(hit);
    final Set<Writable> keySet = details.keySet();
    assertEquals(1, keySet.size());
    final Writable writable = details.get(new Text(binaryFieldName));
    assertNotNull(writable);
    assertThat(writable, instanceOf(BytesWritable.class));
    BytesWritable bytesWritable = (BytesWritable) writable;
    bytesWritable.setCapacity(bytesWritable.getLength());// getBytes() returns
    // the full array
    assertArrayEquals(bytesFieldContent, bytesWritable.getBytes());
    client.close();
}

---

From source file:org.apache.accumulo.core.data.MapFileTest.java

License:Apache License

public void testMapFileIndexing() {
    try {/*  w w w.  j  ava 2s. c om*/
        Configuration conf = CachedConfiguration.getInstance();
        FileSystem fs = FileSystem.get(conf);
        conf.setInt("io.seqfile.compress.blocksize", 2000);

        /*****************************
         * write out the test map file
         */
        MyMapFile.Writer mfw = new MyMapFile.Writer(conf, fs, "/tmp/testMapFileIndexingMap", Text.class,
                BytesWritable.class, CompressionType.BLOCK);
        Text key = new Text();
        BytesWritable value;
        Random r = new Random();
        byte[] bytes = new byte[1024];
        ArrayList<String> keys = new ArrayList<String>();
        ArrayList<String> badKeys = new ArrayList<String>();
        for (int i = 0; i < 1000; i++) {
            String keyString = Integer.toString(i + 1000000);
            keys.add(keyString);
            badKeys.add(keyString + "_");
            key.set(keyString);
            r.nextBytes(bytes);
            bytes[0] = (byte) keyString.charAt(0);
            bytes[1] = (byte) keyString.charAt(1);
            bytes[2] = (byte) keyString.charAt(2);
            bytes[3] = (byte) keyString.charAt(3);
            bytes[4] = (byte) keyString.charAt(4);
            bytes[5] = (byte) keyString.charAt(5);
            bytes[6] = (byte) keyString.charAt(6);
            value = new BytesWritable(bytes);
            mfw.append(key, value);
        }
        mfw.close();

        MyMapFile.Reader mfr = new MyMapFile.Reader(fs, "/tmp/testMapFileIndexingMap", conf);
        value = new BytesWritable();
        // test edge cases
        key.set(keys.get(0));
        assertTrue(mfr.seek(key));
        key.set(keys.get(keys.size() - 1));
        assertTrue(mfr.seek(key));
        key.set("");
        assertFalse(mfr.seek(key));
        key.set(keys.get(keys.size() - 1) + "_");
        assertFalse(mfr.seek(key));

        // test interaction with nextKey
        key.set(keys.get(17));
        assertTrue(mfr.seek(key));
        assertTrue(mfr.next(key, value));
        assertTrue(mfr.seek(key, key));

        // test seeking before the beginning of the file
        key.set("");
        Text closestKey = (Text) mfr.getClosest(key, value, false, null);
        assertTrue(closestKey.toString().equals(keys.get(0)));
        assertTrue(value.getBytes()[6] == (byte) (closestKey.toString().charAt(6)));
        closestKey = (Text) mfr.getClosest(key, value, false, closestKey);
        assertTrue(closestKey.toString().equals(keys.get(0)));
        assertTrue(value.getBytes()[6] == (byte) (closestKey.toString().charAt(6)));
        closestKey = (Text) mfr.getClosest(key, value, false, closestKey);
        assertTrue(closestKey.toString().equals(keys.get(0)));
        assertTrue(value.getBytes()[6] == (byte) (closestKey.toString().charAt(6)));

        // test seeking after the end of the file
        key.set(keys.get(keys.size() - 1));
        closestKey = (Text) mfr.getClosest(key, value, false, null);
        assertTrue(keys.get(keys.size() - 1).equals(closestKey.toString()));
        key.set("z");
        closestKey = (Text) mfr.getClosest(key, value, false, closestKey);
        assertTrue(closestKey == null);
        key.set("zz");
        closestKey = (Text) mfr.getClosest(key, value, false, closestKey);
        assertTrue(closestKey == null);

        key.set(keys.get(keys.size() - 1));
        closestKey = (Text) mfr.getClosest(key, value, false, null);
        assertFalse(mfr.next(closestKey, value));
        key.set("z");
        closestKey = (Text) mfr.getClosest(key, value, false, closestKey);
        assertTrue(closestKey == null);

        // test sequential reads
        for (int sample = 0; sample < 1000; sample++) {
            key.set(keys.get(sample));
            assertTrue(mfr.seek(key));
        }

        // test sequential misses
        for (int sample = 0; sample < 1000; sample++) {
            key.set(badKeys.get(sample));
            assertFalse(mfr.seek(key));
        }

        // test backwards reads
        for (int sample = 0; sample < 1000; sample++) {
            key.set(keys.get(keys.size() - 1 - sample));
            assertTrue(mfr.seek(key));
        }

        // test backwards misses
        for (int sample = 0; sample < 1000; sample++) {
            key.set(badKeys.get(badKeys.size() - 1 - sample));
            assertFalse(mfr.seek(key));
        }

        // test interleaved reads
        for (int sample = 0; sample < 1000; sample++) {
            key.set(keys.get(sample));
            assertTrue(mfr.seek(key));
            key.set(badKeys.get(sample));
            assertFalse(mfr.seek(key));
        }

        // test random reads
        Collections.shuffle(keys);
        Collections.shuffle(badKeys);
        for (int sample = 0; sample < 1000; sample++) {
            key.set(keys.get(sample));
            boolean seekGood = mfr.seek(key);
            if (!seekGood) {
                log.info("Key: " + keys.get(sample));
                if (sample != 0) {
                    log.info("Last key: " + keys.get(sample - 1));
                }
            }
            assertTrue(seekGood);
            key.set(badKeys.get(sample));
            assertTrue(!mfr.seek(key));
        }

        fs.delete(new Path("/tmp/testMapFileIndexingMap"), true);
    } catch (IOException e) {
        e.printStackTrace();
    }
}

---

From source file:org.apache.accumulo.core.file.rfile.bcfile.ByteArray.java

License:Apache License

/**
 * Constructing a ByteArray from a {@link BytesWritable}.
 *//* w w  w .j  av  a  2s  .c om*/
public ByteArray(BytesWritable other) {
    this(other.getBytes(), 0, other.getLength());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• »