Example usage for org.apache.hadoop.io BytesWritable getLength

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable getLength.

Prototype

@Override
public int getLength() 

Source Link

Document

Get the current size of the buffer.

Usage

From source file:org.apache.blur.kvs.HdfsKeyValueStore.java

License:Apache License

private BytesRef getKey(BytesWritable key) {
    return new BytesRef(key.getBytes(), 0, key.getLength());
}

---

From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java

License:Apache License

@Test
public void testSequenceWriteFile() throws Exception {
    if (SKIP) {/*from  w w w.j av a2 s  .com*/
        return;
    }

    final Path file = new Path(new File("target/test/test-camel-simple-write-file1").getAbsolutePath());
    deleteDirectory("target/file-batch2");

    context.addRoutes(new RouteBuilder() {
        @Override
        public void configure() throws Exception {
            from("file://target/file-batch2?sortBy=file:name")
                    .to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE");
        }
    });

    context.start();

    NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create();

    for (int i = 0; i < 10; ++i) {
        template.sendBodyAndHeader("file://target/file-batch2", "CIAO", "CamelFileName", "CIAO" + i);
    }

    Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime());
    context.stop();

    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + file.toUri());
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf);
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
    int i = 0;
    while (reader.next(key, value)) {
        String str = new String(value.getBytes(), 0, value.getLength());
        Assert.assertEquals("CIAO", str);
        i++;
    }
    Assert.assertEquals(10, i);
}

---

From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java

License:Apache License

@Test
public void testSequenceKeyWriteFile() throws Exception {
    if (SKIP) {/*from w  w w.  ja v  a  2 s. c om*/
        return;
    }

    final Path file = new Path(new File("target/test/test-camel-simple-write-file2").getAbsolutePath());
    deleteDirectory("target/file-batch3");

    context.addRoutes(new RouteBuilder() {
        @Override
        public void configure() throws Exception {
            from("file://target/file-batch3?sortBy=file:name").setHeader("KEY")
                    .simple("${in.header.CamelFileName}").to("hdfs:///" + file.toUri()
                            + "?fileSystemType=LOCAL&keyType=TEXT&fileType=SEQUENCE_FILE");
        }
    });

    context.start();

    NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create();

    for (int i = 0; i < 10; ++i) {
        template.sendBodyAndHeader("file://target/file-batch3", "CIAO", "CamelFileName", "CIAO" + i);
    }

    Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime());
    context.stop();

    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + file.toUri());
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf);
    Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
    int i = 0;
    while (reader.next(key, value)) {
        String str = new String(value.getBytes(), 0, value.getLength());
        Assert.assertEquals("CIAO", str);
        Assert.assertEquals("CIAO" + i, key.toString());
        i++;
    }
    Assert.assertEquals(10, i);
}

---

From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java

License:Apache License

@Test
public void testMapKeyWriteFile() throws Exception {
    if (SKIP) {/*w  w w. java2  s  .  com*/
        return;
    }

    final Path file = new Path(new File("target/test/test-camel-simple-write-file1").getAbsolutePath());
    deleteDirectory("target/file-batch4");

    context.addRoutes(new RouteBuilder() {
        @Override
        public void configure() throws Exception {
            from("file://target/file-batch4?sortBy=file:name").setHeader("KEY")
                    .simple("${in.header.CamelFileName}")
                    .to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&keyType=TEXT&fileType=MAP_FILE");
        }
    });

    context.start();

    NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create();

    for (int i = 0; i < 10; ++i) {
        template.sendBodyAndHeader("file://target/file-batch4", "CIAO" + i, "CamelFileName", "CIAO" + i);
    }

    Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime());
    context.stop();

    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + file.toUri());
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    MapFile.Reader reader = new MapFile.Reader(fs1, "target/test/test-camel-simple-write-file1", conf);
    for (int i = 0; i < 10; ++i) {
        Text key = new Text("CIAO" + i);
        BytesWritable value = new BytesWritable();
        reader.get(key, value);
        String str = new String(value.getBytes(), 0, value.getLength());
        Assert.assertEquals("CIAO" + i, str);
        Assert.assertEquals("CIAO" + i, key.toString());
    }
}

---

From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java

License:Apache License

@Test
public void testSequenceKeyWriteBigFile() throws Exception {
    if (SKIP) {/*from  www  .  j  a v  a 2  s. c o  m*/
        return;
    }

    final Path file = new Path(new File("target/test/test-camel-simple-write-file1").getAbsolutePath());
    deleteDirectory("target/file-batch5");

    context.addRoutes(new RouteBuilder() {
        @Override
        public void configure() throws Exception {
            from("file://target/file-batch5?sortBy=file:name").to("hdfs:///" + file.toUri()
                    + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&splitStrategy=IDLE:100&checkIdleInterval=10");
        }
    });

    context.start();

    NotifyBuilder nb = new NotifyBuilder(context).whenDone(2).create();

    ByteBuffer bb = ByteBuffer.allocate(8 * 1024 * 1024);
    for (int i = 0; i < 8 * 1024 * 512; ++i) {
        bb.putChar('A');
    }
    for (int i = 0; i < 2; ++i) {
        template.sendBodyAndHeader("file://target/file-batch5", bb, "CamelFileName", "CIAO" + i);
    }

    Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime());
    context.stop();

    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + file.toUri() + '/' + HdfsConstants.DEFAULT_SEGMENT_PREFIX + 0);
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf);
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
    int i = 0;
    while (reader.next(key, value)) {
        Assert.assertEquals(value.getLength(), 8 * 1024 * 1024);
        i++;
    }
    Assert.assertEquals(2, i);
}

---

From source file:org.apache.crunch.io.hbase.HBaseTypes.java

License:Apache License

public static KeyValue bytesToKeyValue(BytesWritable input) {
    return bytesToKeyValue(input.getBytes(), 0, input.getLength());
}

---

From source file:org.apache.druid.indexer.HadoopyStringInputRowParser.java

License:Apache License

@Override
public List<InputRow> parseBatch(Object input) {
    if (input instanceof Text) {
        return ImmutableList.of(parser.parse(((Text) input).toString()));
    } else if (input instanceof BytesWritable) {
        BytesWritable valueBytes = (BytesWritable) input;
        return parser.parseBatch(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength()));
    } else {// w  w w  .  j a  v  a2 s . c  o m
        throw new IAE("can't convert type [%s] to InputRow", input.getClass().getName());
    }
}

---

From source file:org.apache.flume.sink.customhdfs.TestHDFSEventSink.java

License:Apache License

private void verifyOutputSequenceFiles(FileSystem fs, Configuration conf, String dir, String prefix,
        List<String> bodies) throws IOException {
    int found = 0;
    int expected = bodies.size();
    for (String outputFile : getAllFiles(dir)) {
        String name = (new File(outputFile)).getName();
        if (name.startsWith(prefix)) {
            SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
            LongWritable key = new LongWritable();
            BytesWritable value = new BytesWritable();
            while (reader.next(key, value)) {
                String body = new String(value.getBytes(), 0, value.getLength());
                if (bodies.contains(body)) {
                    LOG.debug("Found event body: {}", body);
                    bodies.remove(body);
                    found++;/*from   w ww .j a va 2s. c  o m*/
                }
            }
            reader.close();
        }
    }
    if (!bodies.isEmpty()) {
        for (String body : bodies) {
            LOG.error("Never found event body: {}", body);
        }
    }
    Assert.assertTrue(
            "Found = " + found + ", Expected = " + expected + ", Left = " + bodies.size() + " " + bodies,
            bodies.size() == 0);

}

---

From source file:org.apache.hama.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream.//from  w w w . ja va2s.c om
 * 
 * @param obj the object to write
 * @throws IOException
 */
protected void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

---

From source file:org.apache.hama.pipes.protocol.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a IntWritable, LongWritable,
 * FloatWritable, DoubleWritable, Text or BytesWritable, write it directly.
 * Otherwise, write it to a buffer and then write the length and data to the
 * stream./*from w w w  .j a  v  a2s  .c o  m*/
 * 
 * @param obj the object to write
 * @throws IOException
 */
protected void writeObject(Writable obj) throws IOException {
    // For basic types IntWritable, LongWritable, Text and BytesWritable,
    // encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(this.outStream, len);
        this.outStream.write(t.getBytes(), 0, len);

    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(this.outStream, len);
        this.outStream.write(b.getBytes(), 0, len);

    } else if (obj instanceof IntWritable) {
        WritableUtils.writeVInt(this.outStream, ((IntWritable) obj).get());

    } else if (obj instanceof LongWritable) {
        WritableUtils.writeVLong(this.outStream, ((LongWritable) obj).get());

    } else {
        // Note: FloatWritable and DoubleWritable are written here
        obj.write(this.outStream);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• »