List of usage examples for org.apache.hadoop.io BytesWritable getBytes
@Override public byte[] getBytes()
From source file:org.apache.blur.kvs.HdfsKeyValueStore.java
License:Apache License
private BytesRef getKey(BytesWritable key) { return new BytesRef(key.getBytes(), 0, key.getLength()); }
From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java
License:Apache License
@Test public void testSequenceWriteFile() throws Exception { if (SKIP) {/*from www .j av a2 s .co m*/ return; } final Path file = new Path(new File("target/test/test-camel-simple-write-file1").getAbsolutePath()); deleteDirectory("target/file-batch2"); context.addRoutes(new RouteBuilder() { @Override public void configure() throws Exception { from("file://target/file-batch2?sortBy=file:name") .to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE"); } }); context.start(); NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create(); for (int i = 0; i < 10; ++i) { template.sendBodyAndHeader("file://target/file-batch2", "CIAO", "CamelFileName", "CIAO" + i); } Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime()); context.stop(); Configuration conf = new Configuration(); Path file1 = new Path("file:///" + file.toUri()); FileSystem fs1 = FileSystem.get(file1.toUri(), conf); SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf); Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf); int i = 0; while (reader.next(key, value)) { String str = new String(value.getBytes(), 0, value.getLength()); Assert.assertEquals("CIAO", str); i++; } Assert.assertEquals(10, i); }
From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java
License:Apache License
@Test public void testSequenceKeyWriteFile() throws Exception { if (SKIP) {/*w ww .jav a 2s . c o m*/ return; } final Path file = new Path(new File("target/test/test-camel-simple-write-file2").getAbsolutePath()); deleteDirectory("target/file-batch3"); context.addRoutes(new RouteBuilder() { @Override public void configure() throws Exception { from("file://target/file-batch3?sortBy=file:name").setHeader("KEY") .simple("${in.header.CamelFileName}").to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&keyType=TEXT&fileType=SEQUENCE_FILE"); } }); context.start(); NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create(); for (int i = 0; i < 10; ++i) { template.sendBodyAndHeader("file://target/file-batch3", "CIAO", "CamelFileName", "CIAO" + i); } Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime()); context.stop(); Configuration conf = new Configuration(); Path file1 = new Path("file:///" + file.toUri()); FileSystem fs1 = FileSystem.get(file1.toUri(), conf); SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf); Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf); int i = 0; while (reader.next(key, value)) { String str = new String(value.getBytes(), 0, value.getLength()); Assert.assertEquals("CIAO", str); Assert.assertEquals("CIAO" + i, key.toString()); i++; } Assert.assertEquals(10, i); }
From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java
License:Apache License
@Test public void testMapKeyWriteFile() throws Exception { if (SKIP) {/* ww w.j a va 2 s . co m*/ return; } final Path file = new Path(new File("target/test/test-camel-simple-write-file1").getAbsolutePath()); deleteDirectory("target/file-batch4"); context.addRoutes(new RouteBuilder() { @Override public void configure() throws Exception { from("file://target/file-batch4?sortBy=file:name").setHeader("KEY") .simple("${in.header.CamelFileName}") .to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&keyType=TEXT&fileType=MAP_FILE"); } }); context.start(); NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create(); for (int i = 0; i < 10; ++i) { template.sendBodyAndHeader("file://target/file-batch4", "CIAO" + i, "CamelFileName", "CIAO" + i); } Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime()); context.stop(); Configuration conf = new Configuration(); Path file1 = new Path("file:///" + file.toUri()); FileSystem fs1 = FileSystem.get(file1.toUri(), conf); MapFile.Reader reader = new MapFile.Reader(fs1, "target/test/test-camel-simple-write-file1", conf); for (int i = 0; i < 10; ++i) { Text key = new Text("CIAO" + i); BytesWritable value = new BytesWritable(); reader.get(key, value); String str = new String(value.getBytes(), 0, value.getLength()); Assert.assertEquals("CIAO" + i, str); Assert.assertEquals("CIAO" + i, key.toString()); } }
From source file:org.apache.crunch.io.hbase.HBaseTypes.java
License:Apache License
public static KeyValue bytesToKeyValue(BytesWritable input) { return bytesToKeyValue(input.getBytes(), 0, input.getLength()); }
From source file:org.apache.druid.indexer.HadoopyStringInputRowParser.java
License:Apache License
@Override public List<InputRow> parseBatch(Object input) { if (input instanceof Text) { return ImmutableList.of(parser.parse(((Text) input).toString())); } else if (input instanceof BytesWritable) { BytesWritable valueBytes = (BytesWritable) input; return parser.parseBatch(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength())); } else {//from www .j av a 2 s. c o m throw new IAE("can't convert type [%s] to InputRow", input.getClass().getName()); } }
From source file:org.apache.flume.sink.customhdfs.TestHDFSEventSink.java
License:Apache License
private void verifyOutputSequenceFiles(FileSystem fs, Configuration conf, String dir, String prefix, List<String> bodies) throws IOException { int found = 0; int expected = bodies.size(); for (String outputFile : getAllFiles(dir)) { String name = (new File(outputFile)).getName(); if (name.startsWith(prefix)) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); while (reader.next(key, value)) { String body = new String(value.getBytes(), 0, value.getLength()); if (bodies.contains(body)) { LOG.debug("Found event body: {}", body); bodies.remove(body); found++;/* w w w. j av a2 s . c om*/ } } reader.close(); } } if (!bodies.isEmpty()) { for (String body : bodies) { LOG.error("Never found event body: {}", body); } } Assert.assertTrue( "Found = " + found + ", Expected = " + expected + ", Left = " + bodies.size() + " " + bodies, bodies.size() == 0); }
From source file:org.apache.giraph.examples.SingleSourceShortestPathOnTemplateNoParent.java
License:Apache License
private void unpackSubgraphMessages( Iterable<SubgraphMessage<LongWritable, BytesWritable>> packedSubGraphMessages, Subgraph<LongWritable, LongWritable, LongWritable, NullWritable, ShortestPathSubgraphValue, NullWritable> subgraph, Set<SubgraphVertex<LongWritable, LongWritable, LongWritable, NullWritable, NullWritable>> rootVertices) throws IOException { ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphVertices().getSubgraphValue(); int unpackedMessageCount = 0; int packedMessageCount = 0; for (SubgraphMessage<LongWritable, BytesWritable> subgraphMessage : packedSubGraphMessages) { packedMessageCount++;/*from w w w . j a v a 2 s .c om*/ BytesWritable subgraphMessageValue = subgraphMessage.getMessage(); ExtendedByteArrayDataInput dataInput = new ExtendedByteArrayDataInput(subgraphMessageValue.getBytes()); while (!dataInput.endOfInput()) { long sinkVertex = dataInput.readLong(); if (sinkVertex == -1) { break; } unpackedMessageCount++; short sinkDistance = dataInput.readShort(); //LOG.info("Test, Sink vertex received: " + sinkVertex); // SubgraphVertex<LongWritable, LongWritable, LongWritable, NullWritable, NullWritable> currentVertex = vertices.get(new LongWritable(sinkVertex)); //LOG.info("Test, Current vertex object: " + currentVertex); //LOG.info("Test, Current vertex: " + currentVertex.getId()); short distance = subgraphValue.shortestDistanceMap.get(sinkVertex); if (sinkDistance < distance) { subgraphValue.shortestDistanceMap.put(sinkVertex, sinkDistance); rootVertices.add(subgraph.getSubgraphVertices().getVertexById(new LongWritable(sinkVertex))); } } } LOG.info("Superstep,SubgraphId,unpackedReceivedMessageCount,packedReceivedMessageCount,rootVertices:" + getSuperstep() + "," + subgraph.getSubgraphId() + "," + unpackedMessageCount + "," + packedMessageCount + "," + rootVertices.size()); }
From source file:org.apache.giraph.examples.SubgraphSingleSourceShortestPathWithWeights.java
License:Apache License
private void unpackSubgraphMessages( Iterable<SubgraphMessage<LongWritable, BytesWritable>> packedSubGraphMessages, Subgraph<LongWritable, LongWritable, LongWritable, DoubleWritable, ShortestPathSubgraphValue, NullWritable> subgraph, Set<SubgraphVertex<LongWritable, LongWritable, LongWritable, DoubleWritable, NullWritable>> rootVertices) throws IOException { ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphVertices().getSubgraphValue(); for (SubgraphMessage<LongWritable, BytesWritable> subgraphMessage : packedSubGraphMessages) { BytesWritable subgraphMessageValue = subgraphMessage.getMessage(); ExtendedByteArrayDataInput dataInput = new ExtendedByteArrayDataInput(subgraphMessageValue.getBytes()); while (!dataInput.endOfInput()) { long sinkVertex = dataInput.readLong(); if (sinkVertex == -1) { break; }/*w ww.j a v a 2s. c o m*/ short sinkDistance = dataInput.readShort(); //LOG.info("Test, Sink vertex received: " + sinkVertex); // SubgraphVertex<LongWritable, LongWritable, LongWritable, NullWritable, NullWritable> currentVertex = vertices.get(new LongWritable(sinkVertex)); //LOG.info("Test, Current vertex object: " + currentVertex); //LOG.info("Test, Current vertex: " + currentVertex.getId()); short distance = subgraphValue.shortestDistanceMap.get(sinkVertex); if (sinkDistance < distance) { subgraphValue.shortestDistanceMap.put(sinkVertex, sinkDistance); rootVertices.add(subgraph.getSubgraphVertices().getVertexById(new LongWritable(sinkVertex))); } } } }
From source file:org.apache.giraph.examples.SubgraphTriangleCount.java
License:Apache License
void unpackMessages(Iterable<SubgraphMessage<LongWritable, BytesWritable>> subgraphMessages, Map<Long, List<Pair<Long, Long>>> ids) throws IOException { for (SubgraphMessage<LongWritable, BytesWritable> messageItem : subgraphMessages) { BytesWritable message = messageItem.getMessage(); ExtendedByteArrayDataInput dataInput = new ExtendedByteArrayDataInput(message.getBytes()); Long targetId;/* ww w. j a v a2s . c om*/ while ((targetId = dataInput.readLong()) != -1) { Long messageId = dataInput.readLong(); Long sourceId = dataInput.readLong(); List<Pair<Long, Long>> idPairs = ids.get(targetId); if (idPairs == null) { idPairs = new LinkedList<Pair<Long, Long>>(); ids.put(targetId, idPairs); } idPairs.add(new Pair<Long, Long>(messageId, sourceId)); } } }