Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Copyright 2013 Josh Elser * */ package cosmos.accumulo; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.SortedMapIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.minicluster.MiniAccumuloConfig; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.VLongWritable; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.io.Files; import cosmos.IntegrationTests; import cosmos.accumulo.GroupByRowSuffixIterator; /** * */ @Category(IntegrationTests.class) public class GroupByRowSuffixIteratorTest { protected static File tmp = Files.createTempDir(); protected static MiniAccumuloCluster mac; protected static MiniAccumuloConfig macConfig; @BeforeClass public static void setup() throws IOException, InterruptedException { macConfig = new MiniAccumuloConfig(tmp, "root"); macConfig.setNumTservers(2); mac = new MiniAccumuloCluster(macConfig); mac.start(); // Do this now so we don't forget later (or get an exception) tmp.deleteOnExit(); } @AfterClass public static void teardown() throws IOException, InterruptedException { mac.stop(); } @Test public void testSingleRow() throws Exception { ZooKeeperInstance zk = new ZooKeeperInstance(mac.getInstanceName(), mac.getZooKeepers()); Connector c = zk.getConnector("root", new PasswordToken("root")); final String tableName = "foo"; if (c.tableOperations().exists(tableName)) { c.tableOperations().delete(tableName); } c.tableOperations().create(tableName); BatchWriter bw = null; try { bw = c.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("1_a"); m.put("a", "a", 0, ""); m.put("a", "b", 0, ""); bw.addMutation(m); m = new Mutation("1_b"); m.put("a", "a", 0, ""); bw.addMutation(m); m = new Mutation("1_c"); m.put("a", "a", 0, ""); m.put("b", "a", 0, ""); m.put("c", "a", 0, ""); bw.addMutation(m); } finally { if (null != bw) { bw.close(); } } Map<Key, Long> results = ImmutableMap.<Key, Long>builder().put(new Key("1_a", "a", "b", 0), 2l) .put(new Key("1_b", "a", "a", 0), 1l).put(new Key("1_c", "c", "a", 0), 3l).build(); BatchScanner bs = c.createBatchScanner(tableName, new Authorizations(), 1); bs.setRanges(Collections.singleton(new Range())); try { IteratorSetting cfg = new IteratorSetting(30, GroupByRowSuffixIterator.class); bs.addScanIterator(cfg); long count = 0; for (Entry<Key, Value> entry : bs) { VLongWritable writable = GroupByRowSuffixIterator.getWritable(entry.getValue()); Assert.assertTrue("Results did not contain: " + entry.getKey(), results.containsKey(entry.getKey())); Assert.assertEquals(results.get(entry.getKey()).longValue(), writable.get()); count++; } Assert.assertEquals(results.size(), count); } finally { if (null != bs) { bs.close(); } } } @Test public void testMultipleRows() throws Exception { ZooKeeperInstance zk = new ZooKeeperInstance(mac.getInstanceName(), mac.getZooKeepers()); Connector c = zk.getConnector("root", new PasswordToken("root")); final String tableName = "foo"; if (c.tableOperations().exists(tableName)) { c.tableOperations().delete(tableName); } c.tableOperations().create(tableName); BatchWriter bw = null; try { bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 1; i < 6; i++) { Mutation m = new Mutation(i + "_a"); m.put("a", "a", 0, ""); m.put("a", "b", 0, ""); bw.addMutation(m); m = new Mutation(i + "_b"); m.put("a", "a", 0, ""); bw.addMutation(m); m = new Mutation(i + "_c"); m.put("a", "a", 0, ""); m.put("b", "a", 0, ""); m.put("c", "a", 0, ""); bw.addMutation(m); } } finally { if (null != bw) { bw.close(); } } for (int i = 1; i < 6; i++) { Map<Key, Long> results = ImmutableMap.<Key, Long>builder().put(new Key(i + "_a", "a", "b", 0), 2l) .put(new Key(i + "_b", "a", "a", 0), 1l).put(new Key(i + "_c", "c", "a", 0), 3l).build(); BatchScanner bs = c.createBatchScanner(tableName, new Authorizations(), 1); bs.setRanges(Collections.singleton(Range.prefix(Integer.toString(i)))); try { IteratorSetting cfg = new IteratorSetting(30, GroupByRowSuffixIterator.class); bs.addScanIterator(cfg); long count = 0; for (Entry<Key, Value> entry : bs) { VLongWritable writable = GroupByRowSuffixIterator.getWritable(entry.getValue()); Assert.assertTrue("Results did not contain: " + entry.getKey(), results.containsKey(entry.getKey())); Assert.assertEquals(results.get(entry.getKey()).longValue(), writable.get()); count++; } Assert.assertEquals(results.size(), count); } finally { if (null != bs) { bs.close(); } } } } @Test public void testSingleRowSingleColumn() throws Exception { ZooKeeperInstance zk = new ZooKeeperInstance(mac.getInstanceName(), mac.getZooKeepers()); Connector c = zk.getConnector("root", new PasswordToken("root")); final String tableName = "foo"; if (c.tableOperations().exists(tableName)) { c.tableOperations().delete(tableName); } c.tableOperations().create(tableName); BatchWriter bw = null; try { bw = c.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("1_a"); m.put("col1", "1", 0, ""); m.put("col1", "2", 0, ""); bw.addMutation(m); m = new Mutation("1_b"); m.put("col1", "1", 0, ""); bw.addMutation(m); m = new Mutation("1_c"); m.put("col1", "1", 0, ""); m.put("col2", "2", 0, ""); m.put("col3", "1", 0, ""); bw.addMutation(m); } finally { if (null != bw) { bw.close(); } } Map<Key, Long> results = ImmutableMap.<Key, Long>builder().put(new Key("1_a", "col1", "2", 0), 2l) .put(new Key("1_b", "col1", "1", 0), 1l).put(new Key("1_c", "col1", "1", 0), 1l).build(); BatchScanner bs = c.createBatchScanner(tableName, new Authorizations(), 1); bs.setRanges(Collections.singleton(new Range())); bs.fetchColumnFamily(new Text("col1")); try { IteratorSetting cfg = new IteratorSetting(30, GroupByRowSuffixIterator.class); bs.addScanIterator(cfg); long count = 0; for (Entry<Key, Value> entry : bs) { VLongWritable writable = GroupByRowSuffixIterator.getWritable(entry.getValue()); Assert.assertTrue("Results did not contain: " + entry.getKey(), results.containsKey(entry.getKey())); Assert.assertEquals(results.get(entry.getKey()).longValue(), writable.get()); count++; } Assert.assertEquals(results.size(), count); } finally { if (null != bs) { bs.close(); } } } @Test public void testManyRowsSingleColumn() throws Exception { ZooKeeperInstance zk = new ZooKeeperInstance(mac.getInstanceName(), mac.getZooKeepers()); Connector c = zk.getConnector("root", new PasswordToken("root")); final String tableName = "foo"; if (c.tableOperations().exists(tableName)) { c.tableOperations().delete(tableName); } c.tableOperations().create(tableName); BatchWriter bw = null; try { bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 1; i < 6; i++) { Mutation m = new Mutation(i + "_a"); m.put("col1", "1", 0, ""); m.put("col1", "2", 0, ""); bw.addMutation(m); m = new Mutation(i + "_b"); m.put("col1", "1", 0, ""); bw.addMutation(m); m = new Mutation(i + "_c"); m.put("col1", "1", 0, ""); m.put("col2", "2", 0, ""); m.put("col3", "1", 0, ""); bw.addMutation(m); } } finally { if (null != bw) { bw.close(); } } for (int i = 1; i < 6; i++) { Map<Key, Long> results = ImmutableMap.<Key, Long>builder().put(new Key(i + "_a", "col1", "2", 0), 2l) .put(new Key(i + "_b", "col1", "1", 0), 1l).put(new Key(i + "_c", "col1", "1", 0), 1l).build(); BatchScanner bs = c.createBatchScanner(tableName, new Authorizations(), 1); bs.setRanges(Collections.singleton(Range.prefix(Integer.toString(i)))); bs.fetchColumnFamily(new Text("col1")); try { IteratorSetting cfg = new IteratorSetting(30, GroupByRowSuffixIterator.class); bs.addScanIterator(cfg); long count = 0; for (Entry<Key, Value> entry : bs) { VLongWritable writable = GroupByRowSuffixIterator.getWritable(entry.getValue()); Assert.assertTrue("Results did not contain: " + entry.getKey(), results.containsKey(entry.getKey())); Assert.assertEquals(results.get(entry.getKey()).longValue(), writable.get()); count++; } Assert.assertEquals(results.size(), count); } finally { if (null != bs) { bs.close(); } } } } @Test public void testReseek() throws Exception { TreeMap<Key, Value> data = Maps.newTreeMap(); data.put(new Key("foo\u0000bell", "RESTAURANT", "f\u00001"), new Value()); data.put(new Key("foo\u0000bell", "RESTAURANT", "f\u00002"), new Value()); data.put(new Key("foo\u0000taco", "RESTAURANT", "f\u00001"), new Value()); data.put(new Key("foo\u0000taco", "RESTAURANT", "f\u00002"), new Value()); data.put(new Key("foo\u0000taco", "RESTAURANT", "f\u00003"), new Value()); SortedMapIterator source = new SortedMapIterator(data); GroupByRowSuffixIterator iter = new GroupByRowSuffixIterator(); iter.init(source, Collections.<String, String>emptyMap(), new IteratorEnvironment() { @Override public SortedKeyValueIterator<Key, Value> reserveMapFileReader(final String mapFileName) throws IOException { return null; } @Override public AccumuloConfiguration getConfig() { return null; } @Override public IteratorScope getIteratorScope() { return null; } @Override public boolean isFullMajorCompaction() { return false; } @Override public void registerSideChannel(final SortedKeyValueIterator<Key, Value> iter) { } }); iter.seek(Range.prefix("foo"), Collections.<ByteSequence>emptySet(), false); Assert.assertTrue(iter.hasTop()); Assert.assertEquals(new Key("foo\u0000bell", "RESTAURANT", "f\u00002"), iter.getTopKey()); VLongWritable actual = new VLongWritable(), expected = new VLongWritable(2); actual.readFields(new DataInputStream(new ByteArrayInputStream(iter.getTopValue().get()))); Assert.assertEquals(expected, actual); iter.seek(new Range(new Key("foo\u0000bell", "RESTAURANT", "f\u00002"), false, new Key("fop"), false), Collections.<ByteSequence>emptySet(), false); Assert.assertTrue(iter.hasTop()); Assert.assertEquals(new Key("foo\u0000taco", "RESTAURANT", "f\u00003"), iter.getTopKey()); actual = new VLongWritable(); expected = new VLongWritable(3); actual.readFields(new DataInputStream(new ByteArrayInputStream(iter.getTopValue().get()))); Assert.assertEquals(expected, actual); iter.next(); Assert.assertFalse(iter.hasTop()); } }