Java tutorial
/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.routing; import com.linkedin.pinot.common.config.AbstractTableConfig; import com.linkedin.pinot.common.metadata.ZKMetadataProvider; import com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata; import com.linkedin.pinot.common.metrics.BrokerMetrics; import com.yammer.metrics.core.MetricsRegistry; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.I0Itec.zkclient.IZkDataListener; import org.apache.commons.configuration.BaseConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang.mutable.MutableBoolean; import org.apache.helix.ZNRecord; import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.model.ExternalView; import org.apache.helix.model.InstanceConfig; import org.apache.helix.store.zk.ZkHelixPropertyStore; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.testng.Assert; import org.testng.annotations.Test; import com.linkedin.pinot.common.response.ServerInstance; import com.linkedin.pinot.common.utils.CommonConstants; import com.linkedin.pinot.common.utils.HLCSegmentName; import com.linkedin.pinot.common.utils.LLCSegmentName; import com.linkedin.pinot.routing.builder.KafkaHighLevelConsumerBasedRoutingTableBuilder; import com.linkedin.pinot.routing.builder.RandomRoutingTableBuilder; import com.linkedin.pinot.routing.builder.RoutingTableBuilder; import com.linkedin.pinot.transport.common.SegmentIdSet; public class RoutingTableTest { public static final String ALL_PARTITIONS = "ALL"; private static Logger LOGGER = org.slf4j.LoggerFactory.getLogger(RoutingTableTest.class); private static RoutingTableSelector NO_LLC_ROUTING = new PercentageBasedRoutingTableSelector(); @Test public void testHelixExternalViewBasedRoutingTable() throws Exception { RoutingTableBuilder routingStrategy = new RandomRoutingTableBuilder(100); HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration()); routingTable.setSmallClusterRoutingTableBuilder(routingStrategy); ExternalView externalView = new ExternalView("testResource0_OFFLINE"); externalView.setState("segment0", "dataServer_instance_0", "ONLINE"); externalView.setState("segment0", "dataServer_instance_1", "ONLINE"); externalView.setState("segment1", "dataServer_instance_1", "ONLINE"); externalView.setState("segment1", "dataServer_instance_2", "ONLINE"); externalView.setState("segment2", "dataServer_instance_2", "ONLINE"); externalView.setState("segment2", "dataServer_instance_0", "ONLINE"); List<InstanceConfig> instanceConfigs = generateInstanceConfigs("dataServer_instance", 0, 2); routingTable.markDataResourceOnline("testResource0_OFFLINE", externalView, instanceConfigs); ExternalView externalView1 = new ExternalView("testResource1_OFFLINE"); externalView1.setState("segment10", "dataServer_instance_0", "ONLINE"); externalView1.setState("segment11", "dataServer_instance_1", "ONLINE"); externalView1.setState("segment12", "dataServer_instance_2", "ONLINE"); routingTable.markDataResourceOnline("testResource1_OFFLINE", externalView1, instanceConfigs); ExternalView externalView2 = new ExternalView("testResource2_OFFLINE"); externalView2.setState("segment20", "dataServer_instance_0", "ONLINE"); externalView2.setState("segment21", "dataServer_instance_0", "ONLINE"); externalView2.setState("segment22", "dataServer_instance_0", "ONLINE"); externalView2.setState("segment20", "dataServer_instance_1", "ONLINE"); externalView2.setState("segment21", "dataServer_instance_1", "ONLINE"); externalView2.setState("segment22", "dataServer_instance_1", "ONLINE"); externalView2.setState("segment20", "dataServer_instance_2", "ONLINE"); externalView2.setState("segment21", "dataServer_instance_2", "ONLINE"); externalView2.setState("segment22", "dataServer_instance_2", "ONLINE"); routingTable.markDataResourceOnline("testResource2_OFFLINE", externalView2, instanceConfigs); for (int numRun = 0; numRun < 100; ++numRun) { assertResourceRequest(routingTable, "testResource0_OFFLINE", "[segment0, segment1, segment2]", 3); } for (int numRun = 0; numRun < 100; ++numRun) { assertResourceRequest(routingTable, "testResource1_OFFLINE", "[segment10, segment11, segment12]", 3); } for (int numRun = 0; numRun < 100; ++numRun) { assertResourceRequest(routingTable, "testResource2_OFFLINE", "[segment20, segment21, segment22]", 3); } } @Test public void testTimeBoundaryRegression() throws Exception { final FakePropertyStore propertyStore = new FakePropertyStore(); final OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata(); offlineSegmentZKMetadata.setTimeUnit(TimeUnit.DAYS); offlineSegmentZKMetadata.setEndTime(1234L); propertyStore.setContents( ZKMetadataProvider.constructPropertyStorePathForSegment("myTable_OFFLINE", "someSegment_0"), offlineSegmentZKMetadata.toZNRecord()); final ExternalView offlineExternalView = new ExternalView("myTable_OFFLINE"); offlineExternalView.setState("someSegment_0", "Server_1.2.3.4_1234", "ONLINE"); final MutableBoolean timeBoundaryUpdated = new MutableBoolean(false); HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(propertyStore, NO_LLC_ROUTING, null, new BaseConfiguration()) { @Override protected ExternalView fetchExternalView(String table) { return offlineExternalView; } @Override protected void updateTimeBoundary(String tableName, ExternalView externalView) { if (tableName.equals("myTable_OFFLINE")) { timeBoundaryUpdated.setValue(true); } } }; routingTable.setBrokerMetrics(new BrokerMetrics(new MetricsRegistry())); Assert.assertFalse(timeBoundaryUpdated.booleanValue()); final ArrayList<InstanceConfig> instanceConfigList = new ArrayList<>(); instanceConfigList.add(new InstanceConfig("Server_1.2.3.4_1234")); routingTable.markDataResourceOnline("myTable_OFFLINE", offlineExternalView, instanceConfigList); routingTable.markDataResourceOnline("myTable_REALTIME", new ExternalView("myTable_REALTIME"), null); Assert.assertTrue(timeBoundaryUpdated.booleanValue()); } private void assertResourceRequest(HelixExternalViewBasedRouting routingTable, String resource, String expectedSegmentList, int expectedNumSegment) { RoutingTableLookupRequest request = new RoutingTableLookupRequest(resource, Collections.<String>emptyList()); Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request); List<String> selectedSegments = new ArrayList<String>(); for (ServerInstance serverInstance : serversMap.keySet()) { LOGGER.trace(serverInstance.toString()); SegmentIdSet segmentIdSet = serversMap.get(serverInstance); LOGGER.trace(segmentIdSet.toString()); selectedSegments.addAll(segmentIdSet.getSegmentsNameList()); } String[] selectedSegmentArray = selectedSegments.toArray(new String[0]); Arrays.sort(selectedSegmentArray); Assert.assertEquals(selectedSegments.size(), expectedNumSegment); Assert.assertEquals(Arrays.toString(selectedSegmentArray), expectedSegmentList); LOGGER.trace("********************************"); } @Test public void testKafkaHighLevelConsumerBasedRoutingTable() throws Exception { RoutingTableBuilder routingStrategy = new KafkaHighLevelConsumerBasedRoutingTableBuilder(); final String group0 = "testResource0_REALTIME_1433316466991_0"; final String group1 = "testResource1_REALTIME_1433316490099_1"; final String group2 = "testResource2_REALTIME_1436589344583_1"; final LLCSegmentName llcSegmentName = new LLCSegmentName("testResource0", 2, 65, System.currentTimeMillis()); HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration()); Field realtimeRTBField = HelixExternalViewBasedRouting.class .getDeclaredField("_realtimeHLCRoutingTableBuilder"); realtimeRTBField.setAccessible(true); realtimeRTBField.set(routingTable, routingStrategy); ExternalView externalView = new ExternalView("testResource0_REALTIME"); // Toss in an llc segment in the mix. Should not affect the results externalView.setState(llcSegmentName.getSegmentName(), "dataServer_instance_0", "CONSUMING"); externalView.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "0").getSegmentName(), "dataServer_instance_0", "ONLINE"); externalView.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "1").getSegmentName(), "dataServer_instance_1", "ONLINE"); externalView.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "2").getSegmentName(), "dataServer_instance_2", "ONLINE"); externalView.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "3").getSegmentName(), "dataServer_instance_3", "ONLINE"); externalView.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "4").getSegmentName(), "dataServer_instance_4", "ONLINE"); externalView.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "5").getSegmentName(), "dataServer_instance_5", "ONLINE"); routingTable.markDataResourceOnline("testResource0_REALTIME", externalView, generateInstanceConfigs("dataServer_instance", 0, 5)); ExternalView externalView1 = new ExternalView("testResource1_REALTIME"); externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "10").getSegmentName(), "dataServer_instance_10", "ONLINE"); externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "11").getSegmentName(), "dataServer_instance_11", "ONLINE"); externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "12").getSegmentName(), "dataServer_instance_12", "ONLINE"); routingTable.markDataResourceOnline("testResource1_REALTIME", externalView1, generateInstanceConfigs("dataServer_instance", 10, 12)); ExternalView externalView2 = new ExternalView("testResource2_REALTIME"); externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "20").getSegmentName(), "dataServer_instance_20", "ONLINE"); externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "21").getSegmentName(), "dataServer_instance_21", "ONLINE"); externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "22").getSegmentName(), "dataServer_instance_22", "ONLINE"); externalView2.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "23").getSegmentName(), "dataServer_instance_23", "ONLINE"); externalView2.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "24").getSegmentName(), "dataServer_instance_24", "ONLINE"); externalView2.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "25").getSegmentName(), "dataServer_instance_25", "ONLINE"); externalView2.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "26").getSegmentName(), "dataServer_instance_26", "ONLINE"); externalView2.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "27").getSegmentName(), "dataServer_instance_27", "ONLINE"); externalView2.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "28").getSegmentName(), "dataServer_instance_28", "ONLINE"); routingTable.markDataResourceOnline("testResource2_REALTIME", externalView2, generateInstanceConfigs("dataServer_instance", 20, 28)); for (int numRun = 0; numRun < 100; ++numRun) { assertResourceRequest(routingTable, "testResource0_REALTIME", new String[] { "[" + new HLCSegmentName(group0, ALL_PARTITIONS, "0").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "1").getSegmentName() + "]", "[" + new HLCSegmentName(group1, ALL_PARTITIONS, "2").getSegmentName() + ", " + new HLCSegmentName(group1, ALL_PARTITIONS, "3").getSegmentName() + "]", "[" + new HLCSegmentName(group2, ALL_PARTITIONS, "4").getSegmentName() + ", " + new HLCSegmentName(group2, ALL_PARTITIONS, "5").getSegmentName() + "]" }, 2); } for (int numRun = 0; numRun < 100; ++numRun) { assertResourceRequest(routingTable, "testResource1_REALTIME", new String[] { "[" + new HLCSegmentName(group0, ALL_PARTITIONS, "10").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "11").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "12").getSegmentName() + "]" }, 3); } for (int numRun = 0; numRun < 100; ++numRun) { assertResourceRequest(routingTable, "testResource2_REALTIME", new String[] { "[" + new HLCSegmentName(group0, ALL_PARTITIONS, "20").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "21").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "22").getSegmentName() + "]", "[" + new HLCSegmentName(group1, ALL_PARTITIONS, "23").getSegmentName() + ", " + new HLCSegmentName(group1, ALL_PARTITIONS, "24").getSegmentName() + ", " + new HLCSegmentName(group1, ALL_PARTITIONS, "25").getSegmentName() + "]", "[" + new HLCSegmentName(group2, ALL_PARTITIONS, "26").getSegmentName() + ", " + new HLCSegmentName(group2, ALL_PARTITIONS, "27").getSegmentName() + ", " + new HLCSegmentName(group2, ALL_PARTITIONS, "28").getSegmentName() + "]" }, 3); } } private void assertResourceRequest(HelixExternalViewBasedRouting routingTable, String resource, String[] expectedSegmentLists, int expectedNumSegment) { RoutingTableLookupRequest request = new RoutingTableLookupRequest(resource, Collections.<String>emptyList()); Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request); List<String> selectedSegments = new ArrayList<String>(); for (ServerInstance serverInstance : serversMap.keySet()) { LOGGER.trace(serverInstance.toString()); SegmentIdSet segmentIdSet = serversMap.get(serverInstance); LOGGER.trace(segmentIdSet.toString()); selectedSegments.addAll(segmentIdSet.getSegmentsNameList()); } String[] selectedSegmentArray = selectedSegments.toArray(new String[0]); Arrays.sort(selectedSegmentArray); Assert.assertEquals(selectedSegments.size(), expectedNumSegment); boolean matchedExpectedLists = false; for (String expectedSegmentList : expectedSegmentLists) { if (expectedSegmentList.equals(Arrays.toString(selectedSegmentArray))) { matchedExpectedLists = true; } } Assert.assertTrue(matchedExpectedLists); LOGGER.trace("********************************"); } // Test that we can switch between llc and hlc routing depending on what the selector tells us. @Test public void testCombinedKafkaRouting() throws Exception { HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration()); final long now = System.currentTimeMillis(); final String tableName = "table"; final String resourceName = tableName + "_REALTIME"; final String group1 = resourceName + "_" + Long.toString(now) + "_0"; final String group2 = resourceName + "_" + Long.toString(now) + "_1"; final String online = "ONLINE"; final String consuming = "CONSUMING"; final int partitionId = 1; final String partitionRange = "JUNK"; final int segId1 = 1; final int segId2 = 2; final int port1 = 1; final int port2 = 2; final String host = "host"; final ServerInstance serverInstance1 = new ServerInstance(host, port1); final ServerInstance serverInstance2 = new ServerInstance(host, port2); final String helixInstance1 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance1; final String helixInstance2 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance2; final HLCSegmentName s1HlcSegment1 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId1)); final HLCSegmentName s1HlcSegment2 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId2)); final HLCSegmentName s2HlcSegment1 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId1)); final HLCSegmentName s2HlcSegment2 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId2)); final LLCSegmentName llcSegment1 = new LLCSegmentName(tableName, partitionId, segId1, now); final LLCSegmentName llcSegment2 = new LLCSegmentName(tableName, partitionId, segId2, now); final List<InstanceConfig> instanceConfigs = new ArrayList<>(2); instanceConfigs.add(new InstanceConfig(helixInstance1)); instanceConfigs.add(new InstanceConfig(helixInstance2)); ExternalView ev = new ExternalView(resourceName); ev.setState(s1HlcSegment1.getSegmentName(), helixInstance1, online); ev.setState(s1HlcSegment2.getSegmentName(), helixInstance1, online); ev.setState(llcSegment1.getSegmentName(), helixInstance2, online); ev.setState(llcSegment2.getSegmentName(), helixInstance2, consuming); routingTable.markDataResourceOnline(resourceName, ev, instanceConfigs); RoutingTableLookupRequest request = new RoutingTableLookupRequest(resourceName, Collections.<String>emptyList()); for (int i = 0; i < 100; i++) { Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request); Assert.assertEquals(routingMap.size(), 1); List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName())); } // Now change the percent value in the routing table selector to be 100, and we should get only LLC segments. Configuration configuration = new PropertiesConfiguration(); configuration.addProperty("class", PercentageBasedRoutingTableSelector.class.getName()); configuration.addProperty("table." + resourceName, new Integer(100)); RoutingTableSelector selector = RoutingTableSelectorFactory.getRoutingTableSelector(configuration, null); selector.init(configuration, null); Field selectorField = HelixExternalViewBasedRouting.class.getDeclaredField("_routingTableSelector"); selectorField.setAccessible(true); selectorField.set(routingTable, selector); // And we should find only LLC segments. for (int i = 0; i < 100; i++) { Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request); Assert.assertEquals(routingMap.size(), 1); List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(llcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(llcSegment2.getSegmentName())); } // Now change it to 50, and we should find both (at least 10 times each). configuration = new PropertiesConfiguration(); configuration.addProperty("table." + resourceName, new Integer(50)); selector = new PercentageBasedRoutingTableSelector(); selector.init(configuration, null); selectorField.set(routingTable, selector); int hlc = 0; int llc = 0; for (int i = 0; i < 100; i++) { Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request); Assert.assertEquals(routingMap.size(), 1); if (routingMap.containsKey(serverInstance2)) { List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(llcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(llcSegment2.getSegmentName())); llc++; } else { List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName())); hlc++; } } // If we do the above iteration 100 times, we should get at least 10 of each type of routing. // If this test fails Assert.assertTrue(hlc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc); Assert.assertTrue(llc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc); // Check that force HLC works request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_HLC")); hlc = 0; llc = 0; for (int i = 0; i < 100; i++) { Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request); Assert.assertEquals(routingMap.size(), 1); if (routingMap.containsKey(serverInstance2)) { List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(llcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(llcSegment2.getSegmentName())); llc++; } else { List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName())); hlc++; } } Assert.assertEquals(hlc, 100); Assert.assertEquals(llc, 0); // Check that force LLC works request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_LLC")); hlc = 0; llc = 0; for (int i = 0; i < 100; i++) { Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request); Assert.assertEquals(routingMap.size(), 1); if (routingMap.containsKey(serverInstance2)) { List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(llcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(llcSegment2.getSegmentName())); llc++; } else { List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList(); Assert.assertEquals(segments.size(), 2); Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName())); Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName())); hlc++; } } Assert.assertEquals(hlc, 0); Assert.assertEquals(llc, 100); } /** * Helper method to generate instance config lists. Instance names are generated as prefix_i, where * i ranges from start to end. * * @param prefix Instance name prefix * @param start Start index * @param end End index * @return */ private List<InstanceConfig> generateInstanceConfigs(String prefix, int start, int end) { List<InstanceConfig> configs = new ArrayList<>(); for (int i = start; i <= end; ++i) { String instance = prefix + "_" + i; configs.add(new InstanceConfig(instance)); } return configs; } class FakePropertyStore extends ZkHelixPropertyStore<ZNRecord> { private Map<String, ZNRecord> _contents = new HashMap<>(); private IZkDataListener _listener = null; public FakePropertyStore() { super((ZkBaseDataAccessor<ZNRecord>) null, null, null); } @Override public ZNRecord get(String path, Stat stat, int options) { return _contents.get(path); } @Override public void subscribeDataChanges(String path, IZkDataListener listener) { _listener = listener; } public void setContents(String path, ZNRecord contents) throws Exception { _contents.put(path, contents); if (_listener != null) { _listener.handleDataChange(path, contents); } } @Override public void start() { // Don't try to connect to zk } } @Test public void testTableConfigRoutingTableSelector() throws Exception { FakePropertyStore fakePropertyStore = new FakePropertyStore(); TableConfigRoutingTableSelector tableConfigRoutingTableSelector = new TableConfigRoutingTableSelector(); tableConfigRoutingTableSelector.init(null, fakePropertyStore); String propertyStoreEntry = "{\n" + " \"tableName\":\"fakeTable\",\n" + " \"segmentsConfig\": {\n" + " \"retentionTimeUnit\":\"DAYS\",\n" + " \"retentionTimeValue\":\"5\",\n" + " \"segmentPushFrequency\":\"daily\",\n" + " \"segmentPushType\":\"APPEND\",\n" + " \"replication\": \"2\",\n" + " \"schemaName\": \"fakeSchema\",\n" + " \"timeColumnName\": \"time\",\n" + " \"timeType\": \"MINUTES\",\n" + " \"segmentAssignmentStrategy\": \"BalanceNumSegmentAssignmentStrategy\"\n" + " },\n" + " \"tableIndexConfig\": {\n" + " \"invertedIndexColumns\": [\"columnA\",\"columnB\"],\n" + " \"loadMode\": \"MMAP\",\n" + " \"lazyLoad\": \"false\",\n" + " \"streamConfigs\": {\n" + " \"streamType\": \"kafka\",\n" + " \"stream.kafka.consumer.type\": \"highLevel,simple\",\n" + " \"stream.kafka.topic.name\": \"FakeTopic\",\n" + " \"stream.kafka.decoder.class.name\": \"com.linkedin.pinot.core.realtime.impl.kafka.KafkaAvroMessageDecoder\",\n" + "\n" + " \"stream.kafka.zk.broker.url\": \"fakezk:1234\",\n" + " \"stream.kafka.hlc.zk.connect.string\": \"fakezk:1234\",\n" + " \"stream.kafka.decoder.prop.schema.registry.rest.url\": \"fakeschemaregistry:1234\",\n" + " \"stream.kafka.decoder.prop.schema.registry.schema.name\": \"FakeSchema\"\n" + " }\n" + " },\n" + " \"tenants\": {\n" + " \"broker\":\"fakebroker\",\n" + " \"server\":\"fakeserver\"\n" + " },\n" + " \"tableType\":\"REALTIME\",\n" + " \"metadata\": {\n" + " \"customConfigs\": {\n" + " \"routing.llc.percentage\": \"50.0\"\n" + " }\n" + " }\n" + " }\n" + "}"; AbstractTableConfig tableConfig = AbstractTableConfig.init(propertyStoreEntry); fakePropertyStore.setContents("/CONFIGS/TABLE/fakeTable_REALTIME", AbstractTableConfig.toZnRecord(tableConfig)); tableConfigRoutingTableSelector.registerTable("fakeTable_REALTIME"); int llcCount = 0; for (int i = 0; i < 10000; ++i) { if (tableConfigRoutingTableSelector.shouldUseLLCRouting("fakeTable_REALTIME")) { llcCount++; } } Assert.assertTrue(4500 <= llcCount && llcCount <= 5500, "Expected approximately 50% probability of picking LLC, got " + llcCount / 100.0 + " %"); tableConfig.getCustomConfigs().setCustomConfigs(Collections.singletonMap("routing.llc.percentage", "0")); fakePropertyStore.setContents("/CONFIGS/TABLE/fakeTable_REALTIME", AbstractTableConfig.toZnRecord(tableConfig)); llcCount = 0; for (int i = 0; i < 10000; ++i) { if (tableConfigRoutingTableSelector.shouldUseLLCRouting("fakeTable_REALTIME")) { llcCount++; } } Assert.assertEquals(llcCount, 0, "Expected 0% probability of picking LLC, got " + llcCount / 100.0 + " %"); tableConfig.getCustomConfigs().setCustomConfigs(Collections.singletonMap("routing.llc.percentage", "100")); fakePropertyStore.setContents("/CONFIGS/TABLE/fakeTable_REALTIME", AbstractTableConfig.toZnRecord(tableConfig)); llcCount = 0; for (int i = 0; i < 10000; ++i) { if (tableConfigRoutingTableSelector.shouldUseLLCRouting("fakeTable_REALTIME")) { llcCount++; } } Assert.assertEquals(llcCount, 10000, "Expected 100% probability of picking LLC, got " + llcCount / 100.0 + " %"); } }