List of usage examples for com.google.common.collect MinMaxPriorityQueue addAll
@Override
public boolean addAll(Collection<? extends E> newElements)
From source file:com.linkedin.pinot.controller.helix.core.relocation.RealtimeSegmentRelocator.java
/** * Given a realtime tag config and an ideal state, relocate the segments * which are completed but still hanging around on consuming servers, one replica at a time * @param realtimeTagConfig/*from w w w.j a va 2s . c o m*/ * @param idealState */ protected void relocateSegments(RealtimeTagConfig realtimeTagConfig, IdealState idealState) { List<String> consumingServers = _helixAdmin.getInstancesInClusterWithTag(_helixManager.getClusterName(), realtimeTagConfig.getConsumingServerTag()); if (consumingServers.isEmpty()) { throw new IllegalStateException( "Found no realtime consuming servers with tag " + realtimeTagConfig.getConsumingServerTag()); } List<String> completedServers = _helixAdmin.getInstancesInClusterWithTag(_helixManager.getClusterName(), realtimeTagConfig.getCompletedServerTag()); if (completedServers.isEmpty()) { throw new IllegalStateException( "Found no realtime completed servers with tag " + realtimeTagConfig.getCompletedServerTag()); } if (completedServers.size() < Integer.valueOf(idealState.getReplicas())) { throw new IllegalStateException("Number of completed servers: " + completedServers.size() + " is less than num replicas: " + idealState.getReplicas()); } // TODO: use segment assignment strategy to decide where to place relocated segment // create map of completed server name to num segments it holds. // This will help us decide which completed server to choose for replacing a consuming server Map<String, Integer> completedServerToNumSegments = new HashMap<>(completedServers.size()); for (String server : completedServers) { completedServerToNumSegments.put(server, 0); } for (String segmentName : idealState.getPartitionSet()) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentName); for (String instance : instanceStateMap.keySet()) { if (completedServers.contains(instance)) { completedServerToNumSegments.put(instance, completedServerToNumSegments.get(instance) + 1); } } } MinMaxPriorityQueue<Map.Entry<String, Integer>> completedServersQueue = MinMaxPriorityQueue .orderedBy(new Comparator<Map.Entry<String, Integer>>() { @Override public int compare(Map.Entry<String, Integer> entry1, Map.Entry<String, Integer> entry2) { return Integer.compare(entry1.getValue(), entry2.getValue()); } }).maximumSize(completedServers.size()).create(); completedServersQueue.addAll(completedServerToNumSegments.entrySet()); // get new mapping for segments that need relocation createNewIdealState(idealState, consumingServers, completedServersQueue); }
From source file:com.davidbracewell.ml.sequence.decoder.LinearViterbi.java
@Override public double[] decode(SequenceModel<V> raw, Sequence<V> sequence) { LinearSequenceModel<V> model = Val.of(raw).cast(); final Feature classFeature = model.getTargetFeature(); final int numStates = classFeature.alphabetSize(); MinMaxPriorityQueue<State> beam = MinMaxPriorityQueue.maximumSize(beamSize).create(); ClassificationResult result = model.classifyItem(0, sequence, new double[0]); for (int ci = 0; ci < numStates; ci++) { if (isValidStartTag(classFeature.valueAtIndex(ci)) && isValidTag(classFeature.valueAtIndex(ci), sequence.getData(0))) { beam.add(new State(Math.log(result.getConfidence(ci)), ci, null, 0)); }/*from w w w. j ava 2s . c om*/ } MinMaxPriorityQueue<State> tempBeam = MinMaxPriorityQueue.maximumSize(beamSize).create(); for (int i = 1; i < sequence.length(); i++) { while (!beam.isEmpty()) { // go through all the previous states State state = beam.removeFirst(); String previousTag = classFeature.valueAtIndex(state.tag); result = model.classifyItem(i, sequence, state.labels()); for (int ci = 0; ci < numStates; ci++) { if (isValidTransition(previousTag, classFeature.valueAtIndex(ci)) && ((i + 1 < sequence.length()) || isValidEndTag(classFeature.valueAtIndex(ci))) && isValidTag(classFeature.valueAtIndex(ci), sequence.getData(i))) { tempBeam.add( new State(state.probability + Math.log(result.getConfidence(ci)), ci, state, i)); } } } beam.addAll(tempBeam); tempBeam.clear(); } return beam.remove().labels(); }
From source file:com.metamx.druid.master.rules.LoadRule.java
private MasterStats drop(int expectedReplicants, int clusterReplicants, final DataSegment segment, final DruidMasterRuntimeParams params) { MasterStats stats = new MasterStats(); final ReplicationThrottler replicationManager = params.getReplicationManager(); if (!params.hasDeletionWaitTimeElapsed()) { return stats; }//from ww w .ja v a 2s .c o m // Make sure we have enough actual replicants in the cluster before doing anything if (clusterReplicants < expectedReplicants) { return stats; } Map<String, Integer> replicantsByType = params.getSegmentReplicantLookup() .getClusterTiers(segment.getIdentifier()); for (Map.Entry<String, Integer> entry : replicantsByType.entrySet()) { String tier = entry.getKey(); int actualNumReplicantsForType = entry.getValue(); int expectedNumReplicantsForType = getReplicants(tier); MinMaxPriorityQueue<ServerHolder> serverQueue = params.getDruidCluster().get(tier); if (serverQueue == null) { log.makeAlert("No holders found for tier[%s]", entry.getKey()).emit(); return stats; } List<ServerHolder> droppedServers = Lists.newArrayList(); while (actualNumReplicantsForType > expectedNumReplicantsForType) { final ServerHolder holder = serverQueue.pollLast(); if (holder == null) { log.warn("Wtf, holder was null? I have no servers serving [%s]?", segment.getIdentifier()); break; } if (holder.isServingSegment(segment)) { if (expectedNumReplicantsForType > 0) { // don't throttle unless we are removing extra replicants if (!replicationManager.canDestroyReplicant(getTier())) { serverQueue.add(holder); break; } replicationManager.registerReplicantTermination(getTier(), segment.getIdentifier(), holder.getServer().getHost()); } holder.getPeon().dropSegment(segment, new LoadPeonCallback() { @Override protected void execute() { replicationManager.unregisterReplicantTermination(getTier(), segment.getIdentifier(), holder.getServer().getHost()); } }); --actualNumReplicantsForType; stats.addToTieredStat("droppedCount", tier, 1); } droppedServers.add(holder); } serverQueue.addAll(droppedServers); } return stats; }
From source file:io.druid.server.coordinator.rules.LoadRule.java
private CoordinatorStats drop(final Map<String, Integer> loadStatus, final DataSegment segment, final DruidCoordinatorRuntimeParams params) { CoordinatorStats stats = new CoordinatorStats(); // Make sure we have enough loaded replicants in the correct tiers in the cluster before doing anything for (Integer leftToLoad : loadStatus.values()) { if (leftToLoad > 0) { return stats; }/*from w w w. j av a 2s.com*/ } final ReplicationThrottler replicationManager = params.getReplicationManager(); // Find all instances of this segment across tiers Map<String, Integer> replicantsByTier = params.getSegmentReplicantLookup() .getClusterTiers(segment.getIdentifier()); for (Map.Entry<String, Integer> entry : replicantsByTier.entrySet()) { final String tier = entry.getKey(); int loadedNumReplicantsForTier = entry.getValue(); int expectedNumReplicantsForTier = getNumReplicants(tier); stats.addToTieredStat(droppedCount, tier, 0); MinMaxPriorityQueue<ServerHolder> serverQueue = params.getDruidCluster().get(tier); if (serverQueue == null) { log.makeAlert("No holders found for tier[%s]", entry.getKey()).emit(); return stats; } List<ServerHolder> droppedServers = Lists.newArrayList(); while (loadedNumReplicantsForTier > expectedNumReplicantsForTier) { final ServerHolder holder = serverQueue.pollLast(); if (holder == null) { log.warn("Wtf, holder was null? I have no servers serving [%s]?", segment.getIdentifier()); break; } if (holder.isServingSegment(segment)) { if (expectedNumReplicantsForTier > 0) { // don't throttle unless we are removing extra replicants if (!replicationManager.canDestroyReplicant(tier)) { serverQueue.add(holder); break; } replicationManager.registerReplicantTermination(tier, segment.getIdentifier(), holder.getServer().getHost()); } holder.getPeon().dropSegment(segment, new LoadPeonCallback() { @Override public void execute() { replicationManager.unregisterReplicantTermination(tier, segment.getIdentifier(), holder.getServer().getHost()); } }); --loadedNumReplicantsForTier; stats.addToTieredStat(droppedCount, tier, 1); } droppedServers.add(holder); } serverQueue.addAll(droppedServers); } return stats; }
From source file:com.linkedin.pinot.controller.helix.core.relocation.RealtimeSegmentRelocator.java
/** * Given the instanceStateMap and a list of consuming and completed servers for a realtime resource, * creates a new instanceStateMap, where one replica's instance is replaced from a consuming server to a completed server * @param instanceStateMap//from w ww. j a v a2 s.c o m * @param consumingServers * @param completedServersQueue * @return */ private Map<String, String> createNewInstanceStateMap(Map<String, String> instanceStateMap, List<String> consumingServers, MinMaxPriorityQueue<Map.Entry<String, Integer>> completedServersQueue) { Map<String, String> newInstanceStateMap = null; // proceed only if all segments are ONLINE, and at least 1 server is from consuming list for (String state : instanceStateMap.values()) { if (!state.equals(PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE)) { return newInstanceStateMap; } } for (String instance : instanceStateMap.keySet()) { if (consumingServers.contains(instance)) { // Decide best strategy to pick completed server. // 1. pick random from list of completed servers // 2. pick completed server with minimum segments, based on ideal state of this resource // 3. pick completed server with minimum segment, based on ideal state of all resources in this tenant // 4. use SegmentAssignmentStrategy // TODO: Using 2 for now. We should use 4. However the current interface and implementations cannot be used as is. // We should refactor the SegmentAssignmentStrategy interface suitably and reuse it here Map.Entry<String, Integer> chosenServer = null; List<Map.Entry<String, Integer>> polledServers = new ArrayList<>(1); while (!completedServersQueue.isEmpty()) { Map.Entry<String, Integer> server = completedServersQueue.pollFirst(); if (instanceStateMap.keySet().contains(server.getKey())) { polledServers.add(server); } else { chosenServer = server; break; } } completedServersQueue.addAll(polledServers); if (chosenServer == null) { throw new IllegalStateException("Could not find server to relocate segment"); } newInstanceStateMap = new HashMap<>(instanceStateMap.size()); newInstanceStateMap.putAll(instanceStateMap); newInstanceStateMap.remove(instance); newInstanceStateMap.put(chosenServer.getKey(), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE); chosenServer.setValue(chosenServer.getValue() + 1); completedServersQueue.add(chosenServer); break; } } return newInstanceStateMap; }
From source file:it.uniroma3.mat.extendedset.intset.ImmutableConciseSet.java
private static ImmutableConciseSet doUnion(Iterator<ImmutableConciseSet> sets) { IntList retVal = new IntList(); // lhs = current word position, rhs = the iterator // Comparison is first by index, then one fills > literals > zero fills // one fills are sorted by length (longer one fills have priority) // similarily, shorter zero fills have priority MinMaxPriorityQueue<WordHolder> theQ = MinMaxPriorityQueue.orderedBy(new Comparator<WordHolder>() { @Override//from ww w . java 2 s . co m public int compare(WordHolder h1, WordHolder h2) { int w1 = h1.getWord(); int w2 = h2.getWord(); int s1 = h1.getIterator().startIndex; int s2 = h2.getIterator().startIndex; if (s1 != s2) { return compareInts(s1, s2); } if (ConciseSetUtils.isOneSequence(w1)) { if (ConciseSetUtils.isOneSequence(w2)) { return -compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } return -1; } else if (ConciseSetUtils.isLiteral(w1)) { if (ConciseSetUtils.isOneSequence(w2)) { return 1; } else if (ConciseSetUtils.isLiteral(w2)) { return 0; } return -1; } else { if (!ConciseSetUtils.isZeroSequence(w2)) { return 1; } return compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } } }).create(); // populate priority queue while (sets.hasNext()) { ImmutableConciseSet set = sets.next(); if (set != null && !set.isEmpty()) { WordIterator itr = set.newWordIterator(); theQ.add(new WordHolder(itr.next(), itr)); } } int currIndex = 0; while (!theQ.isEmpty()) { // create a temp list to hold everything that will get pushed back into the priority queue after each run List<WordHolder> wordsToAdd = Lists.newArrayList(); // grab the top element from the priority queue WordHolder curr = theQ.poll(); int word = curr.getWord(); WordIterator itr = curr.getIterator(); // if the next word in the queue starts at a different point than where we ended off we need to create a zero gap // to fill the space if (currIndex < itr.startIndex) { addAndCompact(retVal, itr.startIndex - currIndex - 1); currIndex = itr.startIndex; } if (ConciseSetUtils.isOneSequence(word)) { // extract a literal from the flip bits of the one sequence int flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(word); // advance everything past the longest ones sequence WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex < itr.wordsWalked) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); if (i.startIndex == itr.startIndex) { // if a literal was created from a flip bit, OR it with other literals or literals from flip bits in the same // position if (ConciseSetUtils.isOneSequence(w)) { flipBitLiteral |= ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); } else if (ConciseSetUtils.isLiteral(w)) { flipBitLiteral |= w; } else { flipBitLiteral |= ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); } } i.advanceTo(itr.wordsWalked); if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } nextVal = theQ.peek(); } // advance longest one literal forward and push result back to priority queue // if a flip bit is still needed, put it in the correct position int newWord = word & 0xC1FFFFFF; if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { flipBitLiteral ^= ConciseSetUtils.ALL_ONES_LITERAL; int position = Integer.numberOfTrailingZeros(flipBitLiteral) + 1; newWord |= (position << 25); } addAndCompact(retVal, newWord); currIndex = itr.wordsWalked; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } } else if (ConciseSetUtils.isLiteral(word)) { // advance all other literals WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); // if we still have zero fills with flipped bits, OR them here if (ConciseSetUtils.isLiteral(w)) { word |= w; } else { int flipBitLiteral = ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ZEROS_LITERAL) { word |= flipBitLiteral; i.advanceTo(itr.wordsWalked); } } if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } nextVal = theQ.peek(); } // advance the set with the current literal forward and push result back to priority queue addAndCompact(retVal, word); currIndex++; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } } else { // zero fills int flipBitLiteral; WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { // check if literal can be created flip bits of other zero sequences WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); flipBitLiteral = ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ZEROS_LITERAL) { wordsToAdd.add(new WordHolder(flipBitLiteral, i)); } else if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } nextVal = theQ.peek(); } // check if a literal needs to be created from the flipped bits of this sequence flipBitLiteral = ConciseSetUtils.getLiteralFromZeroSeqFlipBit(word); if (flipBitLiteral != ConciseSetUtils.ALL_ZEROS_LITERAL) { wordsToAdd.add(new WordHolder(flipBitLiteral, itr)); } else if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } } theQ.addAll(wordsToAdd); } if (retVal.isEmpty()) { return new ImmutableConciseSet(); } return new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray())); }
From source file:it.uniroma3.mat.extendedset.intset.ImmutableSecompaxSet.java
private static ImmutableSecompaxSet doUnion(Iterator<ImmutableSecompaxSet> iterator) { IntList retVal = new IntList(); // lhs = current word position, rhs = the iterator // Comparison is first by index, then one fills > literals > zero fills // one fills are sorted by length (longer one fills have priority) // similarily, shorter zero fills have priority MinMaxPriorityQueue<WordHolder> theQ = MinMaxPriorityQueue.orderedBy(new Comparator<WordHolder>() { @Override//from www. j a v a 2 s. co m public int compare(WordHolder h1, WordHolder h2) { int w1 = h1.getWord(); int w2 = h2.getWord(); int s1 = h1.getIterator().startIndex; int s2 = h2.getIterator().startIndex; if (s1 != s2) { return compareInts(s1, s2); } if (ConciseSetUtils.is1_fill(w1)) { if (ConciseSetUtils.is1_fill(w2)) { return -compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } return -1; } else if (ConciseSetUtils.isLiteral(w1)) { if (ConciseSetUtils.is1_fill(w2)) { return 1; } else if (ConciseSetUtils.isLiteral(w2)) { return 0; } return -1; } else { if (!ConciseSetUtils.is0_fill(w2)) { return 1; } return compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } } }).create(); // populate priority queue while (iterator.hasNext()) { ImmutableSecompaxSet set = iterator.next(); if (set != null && !set.isEmpty()) { WordIterator itr = set.newWordIterator(); theQ.add(new WordHolder(itr.next(), itr)); } } int currIndex = 0; while (!theQ.isEmpty()) { // create a temp list to hold everything that will get pushed back into the priority queue after each run List<WordHolder> wordsToAdd = Lists.newArrayList(); // grab the top element from the priority queue WordHolder curr = theQ.poll(); int word = curr.getWord(); WordIterator itr = curr.getIterator(); // if the next word in the queue starts at a different point than where we ended off we need to create a zero gap // to fill the space if (currIndex < itr.startIndex) { addAndCompact(retVal, itr.startIndex - currIndex); currIndex = itr.startIndex; } if (ConciseSetUtils.is1_fill(word)) { // extract a literal from the flip bits of the one sequence //int flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(word); // advance everything past the longest ones sequence WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex < itr.wordsWalked) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); /*if (i.startIndex == itr.startIndex) { // if a literal was created from a flip bit, OR it with other literals or literals from flip bits in the same // position if (ConciseSetUtils.isOneSequence(w)) { flipBitLiteral |= ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); } else if (ConciseSetUtils.isLiteral(w)) { flipBitLiteral |= w; } else { flipBitLiteral |= ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); } }*/ i.advanceTo(itr.wordsWalked); if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } nextVal = theQ.peek(); } // advance longest one literal forward and push result back to priority queue // if a flip bit is still needed, put it in the correct position /*int newWord = word & 0xC1FFFFFF; if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { flipBitLiteral ^= ConciseSetUtils.ALL_ONES_LITERAL; int position = Integer.numberOfTrailingZeros(flipBitLiteral) + 1; newWord |= (position << 25); }*/ addAndCompact(retVal, word); currIndex = itr.wordsWalked; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } } else if (ConciseSetUtils.isLiteral(word)) { // advance all other literals WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); // if we still have zero fills with flipped bits, OR them here if (ConciseSetUtils.isLiteral(w)) { word |= w; if (word == 0xffffffff) word = 0x10000001; } else { /*int flipBitLiteral = ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ZEROS_LITERAL) { word |= flipBitLiteral; i.advanceTo(itr.wordsWalked); }*/ if (ConciseSetUtils.is1_fill(w)) { word = 0x10000001; i.advanceTo(itr.wordsWalked); } } if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } nextVal = theQ.peek(); } // advance the set with the current literal forward and push result back to priority queue addAndCompact(retVal, word); currIndex++; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } } else { // zero fills WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { // check if literal can be created flip bits of other zero sequences WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } nextVal = theQ.peek(); } // check if a literal needs to be created from the flipped bits of this sequence if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } } theQ.addAll(wordsToAdd); } if (retVal.isEmpty()) { return new ImmutableSecompaxSet(); } return new ImmutableSecompaxSet(IntBuffer.wrap(retVal.toArray())); }
From source file:it.uniroma3.mat.extendedset.intset.ImmutableConciseSet.java
public static ImmutableConciseSet doIntersection(Iterator<ImmutableConciseSet> sets) { IntList retVal = new IntList(); // lhs = current word position, rhs = the iterator // Comparison is first by index, then zero fills > literals > one fills // zero fills are sorted by length (longer zero fills have priority) // similarily, shorter one fills have priority MinMaxPriorityQueue<WordHolder> theQ = MinMaxPriorityQueue.orderedBy(new Comparator<WordHolder>() { @Override/*from w ww.j a v a 2 s.c o m*/ public int compare(WordHolder h1, WordHolder h2) { int w1 = h1.getWord(); int w2 = h2.getWord(); int s1 = h1.getIterator().startIndex; int s2 = h2.getIterator().startIndex; if (s1 != s2) { return compareInts(s1, s2); } if (ConciseSetUtils.isZeroSequence(w1)) { if (ConciseSetUtils.isZeroSequence(w2)) { return -compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } return -1; } else if (ConciseSetUtils.isLiteral(w1)) { if (ConciseSetUtils.isZeroSequence(w2)) { return 1; } else if (ConciseSetUtils.isLiteral(w2)) { return 0; } return -1; } else { if (!ConciseSetUtils.isOneSequence(w2)) { return 1; } return compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } } }).create(); // populate priority queue while (sets.hasNext()) { ImmutableConciseSet set = sets.next(); if (set == null || set.isEmpty()) { return new ImmutableConciseSet(); } WordIterator itr = set.newWordIterator(); theQ.add(new WordHolder(itr.next(), itr)); } int currIndex = 0; int wordsWalkedAtSequenceEnd = Integer.MAX_VALUE; while (!theQ.isEmpty()) { // create a temp list to hold everything that will get pushed back into the priority queue after each run List<WordHolder> wordsToAdd = Lists.newArrayList(); // grab the top element from the priority queue WordHolder curr = theQ.poll(); int word = curr.getWord(); WordIterator itr = curr.getIterator(); // if a sequence has ended, we can break out because of Boolean logic if (itr.startIndex >= wordsWalkedAtSequenceEnd) { break; } // if the next word in the queue starts at a different point than where we ended off we need to create a one gap // to fill the space if (currIndex < itr.startIndex) { // number of 31 bit blocks that compromise the fill minus one addAndCompact(retVal, (ConciseSetUtils.SEQUENCE_BIT | (itr.startIndex - currIndex - 1))); currIndex = itr.startIndex; } if (ConciseSetUtils.isZeroSequence(word)) { // extract a literal from the flip bits of the zero sequence int flipBitLiteral = ConciseSetUtils.getLiteralFromZeroSeqFlipBit(word); // advance everything past the longest zero sequence WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex < itr.wordsWalked) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); if (i.startIndex == itr.startIndex) { // if a literal was created from a flip bit, AND it with other literals or literals from flip bits in the same // position if (ConciseSetUtils.isZeroSequence(w)) { flipBitLiteral &= ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); } else if (ConciseSetUtils.isLiteral(w)) { flipBitLiteral &= w; } else { flipBitLiteral &= ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); } } i.advanceTo(itr.wordsWalked); if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } else { wordsWalkedAtSequenceEnd = Math.min(i.wordsWalked, wordsWalkedAtSequenceEnd); } nextVal = theQ.peek(); } // advance longest zero literal forward and push result back to priority queue // if a flip bit is still needed, put it in the correct position int newWord = word & 0xC1FFFFFF; if (flipBitLiteral != ConciseSetUtils.ALL_ZEROS_LITERAL) { int position = Integer.numberOfTrailingZeros(flipBitLiteral) + 1; newWord = (word & 0xC1FFFFFF) | (position << 25); } addAndCompact(retVal, newWord); currIndex = itr.wordsWalked; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } else { wordsWalkedAtSequenceEnd = Math.min(itr.wordsWalked, wordsWalkedAtSequenceEnd); } } else if (ConciseSetUtils.isLiteral(word)) { // advance all other literals WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); // if we still have one fills with flipped bits, AND them here if (ConciseSetUtils.isLiteral(w)) { word &= w; } else { int flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { word &= flipBitLiteral; i.advanceTo(itr.wordsWalked); } } if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } else { wordsWalkedAtSequenceEnd = Math.min(i.wordsWalked, wordsWalkedAtSequenceEnd); } nextVal = theQ.peek(); } // advance the set with the current literal forward and push result back to priority queue addAndCompact(retVal, word); currIndex++; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } else { wordsWalkedAtSequenceEnd = Math.min(itr.wordsWalked, wordsWalkedAtSequenceEnd); } } else { // one fills int flipBitLiteral; WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { // check if literal can be created flip bits of other one sequences WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { wordsToAdd.add(new WordHolder(flipBitLiteral, i)); } else if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } else { wordsWalkedAtSequenceEnd = Math.min(i.wordsWalked, wordsWalkedAtSequenceEnd); } nextVal = theQ.peek(); } // check if a literal needs to be created from the flipped bits of this sequence flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(word); if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { wordsToAdd.add(new WordHolder(flipBitLiteral, itr)); } else if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } else { wordsWalkedAtSequenceEnd = Math.min(itr.wordsWalked, wordsWalkedAtSequenceEnd); } } theQ.addAll(wordsToAdd); } // fill in any missing one sequences if (currIndex < wordsWalkedAtSequenceEnd) { addAndCompact(retVal, (ConciseSetUtils.SEQUENCE_BIT | (wordsWalkedAtSequenceEnd - currIndex - 1))); } if (retVal.isEmpty()) { return new ImmutableConciseSet(); } return new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray())); }
From source file:it.uniroma3.mat.extendedset.intset.ImmutableSecompaxSet.java
public static ImmutableSecompaxSet doIntersection(Iterator<ImmutableSecompaxSet> sets) { IntList retVal = new IntList(); // lhs = current word position, rhs = the iterator // Comparison is first by index, then zero fills > literals > one fills // zero fills are sorted by length (longer zero fills have priority) // similarily, shorter one fills have priority MinMaxPriorityQueue<WordHolder> theQ = MinMaxPriorityQueue.orderedBy(new Comparator<WordHolder>() { @Override/*from ww w. ja va 2 s .c om*/ public int compare(WordHolder h1, WordHolder h2) { int w1 = h1.getWord(); int w2 = h2.getWord(); int s1 = h1.getIterator().startIndex; int s2 = h2.getIterator().startIndex; if (s1 != s2) { return compareInts(s1, s2); } if (ConciseSetUtils.is0_fill(w1)) { if (ConciseSetUtils.is0_fill(w2)) { return -compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } return -1; } else if (ConciseSetUtils.isLiteral(w1)) { if (ConciseSetUtils.is0_fill(w2)) { return 1; } else if (ConciseSetUtils.isLiteral(w2)) { return 0; } return -1; } else { if (!ConciseSetUtils.is1_fill(w2)) { return 1; } return compareInts(ConciseSetUtils.getSequenceNumWords(w1), ConciseSetUtils.getSequenceNumWords(w2)); } } }).create(); // populate priority queue while (sets.hasNext()) { ImmutableSecompaxSet set = sets.next(); if (set == null || set.isEmpty()) { return new ImmutableSecompaxSet(); } WordIterator itr = set.newWordIterator(); theQ.add(new WordHolder(itr.next(), itr)); } int currIndex = 0; int wordsWalkedAtSequenceEnd = Integer.MAX_VALUE; while (!theQ.isEmpty()) { // create a temp list to hold everything that will get pushed back into the priority queue after each run List<WordHolder> wordsToAdd = Lists.newArrayList(); // grab the top element from the priority queue WordHolder curr = theQ.poll(); int word = curr.getWord(); WordIterator itr = curr.getIterator(); // if a sequence has ended, we can break out because of Boolean logic if (itr.startIndex >= wordsWalkedAtSequenceEnd) { break; } // if the next word in the queue starts at a different point than where we ended off we need to create a one gap // to fill the space if (currIndex < itr.startIndex) { // number of 31 bit blocks that compromise the fill minus one addAndCompact(retVal, (0x10000000 | (itr.startIndex - currIndex))); currIndex = itr.startIndex; } if (ConciseSetUtils.is0_fill(word)) { // extract a literal from the flip bits of the zero sequence //int flipBitLiteral = ConciseSetUtils.getLiteralFromZeroSeqFlipBit(word); // advance everything past the longest zero sequence WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex < itr.wordsWalked) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); /*if (i.startIndex == itr.startIndex) { // if a literal was created from a flip bit, AND it with other literals or literals from flip bits in the same // position if (ConciseSetUtils.isZeroSequence(w)) { flipBitLiteral &= ConciseSetUtils.getLiteralFromZeroSeqFlipBit(w); } else if (ConciseSetUtils.isLiteral(w)) { flipBitLiteral &= w; } else { flipBitLiteral &= ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); } }*/ i.advanceTo(itr.wordsWalked); if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } else { wordsWalkedAtSequenceEnd = Math.min(i.wordsWalked, wordsWalkedAtSequenceEnd); } nextVal = theQ.peek(); } // advance longest zero literal forward and push result back to priority queue // if a flip bit is still needed, put it in the correct position //int newWord = word & 0xC1FFFFFF; /*if (flipBitLiteral != ConciseSetUtils.ALL_ZEROS_LITERAL) { int position = Integer.numberOfTrailingZeros(flipBitLiteral) + 1; newWord = (word & 0xC1FFFFFF) | (position << 25); }*/ addAndCompact(retVal, word); currIndex = itr.wordsWalked; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } else { wordsWalkedAtSequenceEnd = Math.min(itr.wordsWalked, wordsWalkedAtSequenceEnd); } } else if (ConciseSetUtils.isLiteral(word)) { // advance all other literals WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); // if we still have one fills with flipped bits, AND them here if (ConciseSetUtils.isLiteral(w)) { word &= w; } else { /*int flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { word &= flipBitLiteral; i.advanceTo(itr.wordsWalked); }*/ if (ConciseSetUtils.is0_fill(w)) { word = 0x00000001; i.advanceTo(itr.wordsWalked); } else { i.advanceTo(itr.wordsWalked); } } if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } else { wordsWalkedAtSequenceEnd = Math.min(i.wordsWalked, wordsWalkedAtSequenceEnd); } nextVal = theQ.peek(); } // advance the set with the current literal forward and push result back to priority queue addAndCompact(retVal, word); currIndex++; if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } else { wordsWalkedAtSequenceEnd = Math.min(itr.wordsWalked, wordsWalkedAtSequenceEnd); } } else { // one fills //int flipBitLiteral; WordHolder nextVal = theQ.peek(); while (nextVal != null && nextVal.getIterator().startIndex == itr.startIndex) { // check if literal can be created flip bits of other one sequences WordHolder entry = theQ.poll(); int w = entry.getWord(); WordIterator i = entry.getIterator(); i.advanceTo(itr.wordsWalked); /*flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(w); if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { wordsToAdd.add(new WordHolder(flipBitLiteral, i)); } else */if (i.hasNext()) { wordsToAdd.add(new WordHolder(i.next(), i)); } else { wordsWalkedAtSequenceEnd = Math.min(i.wordsWalked, wordsWalkedAtSequenceEnd); } nextVal = theQ.peek(); } // check if a literal needs to be created from the flipped bits of this sequence //flipBitLiteral = ConciseSetUtils.getLiteralFromOneSeqFlipBit(word); /*if (flipBitLiteral != ConciseSetUtils.ALL_ONES_LITERAL) { wordsToAdd.add(new WordHolder(flipBitLiteral, itr)); } else */if (itr.hasNext()) { wordsToAdd.add(new WordHolder(itr.next(), itr)); } else { wordsWalkedAtSequenceEnd = Math.min(itr.wordsWalked, wordsWalkedAtSequenceEnd); } } theQ.addAll(wordsToAdd); } // fill in any missing one sequences if (currIndex < wordsWalkedAtSequenceEnd) { addAndCompact(retVal, (0x10000000 | (wordsWalkedAtSequenceEnd - currIndex))); } if (retVal.isEmpty()) { return new ImmutableSecompaxSet(); } return new ImmutableSecompaxSet(IntBuffer.wrap(retVal.toArray())); }