List of usage examples for com.google.common.collect Multiset add
@Override
boolean add(E element);
From source file:org.dllearner.algorithms.qtl.experiments.QTLEvaluation.java
private RDFResourceTree applyBaseLine(ExamplesWrapper examples, Baseline baselineApproach) { Collection<RDFResourceTree> posExamples = examples.posExamplesMapping.values(); Collection<RDFResourceTree> negExamples = examples.negExamplesMapping.values(); switch (baselineApproach) { case RANDOM:// 1. String query = "SELECT ?cls WHERE {?cls a owl:Class .} ORDER BY RAND() LIMIT 1"; QueryExecution qe = qef.createQueryExecution(query); ResultSet rs = qe.execSelect(); if (rs.hasNext()) { QuerySolution qs = rs.next(); Resource cls = qs.getResource("cls"); RDFResourceTree solution = new RDFResourceTree(); solution.addChild(new RDFResourceTree(cls.asNode()), RDF.type.asNode()); return solution; }//from www . j a v a 2 s. c o m case MOST_POPULAR_TYPE_IN_KB:// 2. query = "SELECT ?cls WHERE {?cls a owl:Class . ?s a ?cls .} ORDER BY DESC(COUNT(?s)) LIMIT 1"; qe = qef.createQueryExecution(query); rs = qe.execSelect(); if (rs.hasNext()) { QuerySolution qs = rs.next(); Resource cls = qs.getResource("cls"); RDFResourceTree solution = new RDFResourceTree(); solution.addChild(new RDFResourceTree(cls.asNode()), RDF.type.asNode()); return solution; } case MOST_FREQUENT_TYPE_IN_EXAMPLES:// 3. Multiset<Node> types = HashMultiset.create(); for (RDFResourceTree ex : posExamples) { List<RDFResourceTree> children = ex.getChildren(RDF.type.asNode()); for (RDFResourceTree child : children) { types.add(child.getData()); } } Node mostFrequentType = Ordering.natural().onResultOf(new Function<Multiset.Entry<Node>, Integer>() { @Override public Integer apply(Multiset.Entry<Node> entry) { return entry.getCount(); } }).max(types.entrySet()).getElement(); RDFResourceTree solution = new RDFResourceTree(); solution.addChild(new RDFResourceTree(mostFrequentType), RDF.type.asNode()); return solution; case MOST_FREQUENT_EDGE_IN_EXAMPLES:// 4. { Multiset<Pair<Node, Node>> pairs = HashMultiset.create(); for (RDFResourceTree ex : posExamples) { SortedSet<Node> edges = ex.getEdges(); for (Node edge : edges) { List<RDFResourceTree> children = ex.getChildren(edge); for (RDFResourceTree child : children) { pairs.add(new Pair<>(edge, child.getData())); } } } Pair<Node, Node> mostFrequentPair = Ordering.natural() .onResultOf(new Function<Multiset.Entry<Pair<Node, Node>>, Integer>() { @Override public Integer apply(Multiset.Entry<Pair<Node, Node>> entry) { return entry.getCount(); } }).max(pairs.entrySet()).getElement(); solution = new RDFResourceTree(); solution.addChild(new RDFResourceTree(mostFrequentPair.getValue()), mostFrequentPair.getKey()); return solution; } case MOST_INFORMATIVE_EDGE_IN_EXAMPLES: // get all p-o in pos examples Multiset<Pair<Node, Node>> edgeObjectPairs = HashMultiset.create(); for (RDFResourceTree ex : posExamples) { SortedSet<Node> edges = ex.getEdges(); for (Node edge : edges) { List<RDFResourceTree> children = ex.getChildren(edge); for (RDFResourceTree child : children) { edgeObjectPairs.add(new Pair<>(edge, child.getData())); } } } double bestAccuracy = -1; solution = new RDFResourceTree(); for (Pair<Node, Node> pair : edgeObjectPairs.elementSet()) { Node edge = pair.getKey(); Node childValue = pair.getValue(); // compute accuracy int tp = edgeObjectPairs.count(pair); int fn = posExamples.size() - tp; int fp = 0; for (RDFResourceTree ex : negExamples) { // compute false positives List<RDFResourceTree> children = ex.getChildren(edge); if (children != null) { for (RDFResourceTree child : children) { if (child.getData().equals(childValue)) { fp++; break; } } } } int tn = negExamples.size() - fp; double accuracy = Heuristics.getPredictiveAccuracy(posExamples.size(), negExamples.size(), tp, tn, 1.0); // update best solution if (accuracy >= bestAccuracy) { solution = new RDFResourceTree(); solution.addChild(new RDFResourceTree(childValue), edge); bestAccuracy = accuracy; } } return solution; case LGG: LGGGenerator lggGenerator = new LGGGeneratorSimple(); RDFResourceTree lgg = lggGenerator.getLGG(Lists.newArrayList(posExamples)); return lgg; default: break; } return null; }
From source file:org.cspoker.ai.opponentmodels.weka.Propositionalizer.java
public void signalCardShowdown(Object id, Card card1, Card card2) { // PlayerData p = players.get(id); if (cards.size() == 5) { //showdown after river Multiset<Integer> ranks = new TreeMultiset<Integer>(); // int minSampleRank = Integer.MAX_VALUE; // int maxSampleRank = Integer.MIN_VALUE; // int sum = 0; int startRank = 53; for (Card card : cards) { startRank = handRanks[card.ordinal() + 1 + startRank]; }// ww w . j ava 2 s. c o m //add real rank int realRank = startRank; realRank = handRanks[card1.ordinal() + 1 + realRank]; realRank = handRanks[card2.ordinal() + 1 + realRank]; int realType = (realRank >>> 12) - 1; realRank = realRank & 0xFFF; realRank = offsets[realType] + realRank - 1; //take rank samples int nbBuckets = 6; int nbSamplesPerBucket = 6; int nbSamples = nbBuckets * nbSamplesPerBucket; for (int i = 0; i < nbSamples; i++) { int rank = startRank; Card sampleCard1; do { sampleCard1 = Card.values()[random.nextInt(Card.values().length)]; } while (cards.contains(sampleCard1)); rank = handRanks[sampleCard1.ordinal() + 1 + rank]; Card sampleCard2; do { sampleCard2 = Card.values()[random.nextInt(Card.values().length)]; } while (cards.contains(sampleCard2) || sampleCard2.equals(sampleCard1)); rank = handRanks[sampleCard2.ordinal() + 1 + rank]; int type = (rank >>> 12) - 1; rank = rank & 0xFFF; rank = offsets[type] + rank - 1; ranks.add(rank); // if(rank<minSampleRank){ // minSampleRank = rank; // } // if(rank>maxSampleRank){ // maxSampleRank = rank; // } // sum += rank; } // double var = 0; // double mean = ((double)sum)/nbSamples; // for (Multiset.Entry<Integer> entry : ranks.entrySet()) { // double diff = mean - entry.getElement(); // var += diff * diff * entry.getCount(); // } // var /= (nbSamples-1); // int averageSampleRank = (int) Math.round(mean); // int sigmaSampleRank = (int) Math.round(Math.sqrt(var)); int[] bucketCounts = new int[nbBuckets]; Iterator<Integer> iter = ranks.iterator(); double realRankCount = ranks.count(realRank); // long avgBucket = -1; double[] bucketDistr = new double[nbBuckets]; if (realRankCount > 0) { for (int bucket = 0; bucket < nbBuckets; bucket++) { for (int i = 0; i < nbSamplesPerBucket; i++) { int rank = iter.next(); if (rank == realRank) { ++bucketCounts[bucket]; } } } int partitionSum = 0; for (int i = 0; i < nbBuckets; i++) { bucketDistr[i] = bucketCounts[i] / realRankCount; partitionSum += bucketCounts[i] * i; } // avgBucket = Math.round(partitionSum/realRankCount); } else { boolean found = false; bucketIteration: for (int bucket = 0; bucket < nbBuckets; bucket++) { for (int i = 0; i < nbSamplesPerBucket; i++) { int rank = iter.next(); if (rank > realRank) { bucketDistr[bucket] = 1; // avgBucket = bucket; found = true; break bucketIteration; } } } if (!found) { bucketCounts[nbBuckets - 1] = 1; // avgBucket = nbBuckets-1; } } logShowdown(id, bucketDistr); } else { //ignore //throw new IllegalStateException("everybody went all-in before the river"); } }
From source file:bots.mctsbot.ai.opponentmodels.weka.Propositionalizer.java
public void signalCardShowdown(Object id, Card card1, Card card2) { // PlayerData p = players.get(id); if (cards.size() == 5) { //showdown after river Multiset<Integer> ranks = new TreeMultiset<Integer>(); // int minSampleRank = Integer.MAX_VALUE; // int maxSampleRank = Integer.MIN_VALUE; // int sum = 0; int startRank = 53; for (int i = 0; i < cards.size(); i++) { startRank = handRanks[CardConverter.toSpears2p2Index(cards.getCard(i + 1)) + startRank]; }/*from w ww . java2 s.c om*/ //add real rank int realRank = startRank; realRank = handRanks[CardConverter.toSpears2p2Index(card1) + realRank]; realRank = handRanks[CardConverter.toSpears2p2Index(card2) + realRank]; int realType = (realRank >>> 12) - 1; realRank = realRank & 0xFFF; realRank = offsets[realType] + realRank - 1; //take rank samples int nbBuckets = 6; int nbSamplesPerBucket = 6; int nbSamples = nbBuckets * nbSamplesPerBucket; for (int i = 0; i < nbSamples; i++) { int rank = startRank; Card sampleCard1; do { sampleCard1 = new Card(random.nextInt(52)); } while (cards.contains(sampleCard1)); rank = handRanks[CardConverter.toSpears2p2Index(sampleCard1) + rank]; Card sampleCard2; do { sampleCard2 = new Card(random.nextInt(52)); } while (cards.contains(sampleCard2) || sampleCard2.equals(sampleCard1)); rank = handRanks[CardConverter.toSpears2p2Index(sampleCard2) + rank]; int type = (rank >>> 12) - 1; rank = rank & 0xFFF; rank = offsets[type] + rank - 1; ranks.add(rank); // if(rank<minSampleRank){ // minSampleRank = rank; // } // if(rank>maxSampleRank){ // maxSampleRank = rank; // } // sum += rank; } // double var = 0; // double mean = ((double)sum)/nbSamples; // for (Multiset.Entry<Integer> entry : ranks.entrySet()) { // double diff = mean - entry.getElement(); // var += diff * diff * entry.getCount(); // } // var /= (nbSamples-1); // int averageSampleRank = (int) Math.round(mean); // int sigmaSampleRank = (int) Math.round(Math.sqrt(var)); int[] bucketCounts = new int[nbBuckets]; Iterator<Integer> iter = ranks.iterator(); double realRankCount = ranks.count(realRank); // long avgBucket = -1; double[] bucketDistr = new double[nbBuckets]; if (realRankCount > 0) { for (int bucket = 0; bucket < nbBuckets; bucket++) { for (int i = 0; i < nbSamplesPerBucket; i++) { int rank = iter.next(); if (rank == realRank) { ++bucketCounts[bucket]; } } } int partitionSum = 0; for (int i = 0; i < nbBuckets; i++) { bucketDistr[i] = bucketCounts[i] / realRankCount; partitionSum += bucketCounts[i] * i; } // avgBucket = Math.round(partitionSum/realRankCount); } else { boolean found = false; bucketIteration: for (int bucket = 0; bucket < nbBuckets; bucket++) { for (int i = 0; i < nbSamplesPerBucket; i++) { int rank = iter.next(); if (rank > realRank) { bucketDistr[bucket] = 1; // avgBucket = bucket; found = true; break bucketIteration; } } } if (!found) { bucketCounts[nbBuckets - 1] = 1; // avgBucket = nbBuckets-1; } } logShowdown(id, bucketDistr); } else { //ignore //throw new IllegalStateException("everybody went all-in before the river"); } }
From source file:de.huberlin.german.korpling.laudatioteitool.SplitTEI.java
private TEIValidator.Errors extractPreparationSteps(Document doc) throws LaudatioException, IOException, SAXException { TEIValidator validator = preparationSchemeURL == null ? new TEIPreparationValidator() : new FromURLValidator(preparationSchemeURL); Multiset<String> knownPreparationTitles = HashMultiset.create(); File documentDir = new File(outputDirectory, "PreparationHeader"); if (!documentDir.exists() && !documentDir.mkdir()) { throw new LaudatioException( messages.getString("COULD NOT CREATE DIRECTORY") + documentDir.getAbsolutePath()); }//from w w w . j a va2 s . c om Preconditions.checkNotNull(doc.getRootElement().getChild("teiCorpus", null)); Element preparationRoot = Preconditions .checkNotNull(doc.getRootElement().getChild("teiCorpus", null).getChild("teiCorpus", null)); for (Element preparationHeader : preparationRoot.getChildren("teiHeader", null)) { Preconditions.checkState("PreparationHeader".equals(preparationHeader.getAttributeValue("type"))); // create the subtree for the global corpus header Namespace teiNS = Namespace.getNamespace("http://www.tei-c.org/ns/1.0"); Element tei = new Element("TEI", teiNS); tei.addContent(preparationHeader.clone()); Document newDoc = new Document(tei); if (preparationSchemeURL == null) { newDoc.addContent(0, new ProcessingInstruction("xml-model", "href=\"" + TEIPreparationValidator.DEFAULT_SCHEME_URL + "\"")); } else { newDoc.addContent(0, new ProcessingInstruction("xml-model", "href=\"" + preparationSchemeURL + "\"")); } // we need to append an empty "text" element after the header Element text = new Element("text", teiNS); text.setText(""); tei.addContent(text); Element fileDesc = Preconditions .checkNotNull(tei.getChild("teiHeader", null).getChild("fileDesc", null)); String outName = UUID.randomUUID().toString(); Element titleStmt = Preconditions.checkNotNull(fileDesc.getChild("titleStmt", null)); Element title = Preconditions.checkNotNull(titleStmt.getChild("title", null)); String corresp = title.getAttributeValue("corresp"); if (corresp != null) { if (knownPreparationTitles.contains(corresp)) { knownPreparationTitles.add(corresp); outName = corresp + "_" + knownPreparationTitles.count(corresp); log.warn(messages.getString("MORE THAN ONE PREPARATION HEADER"), corresp); } else { outName = corresp; knownPreparationTitles.add(corresp); } } File outputFile = new File(documentDir, outName + ".xml"); XMLOutputter xmlOut = new XMLOutputter(Format.getPrettyFormat()); xmlOut.output(newDoc, new OutputStreamWriter(new FileOutputStream(outputFile), "UTF-8")); log.info(messages.getString("WRITTEN PREPARATION HEADER"), outputFile.getPath()); validator.validate(outputFile); } return validator.getErrors(); }
From source file:com.google.googlejavaformat.java.JavaInputAstVisitor.java
/** Returns true if {@code atLeastM} of the expressions in the given column are the same kind. */ private static boolean expressionsAreParallel(List<List<ExpressionTree>> rows, int column, int atLeastM) { Multiset<Tree.Kind> nodeTypes = HashMultiset.create(); for (List<? extends ExpressionTree> row : rows) { if (column >= row.size()) { continue; }//from w w w . ja v a2 s . co m nodeTypes.add(row.get(column).getKind()); } for (Multiset.Entry<Tree.Kind> nodeType : nodeTypes.entrySet()) { if (nodeType.getCount() >= atLeastM) { return true; } } return false; }
From source file:org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.java
/** * Process all the nodes and create splits that are local to a node. * Generate one split per node iteration, and walk over nodes multiple times * to distribute the splits across nodes. * <p>//from www . j a va2s .c o m * Note: The order of processing the nodes is undetermined because the * implementation of nodeToBlocks is {@link java.util.HashMap} and its order * of the entries is undetermined. * @param nodeToBlocks Mapping from a node to the list of blocks that * it contains. * @param blockToNodes Mapping from a block to the nodes on which * it has replicas. * @param rackToBlocks Mapping from a rack name to the list of blocks it has. * @param totLength Total length of the input files. * @param maxSize Max size of each split. * If set to 0, disable smoothing load. * @param minSizeNode Minimum split size per node. * @param minSizeRack Minimum split size per rack. * @param splits New splits created by this method are added to the list. */ @VisibleForTesting void createSplits(Map<String, Set<OneBlockInfo>> nodeToBlocks, Map<OneBlockInfo, String[]> blockToNodes, Map<String, List<OneBlockInfo>> rackToBlocks, long totLength, long maxSize, long minSizeNode, long minSizeRack, List<InputSplit> splits) { ArrayList<OneBlockInfo> validBlocks = new ArrayList<OneBlockInfo>(); long curSplitSize = 0; int totalNodes = nodeToBlocks.size(); long totalLength = totLength; Multiset<String> splitsPerNode = HashMultiset.create(); Set<String> completedNodes = new HashSet<String>(); while (true) { for (Iterator<Map.Entry<String, Set<OneBlockInfo>>> iter = nodeToBlocks.entrySet().iterator(); iter .hasNext();) { Map.Entry<String, Set<OneBlockInfo>> one = iter.next(); String node = one.getKey(); // Skip the node if it has previously been marked as completed. if (completedNodes.contains(node)) { continue; } Set<OneBlockInfo> blocksInCurrentNode = one.getValue(); // for each block, copy it into validBlocks. Delete it from // blockToNodes so that the same block does not appear in // two different splits. Iterator<OneBlockInfo> oneBlockIter = blocksInCurrentNode.iterator(); while (oneBlockIter.hasNext()) { OneBlockInfo oneblock = oneBlockIter.next(); // Remove all blocks which may already have been assigned to other // splits. if (!blockToNodes.containsKey(oneblock)) { oneBlockIter.remove(); continue; } validBlocks.add(oneblock); blockToNodes.remove(oneblock); curSplitSize += oneblock.length; // if the accumulated split size exceeds the maximum, then // create this split. if (maxSize != 0 && curSplitSize >= maxSize) { // create an input split and add it to the splits array addCreatedSplit(splits, Collections.singleton(node), validBlocks); totalLength -= curSplitSize; curSplitSize = 0; splitsPerNode.add(node); // Remove entries from blocksInNode so that we don't walk these // again. blocksInCurrentNode.removeAll(validBlocks); validBlocks.clear(); // Done creating a single split for this node. Move on to the next // node so that splits are distributed across nodes. break; } } if (validBlocks.size() != 0) { // This implies that the last few blocks (or all in case maxSize=0) // were not part of a split. The node is complete. // if there were any blocks left over and their combined size is // larger than minSplitNode, then combine them into one split. // Otherwise add them back to the unprocessed pool. It is likely // that they will be combined with other blocks from the // same rack later on. // This condition also kicks in when max split size is not set. All // blocks on a node will be grouped together into a single split. if (minSizeNode != 0 && curSplitSize >= minSizeNode && splitsPerNode.count(node) == 0) { // haven't created any split on this machine. so its ok to add a // smaller one for parallelism. Otherwise group it in the rack for // balanced size create an input split and add it to the splits // array addCreatedSplit(splits, Collections.singleton(node), validBlocks); totalLength -= curSplitSize; splitsPerNode.add(node); // Remove entries from blocksInNode so that we don't walk this again. blocksInCurrentNode.removeAll(validBlocks); // The node is done. This was the last set of blocks for this node. } else { // Put the unplaced blocks back into the pool for later rack-allocation. for (OneBlockInfo oneblock : validBlocks) { blockToNodes.put(oneblock, oneblock.hosts); } } validBlocks.clear(); curSplitSize = 0; completedNodes.add(node); } else { // No in-flight blocks. if (blocksInCurrentNode.size() == 0) { // Node is done. All blocks were fit into node-local splits. completedNodes.add(node); } // else Run through the node again. } } // Check if node-local assignments are complete. if (completedNodes.size() == totalNodes || totalLength == 0) { // All nodes have been walked over and marked as completed or all blocks // have been assigned. The rest should be handled via rackLock assignment. LOG.info("DEBUG: Terminated node allocation with : CompletedNodes: " + completedNodes.size() + ", size left: " + totalLength); break; } } // if blocks in a rack are below the specified minimum size, then keep them // in 'overflow'. After the processing of all racks is complete, these // overflow blocks will be combined into splits. ArrayList<OneBlockInfo> overflowBlocks = new ArrayList<OneBlockInfo>(); Set<String> racks = new HashSet<String>(); // Process all racks over and over again until there is no more work to do. while (blockToNodes.size() > 0) { // Create one split for this rack before moving over to the next rack. // Come back to this rack after creating a single split for each of the // remaining racks. // Process one rack location at a time, Combine all possible blocks that // reside on this rack as one split. (constrained by minimum and maximum // split size). // iterate over all racks for (Iterator<Map.Entry<String, List<OneBlockInfo>>> iter = rackToBlocks.entrySet().iterator(); iter .hasNext();) { Map.Entry<String, List<OneBlockInfo>> one = iter.next(); racks.add(one.getKey()); List<OneBlockInfo> blocks = one.getValue(); // for each block, copy it into validBlocks. Delete it from // blockToNodes so that the same block does not appear in // two different splits. boolean createdSplit = false; for (OneBlockInfo oneblock : blocks) { if (blockToNodes.containsKey(oneblock)) { validBlocks.add(oneblock); blockToNodes.remove(oneblock); curSplitSize += oneblock.length; // if the accumulated split size exceeds the maximum, then // create this split. if (maxSize != 0 && curSplitSize >= maxSize) { // create an input split and add it to the splits array addCreatedSplit(splits, getHosts(racks), validBlocks); createdSplit = true; break; } } } // if we created a split, then just go to the next rack if (createdSplit) { curSplitSize = 0; validBlocks.clear(); racks.clear(); continue; } if (!validBlocks.isEmpty()) { if (minSizeRack != 0 && curSplitSize >= minSizeRack) { // if there is a minimum size specified, then create a single split // otherwise, store these blocks into overflow data structure addCreatedSplit(splits, getHosts(racks), validBlocks); } else { // There were a few blocks in this rack that // remained to be processed. Keep them in 'overflow' block list. // These will be combined later. overflowBlocks.addAll(validBlocks); } } curSplitSize = 0; validBlocks.clear(); racks.clear(); } } assert blockToNodes.isEmpty(); assert curSplitSize == 0; assert validBlocks.isEmpty(); assert racks.isEmpty(); // Process all overflow blocks for (OneBlockInfo oneblock : overflowBlocks) { validBlocks.add(oneblock); curSplitSize += oneblock.length; // This might cause an exiting rack location to be re-added, // but it should be ok. for (int i = 0; i < oneblock.racks.length; i++) { racks.add(oneblock.racks[i]); } // if the accumulated split size exceeds the maximum, then // create this split. if (maxSize != 0 && curSplitSize >= maxSize) { // create an input split and add it to the splits array addCreatedSplit(splits, getHosts(racks), validBlocks); curSplitSize = 0; validBlocks.clear(); racks.clear(); } } // Process any remaining blocks, if any. if (!validBlocks.isEmpty()) { addCreatedSplit(splits, getHosts(racks), validBlocks); } }
From source file:com.ikanow.aleph2.analytics.hadoop.assets.UpdatedCombineFileInputFormat.java
@VisibleForTesting void createSplits(Map<String, Set<OneBlockInfo>> nodeToBlocks, Map<OneBlockInfo, String[]> blockToNodes, Map<String, List<OneBlockInfo>> rackToBlocks, long totLength, long maxSize, long minSizeNode, long minSizeRack, List<InputSplit> splits) { ArrayList<OneBlockInfo> validBlocks = new ArrayList<OneBlockInfo>(); long curSplitSize = 0; int totalNodes = nodeToBlocks.size(); long totalLength = totLength; Multiset<String> splitsPerNode = HashMultiset.create(); Set<String> completedNodes = new HashSet<String>(); while (true) { // it is allowed for maxSize to be 0. Disable smoothing load for such cases // process all nodes and create splits that are local to a node. Generate // one split per node iteration, and walk over nodes multiple times to // distribute the splits across nodes. for (Iterator<Map.Entry<String, Set<OneBlockInfo>>> iter = nodeToBlocks.entrySet().iterator(); iter .hasNext();) {/*from ww w . j a va2s. c om*/ Map.Entry<String, Set<OneBlockInfo>> one = iter.next(); String node = one.getKey(); // Skip the node if it has previously been marked as completed. if (completedNodes.contains(node)) { continue; } Set<OneBlockInfo> blocksInCurrentNode = one.getValue(); // for each block, copy it into validBlocks. Delete it from // blockToNodes so that the same block does not appear in // two different splits. Iterator<OneBlockInfo> oneBlockIter = blocksInCurrentNode.iterator(); while (oneBlockIter.hasNext()) { OneBlockInfo oneblock = oneBlockIter.next(); // Remove all blocks which may already have been assigned to other // splits. if (!blockToNodes.containsKey(oneblock)) { oneBlockIter.remove(); continue; } validBlocks.add(oneblock); blockToNodes.remove(oneblock); curSplitSize += oneblock.length; // if the accumulated split size exceeds the maximum, then // create this split. if (maxSize != 0 && curSplitSize >= maxSize) { // create an input split and add it to the splits array addCreatedSplit(splits, Collections.singleton(node), validBlocks); totalLength -= curSplitSize; curSplitSize = 0; splitsPerNode.add(node); // Remove entries from blocksInNode so that we don't walk these // again. blocksInCurrentNode.removeAll(validBlocks); validBlocks.clear(); // Done creating a single split for this node. Move on to the next // node so that splits are distributed across nodes. break; } } if (validBlocks.size() != 0) { // This implies that the last few blocks (or all in case maxSize=0) // were not part of a split. The node is complete. // if there were any blocks left over and their combined size is // larger than minSplitNode, then combine them into one split. // Otherwise add them back to the unprocessed pool. It is likely // that they will be combined with other blocks from the // same rack later on. // This condition also kicks in when max split size is not set. All // blocks on a node will be grouped together into a single split. if (minSizeNode != 0 && curSplitSize >= minSizeNode && splitsPerNode.count(node) == 0) { // haven't created any split on this machine. so its ok to add a // smaller one for parallelism. Otherwise group it in the rack for // balanced size create an input split and add it to the splits // array addCreatedSplit(splits, Collections.singleton(node), validBlocks); totalLength -= curSplitSize; splitsPerNode.add(node); // Remove entries from blocksInNode so that we don't walk this again. blocksInCurrentNode.removeAll(validBlocks); // The node is done. This was the last set of blocks for this node. } else { // Put the unplaced blocks back into the pool for later rack-allocation. for (OneBlockInfo oneblock : validBlocks) { blockToNodes.put(oneblock, oneblock.hosts); } } validBlocks.clear(); curSplitSize = 0; completedNodes.add(node); } else { // No in-flight blocks. if (blocksInCurrentNode.size() == 0) { // Node is done. All blocks were fit into node-local splits. completedNodes.add(node); } // else Run through the node again. } } // Check if node-local assignments are complete. if (completedNodes.size() == totalNodes || totalLength == 0) { // All nodes have been walked over and marked as completed or all blocks // have been assigned. The rest should be handled via rackLock assignment. LOG.info("DEBUG: Terminated node allocation with : CompletedNodes: " + completedNodes.size() + ", size left: " + totalLength); break; } } // if blocks in a rack are below the specified minimum size, then keep them // in 'overflow'. After the processing of all racks is complete, these // overflow blocks will be combined into splits. ArrayList<OneBlockInfo> overflowBlocks = new ArrayList<OneBlockInfo>(); Set<String> racks = new HashSet<String>(); // Process all racks over and over again until there is no more work to do. while (blockToNodes.size() > 0) { // Create one split for this rack before moving over to the next rack. // Come back to this rack after creating a single split for each of the // remaining racks. // Process one rack location at a time, Combine all possible blocks that // reside on this rack as one split. (constrained by minimum and maximum // split size). // iterate over all racks for (Iterator<Map.Entry<String, List<OneBlockInfo>>> iter = rackToBlocks.entrySet().iterator(); iter .hasNext();) { Map.Entry<String, List<OneBlockInfo>> one = iter.next(); racks.add(one.getKey()); List<OneBlockInfo> blocks = one.getValue(); // for each block, copy it into validBlocks. Delete it from // blockToNodes so that the same block does not appear in // two different splits. boolean createdSplit = false; for (OneBlockInfo oneblock : blocks) { if (blockToNodes.containsKey(oneblock)) { validBlocks.add(oneblock); blockToNodes.remove(oneblock); curSplitSize += oneblock.length; // if the accumulated split size exceeds the maximum, then // create this split. if (maxSize != 0 && curSplitSize >= maxSize) { // create an input split and add it to the splits array addCreatedSplit(splits, getHosts(racks), validBlocks); createdSplit = true; break; } } } // if we created a split, then just go to the next rack if (createdSplit) { curSplitSize = 0; validBlocks.clear(); racks.clear(); continue; } if (!validBlocks.isEmpty()) { if (minSizeRack != 0 && curSplitSize >= minSizeRack) { // if there is a minimum size specified, then create a single split // otherwise, store these blocks into overflow data structure addCreatedSplit(splits, getHosts(racks), validBlocks); } else { // There were a few blocks in this rack that // remained to be processed. Keep them in 'overflow' block list. // These will be combined later. overflowBlocks.addAll(validBlocks); } } curSplitSize = 0; validBlocks.clear(); racks.clear(); } } assert blockToNodes.isEmpty(); assert curSplitSize == 0; assert validBlocks.isEmpty(); assert racks.isEmpty(); // Process all overflow blocks for (OneBlockInfo oneblock : overflowBlocks) { validBlocks.add(oneblock); curSplitSize += oneblock.length; // This might cause an exiting rack location to be re-added, // but it should be ok. for (int i = 0; i < oneblock.racks.length; i++) { racks.add(oneblock.racks[i]); } // if the accumulated split size exceeds the maximum, then // create this split. if (maxSize != 0 && curSplitSize >= maxSize) { // create an input split and add it to the splits array addCreatedSplit(splits, getHosts(racks), validBlocks); curSplitSize = 0; validBlocks.clear(); racks.clear(); } } // Process any remaining blocks, if any. if (!validBlocks.isEmpty()) { addCreatedSplit(splits, getHosts(racks), validBlocks); } }
From source file:edu.uci.ics.sourcerer.tools.java.metrics.db.NumberOfInterfaceRelativesCalculator.java
@Override public void calculate(QueryExecutor exec, Integer projectID, ProjectMetricModel metrics, TypeModel model) { TaskProgressLogger task = TaskProgressLogger.get(); Pattern anon = Pattern.compile(".*\\$\\d+$"); task.start("Computing NumberOfInterfaceRelatives"); Multiset<ModeledStructuralEntity> parents = HashMultiset.create(); Multiset<ModeledDeclaredType> directParents = HashMultiset.create(); Averager<Double> avgNoii = Averager.create(); Averager<Double> avgNosi = Averager.create(); task.start("Processing types", "types processed", 0); for (ModeledEntity entity : model.getEntities()) { if (projectID.equals(entity.getProjectID())) { if (entity.getType().is(Entity.CLASS, Entity.ENUM) && !anon.matcher(entity.getFqn()).matches()) { ModeledDeclaredType dec = (ModeledDeclaredType) entity; Double value = (double) dec.getInterfaces().size(); if (metrics.missingEntityValue(dec.getEntityID(), Metric.NUMBER_OF_IMPLEMENTED_INTERFACES)) { metrics.setEntityValue(dec.getEntityID(), dec.getFileID(), Metric.NUMBER_OF_IMPLEMENTED_INTERFACES, value); exec.insert(EntityMetricsTable.createInsert(projectID, dec.getFileID(), dec.getEntityID(), Metric.NUMBER_OF_IMPLEMENTED_INTERFACES, value)); }/*from ww w.j a v a 2s.c o m*/ avgNoii.addValue(value); task.progress(); } else if (entity.getType() == Entity.INTERFACE) { ModeledDeclaredType dec = (ModeledDeclaredType) entity; Double value = (double) dec.getInterfaces().size(); if (metrics.missingEntityValue(dec.getEntityID(), Metric.NUMBER_OF_SUPER_INTERFACES)) { metrics.setEntityValue(dec.getEntityID(), dec.getFileID(), Metric.NUMBER_OF_SUPER_INTERFACES, value); exec.insert(EntityMetricsTable.createInsert(projectID, dec.getFileID(), dec.getEntityID(), Metric.NUMBER_OF_SUPER_INTERFACES, value)); } avgNosi.addValue(value); Set<ModeledEntity> seen = new HashSet<>(); for (ModeledEntity iface : dec.getInterfaces()) { Deque<ModeledEntity> stack = new LinkedList<>(); stack.push(iface); boolean first = true; while (!stack.isEmpty()) { ModeledEntity next = stack.pop(); if (!seen.contains(next)) { if (next.getType() == Entity.PARAMETERIZED_TYPE) { ModeledEntity base = ((ModeledParametrizedType) next).getBaseType(); if (base != null) { stack.push(base); } } else if (projectID.equals(next.getProjectID())) { ModeledDeclaredType face = (ModeledDeclaredType) next; if (first) { directParents.add(face); first = false; } seen.add(face); parents.add(face); stack.addAll(face.getInterfaces()); } } } } task.progress(); } } } task.finish(); if (metrics.missingValue(Metric.NUMBER_OF_IMPLEMENTED_INTERFACES)) { metrics.setValue(Metric.NUMBER_OF_IMPLEMENTED_INTERFACES, avgNoii); exec.insert( ProjectMetricsTable.createInsert(projectID, Metric.NUMBER_OF_IMPLEMENTED_INTERFACES, avgNoii)); } if (metrics.missingValue(Metric.NUMBER_OF_SUPER_INTERFACES)) { metrics.setValue(Metric.NUMBER_OF_SUPER_INTERFACES, avgNosi); exec.insert(ProjectMetricsTable.createInsert(projectID, Metric.NUMBER_OF_SUPER_INTERFACES, avgNosi)); } Averager<Double> avgNoi = Averager.create(); Averager<Double> avgDnoi = Averager.create(); for (ModeledEntity entity : model.getEntities()) { if (projectID.equals(entity.getProjectID()) && entity.getType() == Entity.INTERFACE) { ModeledDeclaredType dec = (ModeledDeclaredType) entity; Double value = (double) parents.count(dec); if (metrics.missingEntityValue(dec.getEntityID(), Metric.NUMBER_OF_INTERFACE_CHILDREN)) { metrics.setEntityValue(dec.getEntityID(), dec.getFileID(), Metric.NUMBER_OF_INTERFACE_CHILDREN, value); exec.insert(EntityMetricsTable.createInsert(projectID, dec.getFileID(), dec.getEntityID(), Metric.NUMBER_OF_INTERFACE_CHILDREN, value)); } avgNoi.addValue(value); value = (double) directParents.count(dec); if (metrics.missingEntityValue(dec.getEntityID(), Metric.NUMBER_OF_DIRECT_INTERFACE_CHILDREN)) { metrics.setEntityValue(dec.getEntityID(), dec.getFileID(), Metric.NUMBER_OF_DIRECT_INTERFACE_CHILDREN, value); exec.insert(EntityMetricsTable.createInsert(projectID, dec.getFileID(), dec.getEntityID(), Metric.NUMBER_OF_DIRECT_INTERFACE_CHILDREN, value)); } avgDnoi.addValue(value); } } if (metrics.missingValue(Metric.NUMBER_OF_INTERFACE_CHILDREN)) { metrics.setValue(Metric.NUMBER_OF_INTERFACE_CHILDREN, avgNoi); exec.insert(ProjectMetricsTable.createInsert(projectID, Metric.NUMBER_OF_INTERFACE_CHILDREN, avgNoi)); } if (metrics.missingValue(Metric.NUMBER_OF_DIRECT_INTERFACE_CHILDREN)) { metrics.setValue(Metric.NUMBER_OF_DIRECT_INTERFACE_CHILDREN, avgDnoi); exec.insert(ProjectMetricsTable.createInsert(projectID, Metric.NUMBER_OF_DIRECT_INTERFACE_CHILDREN, avgDnoi)); } task.finish(); }
From source file:org.onebusaway.nyc.vehicle_tracking.impl.inference.ParticleFactoryImpl.java
@Override public Multiset<Particle> createParticles(double timestamp, Observation obs) throws ParticleFilterException { final Set<BlockStateObservation> potentialBlocks = _blocksFromObservationService .determinePotentialBlockStatesForObservation(obs); final Multiset<Particle> particles = HashMultiset.create(); double normOffset = Double.NEGATIVE_INFINITY; for (int i = 0; i < _initialNumberOfParticles; ++i) { final CategoricalDist<Particle> transitionProb = new CategoricalDist<Particle>(); for (final BlockStateObservation blockState : potentialBlocks) { final SensorModelResult transProb = new SensorModelResult("transition"); final double inMotionSample = threadLocalRng.get().nextDouble(); final boolean vehicleNotMoved = inMotionSample < 0.5; final MotionState motionState = _motionModel.updateMotionState(obs, vehicleNotMoved); BlockStateObservation sampledBlockState; if (blockState != null) { /*// w w w.j a v a2 s .co m * Sample a distance along the block using the snapped observation * results as priors. */ if (blockState.isSnapped()) { sampledBlockState = blockState; } else { sampledBlockState = _blockStateSamplingStrategy .samplePriorScheduleState(blockState.getBlockState().getBlockInstance(), obs); } } else { sampledBlockState = null; } final JourneyState journeyState = _journeyStateTransitionModel.getJourneyState(sampledBlockState, null, obs, vehicleNotMoved); final VehicleState state = vehicleState(motionState, sampledBlockState, journeyState, obs); final Context context = new Context(null, state, obs); transProb.addResultAsAnd(_motionModel.getEdgeLikelihood().likelihood(context)); transProb.addResultAsAnd(_motionModel.getGpsLikelihood().likelihood(context)); transProb.addResultAsAnd(_motionModel.getSchedLikelihood().likelihood(context)); transProb.addResultAsAnd(_motionModel.dscLikelihood.likelihood(context)); transProb.addResultAsAnd(_motionModel.runLikelihood.likelihood(context)); transProb.addResultAsAnd(_motionModel.runTransitionLikelihood.likelihood(context)); transProb.addResultAsAnd(_motionModel.nullStateLikelihood.likelihood(context)); transProb.addResultAsAnd(_motionModel.nullLocationLikelihood.likelihood(context)); final Particle newParticle = new Particle(timestamp, null, 0.0, state); newParticle.setResult(transProb); transitionProb.logPut(transProb.getLogProbability(), newParticle); } final Particle newSample; if (transitionProb.canSample()) { newSample = transitionProb.sample(); newSample.setLogWeight(newSample.getResult().getLogProbability()); particles.add(newSample); } else { final double inMotionSample = ParticleFactoryImpl.getThreadLocalRng().get().nextDouble(); final boolean vehicleNotMoved = inMotionSample < 0.5; final MotionState motionState = _motionModel.updateMotionState(obs, vehicleNotMoved); final JourneyState journeyState = _journeyStateTransitionModel.getJourneyState(null, null, obs, vehicleNotMoved); final VehicleState nullState = new VehicleState(motionState, null, journeyState, null, obs); final Context context = new Context(null, nullState, obs); final SensorModelResult priorProb = new SensorModelResult("prior creation"); priorProb.addResultAsAnd(_motionModel.getEdgeLikelihood().likelihood(context)); priorProb.addResultAsAnd(_motionModel.getGpsLikelihood().likelihood(context)); priorProb.addResultAsAnd(_motionModel.getSchedLikelihood().likelihood(context)); priorProb.addResultAsAnd(_motionModel.dscLikelihood.likelihood(context)); priorProb.addResultAsAnd(_motionModel.runLikelihood.likelihood(context)); priorProb.addResultAsAnd(_motionModel.runTransitionLikelihood.likelihood(context)); priorProb.addResultAsAnd(_motionModel.nullStateLikelihood.likelihood(context)); priorProb.addResultAsAnd(_motionModel.nullLocationLikelihood.likelihood(context)); newSample = new Particle(timestamp, null, 0.0, nullState); newSample.setResult(priorProb); particles.add(newSample); newSample.setLogWeight(newSample.getResult().getLogProbability()); } normOffset = LogMath.add(newSample.getLogWeight(), normOffset); } /* * Normalize */ for (final Entry<Particle> p : particles.entrySet()) { p.getElement() .setLogNormedWeight(p.getElement().getLogWeight() + FastMath.log(p.getCount()) - normOffset); } return particles; }
From source file:org.onebusaway.nyc.vehicle_tracking.impl.inference.MotionModelImpl.java
private Particle sampleTransitionParticle(Entry<Particle> parent, BlockStateObservation newParentBlockStateObs, Observation obs, final double vehicleHasNotMovedProb, Set<BlockStateObservation> transitions) throws ParticleFilterException { final long timestamp = obs.getTime(); final VehicleState parentState = parent.getElement().getData(); final CategoricalDist<Particle> transitionDist = new CategoricalDist<Particle>(); Multiset<Particle> debugTransitions = null; if (ParticleFilter.getDebugEnabled()) debugTransitions = HashMultiset.create(); for (final BlockStateObservation proposalEdge : transitions) { final SensorModelResult transProb = new SensorModelResult("Transition"); final double inMotionSample = ParticleFactoryImpl.getThreadLocalRng().get().nextDouble(); final boolean vehicleNotMoved = inMotionSample < vehicleHasNotMovedProb; final MotionState motionState = updateMotionState(parentState, obs, vehicleNotMoved); final Map.Entry<BlockSampleType, BlockStateObservation> newEdge; JourneyState journeyState;/*from w w w.ja va 2 s . c o m*/ /* * We have to allow for a driver to start a trip late, so we check to see * if during the transition it was snapped or not. If not, then we assume * it may still be not in service */ newEdge = sampleEdgeFromProposal(newParentBlockStateObs, proposalEdge, obs, parentState.getJourneyState().getPhase(), vehicleNotMoved); journeyState = _journeyStateTransitionModel.getJourneyState(newEdge.getValue(), parentState, obs, vehicleNotMoved); final VehicleState newState = new VehicleState(motionState, newEdge.getValue(), journeyState, null, obs); final Context context = new Context(parentState, newState, obs); transProb.addResultAsAnd(edgeLikelihood.likelihood(context)); transProb.addResultAsAnd(gpsLikelihood.likelihood(context)); transProb.addResultAsAnd(dscLikelihood.likelihood(context)); transProb.addResultAsAnd(runLikelihood.likelihood(context)); transProb.addResultAsAnd(schedLikelihood.likelihood(context)); transProb.addResultAsAnd(runTransitionLikelihood.likelihood(context)); transProb.addResultAsAnd(nullStateLikelihood.likelihood(context)); transProb.addResultAsAnd(nullLocationLikelihood.likelihood(context)); transProb.addResultAsAnd(movedLikelihood.likelihood(context)); /* * TODO: this is mainly for debug and can/should be removed. */ transProb.addLogResultAsAnd(newEdge.getKey().name(), 0); final Particle newParticle = new Particle(timestamp, parent.getElement(), 0.0, newState); newParticle.setResult(transProb); if (ParticleFilter.getDebugEnabled()) { final double logWeight = parent.getElement().getLogWeight() + newParticle.getResult().getLogProbability(); newParticle.setLogWeight(logWeight); debugTransitions.add(newParticle); } transitionDist.logPut(transProb.getLogProbability(), newParticle); } final Particle newParticle; if (transitionDist.canSample()) { newParticle = transitionDist.sample(); final double logWeight = parent.getElement().getLogWeight() + newParticle.getResult().getLogProbability(); newParticle.setLogWeight(logWeight); } else { final SensorModelResult transProb = new SensorModelResult("Transition (null)"); final double inMotionSample = ParticleFactoryImpl.getThreadLocalRng().get().nextDouble(); final boolean vehicleNotMoved = inMotionSample < vehicleHasNotMovedProb; final MotionState motionState = updateMotionState(parentState, obs, vehicleNotMoved); final JourneyState journeyState = _journeyStateTransitionModel.getJourneyState(null, null, obs, vehicleNotMoved); final VehicleState nullState = new VehicleState(motionState, null, journeyState, null, obs); final Context context = new Context(parentState, nullState, obs); transProb.addResultAsAnd(edgeLikelihood.likelihood(context)); transProb.addResultAsAnd(gpsLikelihood.likelihood(context)); transProb.addResultAsAnd(dscLikelihood.likelihood(context)); transProb.addResultAsAnd(runLikelihood.likelihood(context)); transProb.addResultAsAnd(schedLikelihood.likelihood(context)); transProb.addResultAsAnd(runTransitionLikelihood.likelihood(context)); transProb.addResultAsAnd(nullStateLikelihood.likelihood(context)); transProb.addResultAsAnd(nullLocationLikelihood.likelihood(context)); transProb.addResultAsAnd(movedLikelihood.likelihood(context)); newParticle = new Particle(timestamp, parent.getElement(), 0.0, nullState); newParticle.setResult(transProb); final double logWeight = parent.getElement().getLogWeight() + newParticle.getResult().getLogProbability(); newParticle.setLogWeight(logWeight); } if (ParticleFilter.getDebugEnabled()) newParticle.setTransitions(debugTransitions); return newParticle; }