List of usage examples for com.google.common.collect Multimap toString
public String toString()
From source file:org.yakindu.sct.model.stext.expressions.STextExpressionParser.java
public EObject parseExpression(String expression, String ruleName, String specification) { StextResource resource = getResource(); resource.setURI(URI.createURI("path", true)); ParserRule parserRule = XtextFactory.eINSTANCE.createParserRule(); parserRule.setName(ruleName);//from ww w . j a v a 2 s . co m IParseResult result = parser.parse(parserRule, new StringReader(expression)); EObject rootASTElement = result.getRootASTElement(); resource.getContents().add(rootASTElement); ListBasedDiagnosticConsumer diagnosticsConsumer = new ListBasedDiagnosticConsumer(); Statechart sc = SGraphFactory.eINSTANCE.createStatechart(); sc.setDomainID(domainId); sc.setName("sc"); if (specification != null) { sc.setSpecification(specification); } resource.getContents().add(sc); linker.linkModel(sc, diagnosticsConsumer); linker.linkModel(rootASTElement, diagnosticsConsumer); resource.resolveLazyCrossReferences(CancelIndicator.NullImpl); resource.resolveLazyCrossReferences(CancelIndicator.NullImpl); Multimap<SpecificationElement, Diagnostic> diagnostics = resource.getLinkingDiagnostics(); if (diagnostics.size() > 0) { throw new LinkingException(diagnostics.toString()); } if (result.hasSyntaxErrors()) { StringBuilder errorMessages = new StringBuilder(); Iterable<INode> syntaxErrors = result.getSyntaxErrors(); for (INode iNode : syntaxErrors) { errorMessages.append(iNode.getSyntaxErrorMessage()); errorMessages.append("\n"); } throw new SyntaxException("Could not parse expression, syntax errors: " + errorMessages); } if (diagnosticsConsumer.hasConsumedDiagnostics(Severity.ERROR)) { throw new LinkingException("Error during linking: " + diagnosticsConsumer.getResult(Severity.ERROR)); } return rootASTElement; }
From source file:io.soliton.shapeshifter.ProtoDescriptorGraph.java
@Override public String toString() { Multimap<String, String> namedGraph = HashMultimap.create(); for (Map.Entry<Descriptor, Descriptor> entry : graph.entries()) { namedGraph.put(entry.getKey().getName(), entry.getValue().getName()); }//from w w w . j av a2 s. c om return namedGraph.toString(); }
From source file:com.github.benmanes.multiway.TransferPool.java
@Override public String toString() { Multimap<K, R> multimap = ArrayListMultimap.create(); for (Entry<ResourceKey<K>, R> entry : cache.entrySet()) { multimap.put(entry.getKey().getKey(), entry.getValue()); }/*from w w w. j a v a 2 s. co m*/ return multimap.toString(); }
From source file:de.esukom.decoit.ifmapclient.iptables.PollResultChecker.java
/** * check if passed in entry from poll-result-meta-data matches an "allowance" or an enforcement * filter-strings from enforcement.properties * // w ww .ja v a2s. c o m * @param filterStrings map of rules to check against * @param resultPropertyValue the property of the entry to check * @param resultFilterValue value to compare to entry-content * @param checkType type of check to perform (contains/matches) * * @return true, if passed in result-filter-value is contained or matched with entry inside * filter-string--map */ public boolean checkPollResultForIPTablesStringRules(Multimap<String, String> filterStrings, String resultPropertyValue, String resultFilterValue, byte checkType, byte resultType) { IfMapClient.LOGGER.fine("checking poll result for string rules..."); IfMapClient.LOGGER.fine("resultPropertyValue: " + resultPropertyValue); IfMapClient.LOGGER.fine("resultFilterValue: " + resultFilterValue); IfMapClient.LOGGER.fine("filterStrings: " + filterStrings.toString()); // contains-match if (checkType == RULECHECK_TYPE_CONTAINS) { if (resultType == RESULTCHECK_TYPE_SEARCHRESULT) { if (filterStrings.containsKey("role_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "role_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!role_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "!role_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("capability_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "capability_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!capability_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "!capability_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("device-characteristic_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "device-characteristic_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!device-characteristic_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "!device-characteristic_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("access-request-ip_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "access-request-ip_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!access-request-ip_" + resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, "!access-request-ip_" + resultPropertyValue, resultFilterValue); } } else { if (filterStrings.containsKey(resultPropertyValue)) { return checkStringAttributeContainsValue(filterStrings, resultPropertyValue, resultFilterValue); } } } // match-match ;-) else /* if (checkType == RULECHECK_TYPE_MATCHES) */ { if (resultType == RESULTCHECK_TYPE_SEARCHRESULT) { if (filterStrings.containsKey("role_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "role_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!role_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "!role_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("capability_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "capability_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!capability_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "!capability_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("device-characteristic_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "device-characteristic_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!device-characteristic_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "!device-characteristic_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("access-request-ip_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "access-request-ip_" + resultPropertyValue, resultFilterValue); } else if (filterStrings.containsKey("!access-request-ip_" + resultPropertyValue)) { return checkStringAttributeMatchesValue(filterStrings, "!access-request-ip_" + resultPropertyValue, resultFilterValue); } } else { return checkStringAttributeMatchesValue(filterStrings, resultPropertyValue, resultFilterValue); } } return false; }
From source file:com.alibaba.otter.node.etl.transform.transformer.RowDataTransformer.java
/** * ???manager???//www. ja va 2s .c o m */ private String translateColumnName(String srcColumnName, DataMediaPair dataMediaPair, Multimap<String, String> translateDict) { if (dataMediaPair.getColumnPairMode().isExclude() || CollectionUtils.isEmpty(dataMediaPair.getColumnPairs())) { return srcColumnName; // ??? } Collection<String> tColumnNames = translateDict.get(srcColumnName); if (CollectionUtils.isEmpty(tColumnNames)) { throw new TransformException( srcColumnName + " is not found in column pairs: " + translateDict.toString()); } String columnName = tColumnNames.iterator().next(); return columnName; }
From source file:org.summer.dsl.model.types.util.TypeConformanceComputer.java
protected JvmTypeReference getFirstForRawType(Multimap<JvmType, JvmTypeReference> all, JvmType rawType) { Iterator<JvmTypeReference> iterator = all.get(rawType).iterator(); while (iterator.hasNext()) { JvmTypeReference result = iterator.next(); if (result instanceof JvmParameterizedTypeReference || result instanceof JvmGenericArrayTypeReference) { return result; }//from w ww . j a v a2 s . c o m } throw new IllegalStateException(all.toString() + " does not contain a useful type reference for rawtype " + rawType.getQualifiedName()); }
From source file:org.summer.dsl.xbase.typesystem.conformance.TypeConformanceComputer.java
protected LightweightTypeReference getFirstForRawType(Multimap<JvmType, LightweightTypeReference> all, JvmType rawType) {/* ww w.j a v a 2s . com*/ Iterator<LightweightTypeReference> iterator = all.get(rawType).iterator(); while (iterator.hasNext()) { LightweightTypeReference result = iterator.next(); if (result instanceof ParameterizedTypeReference || result instanceof ArrayTypeReference) { return result; } } throw new IllegalStateException(all.toString() + " does not contain a useful type reference for rawtype " + rawType.getIdentifier()); }
From source file:de.hzi.helmholtz.Compare.PathwayComparisonWithModules.java
public Multimap<Double, String> SubsetsMatching(final PathwayWithModules firstPathway, final PathwayWithModules secondPathway, BiMap<Integer, Integer> newSourceGeneIdToPositionMap, BiMap<Integer, Integer> newTargetGeneIdToPositionMap, int Yes) { Multimap<Double, String> resultPerfect = TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural());// w ww .j av a2 s . co m PathwayWithModules firstPathwayCopy = new PathwayWithModules(firstPathway);// Copy of the Query pathway PathwayWithModules secondPathwayCopy = new PathwayWithModules(secondPathway);// Copy of the Target pathway' // PathwayWithModules secondPathwayCopy1 = new PathwayWithModules(secondPathway); int currentQueryGene = 0; Iterator<ModuleGene> sourceGeneIt = firstPathway.moduleGeneIterator(); List<Integer> QueryToRemove = new ArrayList<Integer>(); List<Integer> TargetToRemove = new ArrayList<Integer>(); while (sourceGeneIt.hasNext()) { currentQueryGene++; ModuleGene queryGene = sourceGeneIt.next(); int currentTargetGene = 0; Multiset<String> qfunction = LinkedHashMultiset.create(); List<String> qfunctionList = new ArrayList<String>(); List<String> qactivity = new ArrayList<String>(); List<Set<String>> qsubstrate = new ArrayList<Set<String>>(); for (Module m : queryGene.getModule()) { for (Domain d : m.getDomains()) { qfunction.add(d.getDomainFunctionString()); qfunctionList.add(d.getDomainFunctionString()); qactivity.add(d.getStatus().toString()); qsubstrate.add(d.getSubstrates()); } } Iterator<ModuleGene> targetGeneIt = secondPathway.moduleGeneIterator(); while (targetGeneIt.hasNext()) { currentTargetGene++; ModuleGene targetGene = targetGeneIt.next(); Multiset<String> tfunction = LinkedHashMultiset.create(); List<String> tfunctionList = new ArrayList<String>(); List<String> tactivity = new ArrayList<String>(); List<Set<String>> tsubstrate = new ArrayList<Set<String>>(); for (Module m : targetGene.getModule()) { for (Domain d : m.getDomains()) { tfunctionList.add(d.getDomainFunctionString()); tfunction.add(d.getDomainFunctionString()); tactivity.add(d.getStatus().toString()); tsubstrate.add(d.getSubstrates()); } } Multiset<String> DomainsCovered = Multisets.intersection(qfunction, tfunction); if (DomainsCovered.size() == qfunction.size() && DomainsCovered.size() == tfunction.size()) { Multimap<Double, Multimap<String, Integer>> activityscores = myFunction.calculate(qactivity, tactivity); Multimap<String, Integer> Functionscores = ArrayListMultimap.create(); int TranspositionDomains = LevenshteinDistance.computeLevenshteinDistance(qfunctionList, tfunctionList); if (TranspositionDomains > 0) { TranspositionDomains = 1; } Functionscores.put(qfunction.size() + "-0", TranspositionDomains); Multimap<Double, Multimap<String, Integer>> substratescore = myFunction .calculate(getSubstrateList(qsubstrate), getSubstrateList(tsubstrate)); Object activityScore = activityscores.asMap().keySet().toArray()[0]; Object substrateScore = substratescore.asMap().keySet().toArray()[0]; double finalScore = Math .round((((2.9 * 1.0) + (0.05 * Double.parseDouble(activityScore.toString().trim())) + (0.05 * Double.parseDouble(substrateScore.toString().trim()))) / 3) * 100.0) / 100.0; String ConvertedGeneIDs = ""; if (Yes == 0) { ConvertedGeneIDs = reconstructWithGeneId(Integer.toString(currentQueryGene), newSourceGeneIdToPositionMap) + "->" + reconstructWithGeneId(Integer.toString(currentTargetGene), newTargetGeneIdToPositionMap); } else { ConvertedGeneIDs = reconstructWithGeneId(Integer.toString(currentTargetGene), newTargetGeneIdToPositionMap) + "->" + reconstructWithGeneId(Integer.toString(currentQueryGene), newSourceGeneIdToPositionMap); } resultPerfect.put(finalScore, ConvertedGeneIDs); ScoreFunctionMatchMisMatch.put(ConvertedGeneIDs, Functionscores); ScoreStatusMatchMisMatch.putAll(ConvertedGeneIDs, activityscores.values()); ScoreSubstrateMatchMisMatch.putAll(ConvertedGeneIDs, substratescore.values()); TargetToRemove.add(currentTargetGene); QueryToRemove.add(currentQueryGene); } } } for (int i : TargetToRemove) { secondPathwayCopy.removeGene(i); } for (int i : QueryToRemove) { firstPathwayCopy.removeGene(i); } if (firstPathwayCopy.size() > 0 && secondPathwayCopy.size() > 0) { // Re-construct the bimaps newSourceGeneIdToPositionMap = HashBiMap.create(); int temp = 0; for (ModuleGene e : firstPathwayCopy.getModulegenes()) { temp = temp + 1; newSourceGeneIdToPositionMap.put(e.getGeneId(), temp); } newTargetGeneIdToPositionMap = HashBiMap.create(); temp = 0; for (ModuleGene e : secondPathwayCopy.getModulegenes()) { temp = temp + 1; newTargetGeneIdToPositionMap.put(e.getGeneId(), temp); } resultPerfect.putAll(SubsetIdentification(firstPathwayCopy, secondPathwayCopy, newSourceGeneIdToPositionMap, newTargetGeneIdToPositionMap, Yes)); } System.out.println(resultPerfect); return resultPerfect; }
From source file:de.hzi.helmholtz.Compare.PathwayComparisonUsingModules.java
public Multimap<Double, String> SubsetsMatching(final PathwayUsingModules firstPathway, final PathwayUsingModules secondPathway, BiMap<String, Integer> newSourceGeneIdToPositionMap, BiMap<String, Integer> newTargetGeneIdToPositionMap, int Yes) { Multimap<Double, String> resultPerfect = TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural());/*from w ww. j a va2 s . c om*/ PathwayUsingModules firstPathwayCopy = new PathwayUsingModules(firstPathway);// Copy of the Query pathway PathwayUsingModules secondPathwayCopy = new PathwayUsingModules(secondPathway);// Copy of the Target pathway' // PathwayUsingModules secondPathwayCopy1 = new PathwayUsingModules(secondPathway); int currentQueryGene = 0; Iterator<Module> sourceGeneIt = firstPathway.geneIterator(); List<String> QueryToRemove = new ArrayList<String>(); List<String> TargetToRemove = new ArrayList<String>(); while (sourceGeneIt.hasNext()) { currentQueryGene++; Module queryGene = sourceGeneIt.next(); int currentTargetGene = 0; Multiset<String> qfunction = LinkedHashMultiset.create(); List<String> qfunctionList = new ArrayList<String>(); List<String> qactivity = new ArrayList<String>(); List<Set<String>> qsubstrate = new ArrayList<Set<String>>(); for (Domain d : queryGene.getDomains()) { qfunction.add(d.getDomainFunctionString()); qfunctionList.add(d.getDomainFunctionString()); qactivity.add(d.getStatus().toString()); qsubstrate.add(d.getSubstrates()); } Iterator<Module> targetGeneIt = secondPathway.geneIterator(); while (targetGeneIt.hasNext()) { currentTargetGene++; Module targetGene = targetGeneIt.next(); Multiset<String> tfunction = LinkedHashMultiset.create(); List<String> tfunctionList = new ArrayList<String>(); List<String> tactivity = new ArrayList<String>(); List<Set<String>> tsubstrate = new ArrayList<Set<String>>(); for (Domain d : targetGene.getDomains()) { tfunctionList.add(d.getDomainFunctionString()); tfunction.add(d.getDomainFunctionString()); tactivity.add(d.getStatus().toString()); tsubstrate.add(d.getSubstrates()); } Multiset<String> DomainsCovered = Multisets.intersection(qfunction, tfunction); if (DomainsCovered.size() == qfunction.size() && DomainsCovered.size() == tfunction.size()) { Multimap<Double, Multimap<String, Integer>> activityscores = myFunction.calculate(qactivity, tactivity); Multimap<String, Integer> Functionscores = ArrayListMultimap.create(); int TranspositionDomains = LevenshteinDistance.computeLevenshteinDistance(qfunctionList, tfunctionList); if (TranspositionDomains > 0) { TranspositionDomains = 1; } Functionscores.put(qfunction.size() + "-0", TranspositionDomains); Multimap<Double, Multimap<String, Integer>> substratescore = myFunction .calculate(getSubstrateList(qsubstrate), getSubstrateList(tsubstrate)); Object activityScore = activityscores.asMap().keySet().toArray()[0]; Object substrateScore = substratescore.asMap().keySet().toArray()[0]; double finalScore = Math .round((((2.9 * 1.0) + (0.05 * Double.parseDouble(activityScore.toString().trim())) + (0.05 * Double.parseDouble(substrateScore.toString().trim()))) / 3) * 100.0) / 100.0; String ConvertedGeneIDs = ""; if (Yes == 0) { ConvertedGeneIDs = reconstructWithGeneId(Integer.toString(currentQueryGene), newSourceGeneIdToPositionMap) + "->" + reconstructWithGeneId(Integer.toString(currentTargetGene), newTargetGeneIdToPositionMap); } else { ConvertedGeneIDs = reconstructWithGeneId(Integer.toString(currentTargetGene), newTargetGeneIdToPositionMap) + "->" + reconstructWithGeneId(Integer.toString(currentQueryGene), newSourceGeneIdToPositionMap); } resultPerfect.put(finalScore, ConvertedGeneIDs); ScoreFunctionMatchMisMatch.put(ConvertedGeneIDs, Functionscores); ScoreStatusMatchMisMatch.putAll(ConvertedGeneIDs, activityscores.values()); ScoreSubstrateMatchMisMatch.putAll(ConvertedGeneIDs, substratescore.values()); TargetToRemove.add(reconstructWithGeneId(Integer.toString(currentTargetGene), newTargetGeneIdToPositionMap)); QueryToRemove.add(reconstructWithGeneId(Integer.toString(currentQueryGene), newSourceGeneIdToPositionMap)); } } } for (String i : TargetToRemove) { secondPathwayCopy.removeModule(i); } for (String i : QueryToRemove) { firstPathwayCopy.removeModule(i); } if (firstPathwayCopy.size() > 0 && secondPathwayCopy.size() > 0) { // Re-construct the bimaps newSourceGeneIdToPositionMap = HashBiMap.create(); int temp = 0; for (Module e : firstPathwayCopy.getModules()) { temp = temp + 1; newSourceGeneIdToPositionMap.put(e.getModuleId(), temp); } newTargetGeneIdToPositionMap = HashBiMap.create(); temp = 0; for (Module e : secondPathwayCopy.getModules()) { temp = temp + 1; newTargetGeneIdToPositionMap.put(e.getModuleId(), temp); } resultPerfect.putAll(SubsetIdentification(firstPathwayCopy, secondPathwayCopy, newSourceGeneIdToPositionMap, newTargetGeneIdToPositionMap, Yes)); } ////System.out.println(resultPerfect); return resultPerfect; }
From source file:org.apache.accumulo.examples.wikisearch.logic.AbstractQueryLogic.java
public Results runQuery(Connector connector, List<String> authorizations, String query, Date beginDate, Date endDate, Set<String> types) { if (StringUtils.isEmpty(query)) { throw new IllegalArgumentException( "NULL QueryNode reference passed to " + this.getClass().getSimpleName()); }/*from w w w . j a v a 2 s . c om*/ Set<Range> ranges = new HashSet<Range>(); Set<String> typeFilter = types; String array[] = authorizations.toArray(new String[0]); Authorizations auths = new Authorizations(array); Results results = new Results(); // Get the query string String queryString = query; StopWatch abstractQueryLogic = new StopWatch(); StopWatch optimizedQuery = new StopWatch(); StopWatch queryGlobalIndex = new StopWatch(); StopWatch optimizedEventQuery = new StopWatch(); StopWatch fullScanQuery = new StopWatch(); StopWatch processResults = new StopWatch(); abstractQueryLogic.start(); StopWatch parseQuery = new StopWatch(); parseQuery.start(); QueryParser parser; try { if (log.isDebugEnabled()) { log.debug("ShardQueryLogic calling QueryParser.execute"); } parser = new QueryParser(); parser.execute(queryString); } catch (org.apache.commons.jexl2.parser.ParseException e1) { throw new IllegalArgumentException("Error parsing query", e1); } int hash = parser.getHashValue(); parseQuery.stop(); if (log.isDebugEnabled()) { log.debug(hash + " Query: " + queryString); } Set<String> fields = new HashSet<String>(); for (String f : parser.getQueryIdentifiers()) { fields.add(f); } if (log.isDebugEnabled()) { log.debug("getQueryIdentifiers: " + parser.getQueryIdentifiers().toString()); } // Remove any negated fields from the fields list, we don't want to lookup negated fields // in the index. fields.removeAll(parser.getNegatedTermsForOptimizer()); if (log.isDebugEnabled()) { log.debug("getQueryIdentifiers: " + parser.getQueryIdentifiers().toString()); } // Get the mapping of field name to QueryTerm object from the query. The query term object // contains the operator, whether its negated or not, and the literal to test against. Multimap<String, QueryTerm> terms = parser.getQueryTerms(); // Find out which terms are indexed // TODO: Should we cache indexed terms or does that not make sense since we are always // loading data. StopWatch queryMetadata = new StopWatch(); queryMetadata.start(); Map<String, Multimap<String, Class<? extends Normalizer>>> metadataResults; try { metadataResults = findIndexedTerms(connector, auths, fields, typeFilter); } catch (Exception e1) { throw new RuntimeException("Error in metadata lookup", e1); } // Create a map of indexed term to set of normalizers for it Multimap<String, Normalizer> indexedTerms = HashMultimap.create(); for (Entry<String, Multimap<String, Class<? extends Normalizer>>> entry : metadataResults.entrySet()) { // Get the normalizer from the normalizer cache for (Class<? extends Normalizer> clazz : entry.getValue().values()) { indexedTerms.put(entry.getKey(), normalizerCacheMap.get(clazz)); } } queryMetadata.stop(); if (log.isDebugEnabled()) { log.debug(hash + " Indexed Terms: " + indexedTerms.toString()); } Set<String> orTerms = parser.getOrTermsForOptimizer(); // Iterate over the query terms to get the operators specified in the query. ArrayList<String> unevaluatedExpressions = new ArrayList<String>(); boolean unsupportedOperatorSpecified = false; for (Entry<String, QueryTerm> entry : terms.entries()) { if (null == entry.getValue()) { continue; } if (null != this.unevaluatedFields && this.unevaluatedFields.contains(entry.getKey().trim())) { unevaluatedExpressions.add(entry.getKey().trim() + " " + entry.getValue().getOperator() + " " + entry.getValue().getValue()); } int operator = JexlOperatorConstants.getJJTNodeType(entry.getValue().getOperator()); if (!(operator == ParserTreeConstants.JJTEQNODE || operator == ParserTreeConstants.JJTNENODE || operator == ParserTreeConstants.JJTLENODE || operator == ParserTreeConstants.JJTLTNODE || operator == ParserTreeConstants.JJTGENODE || operator == ParserTreeConstants.JJTGTNODE || operator == ParserTreeConstants.JJTERNODE)) { unsupportedOperatorSpecified = true; break; } } if (null != unevaluatedExpressions) unevaluatedExpressions.trimToSize(); if (log.isDebugEnabled()) { log.debug(hash + " unsupportedOperators: " + unsupportedOperatorSpecified + " indexedTerms: " + indexedTerms.toString() + " orTerms: " + orTerms.toString() + " unevaluatedExpressions: " + unevaluatedExpressions.toString()); } // We can use the intersecting iterator over the field index as an optimization under the // following conditions // // 1. No unsupported operators in the query. // 2. No 'or' operators and at least one term indexed // or // 1. No unsupported operators in the query. // 2. and all terms indexed // or // 1. All or'd terms are indexed. NOTE, this will potentially skip some queries and push to a full table scan // // WE should look into finding a better way to handle whether we do an optimized query or not. boolean optimizationSucceeded = false; boolean orsAllIndexed = false; if (orTerms.isEmpty()) { orsAllIndexed = false; } else { orsAllIndexed = indexedTerms.keySet().containsAll(orTerms); } if (log.isDebugEnabled()) { log.debug("All or terms are indexed"); } if (!unsupportedOperatorSpecified && (((null == orTerms || orTerms.isEmpty()) && indexedTerms.size() > 0) || (fields.size() > 0 && indexedTerms.size() == fields.size()) || orsAllIndexed)) { optimizedQuery.start(); // Set up intersecting iterator over field index. // Get information from the global index for the indexed terms. The results object will contain the term // mapped to an object that contains the total count, and partitions where this term is located. // TODO: Should we cache indexed term information or does that not make sense since we are always loading data queryGlobalIndex.start(); IndexRanges termIndexInfo; try { // If fields is null or zero, then it's probably the case that the user entered a value // to search for with no fields. Check for the value in index. if (fields.isEmpty()) { termIndexInfo = this.getTermIndexInformation(connector, auths, queryString, typeFilter); if (null != termIndexInfo && termIndexInfo.getRanges().isEmpty()) { // Then we didn't find anything in the index for this query. This may happen for an indexed term that has wildcards // in unhandled locations. // Break out of here by throwing a named exception and do full scan throw new DoNotPerformOptimizedQueryException(); } // We need to rewrite the query string here so that it's valid. if (termIndexInfo instanceof UnionIndexRanges) { UnionIndexRanges union = (UnionIndexRanges) termIndexInfo; StringBuilder buf = new StringBuilder(); String sep = ""; for (String fieldName : union.getFieldNamesAndValues().keySet()) { buf.append(sep).append(fieldName).append(" == "); if (!(queryString.startsWith("'") && queryString.endsWith("'"))) { buf.append("'").append(queryString).append("'"); } else { buf.append(queryString); } sep = " or "; } if (log.isDebugEnabled()) { log.debug("Rewrote query for non-fielded single term query: " + queryString + " to " + buf.toString()); } queryString = buf.toString(); } else { throw new RuntimeException("Unexpected IndexRanges implementation"); } } else { RangeCalculator calc = this.getTermIndexInformation(connector, auths, indexedTerms, terms, this.getIndexTableName(), this.getReverseIndexTableName(), queryString, this.queryThreads, typeFilter); if (null == calc.getResult() || calc.getResult().isEmpty()) { // Then we didn't find anything in the index for this query. This may happen for an indexed term that has wildcards // in unhandled locations. // Break out of here by throwing a named exception and do full scan throw new DoNotPerformOptimizedQueryException(); } termIndexInfo = new UnionIndexRanges(); termIndexInfo.setIndexValuesToOriginalValues(calc.getIndexValues()); termIndexInfo.setFieldNamesAndValues(calc.getIndexEntries()); termIndexInfo.getTermCardinality().putAll(calc.getTermCardinalities()); for (Range r : calc.getResult()) { // foo is a placeholder and is ignored. termIndexInfo.add("foo", r); } } } catch (TableNotFoundException e) { log.error(this.getIndexTableName() + "not found", e); throw new RuntimeException(this.getIndexTableName() + "not found", e); } catch (org.apache.commons.jexl2.parser.ParseException e) { throw new RuntimeException("Error determining ranges for query: " + queryString, e); } catch (DoNotPerformOptimizedQueryException e) { log.info("Indexed fields not found in index, performing full scan"); termIndexInfo = null; } queryGlobalIndex.stop(); // Determine if we should proceed with optimized query based on results from the global index boolean proceed = false; if (null == termIndexInfo || termIndexInfo.getFieldNamesAndValues().values().size() == 0) { proceed = false; } else if (null != orTerms && orTerms.size() > 0 && (termIndexInfo.getFieldNamesAndValues().values().size() == indexedTerms.size())) { proceed = true; } else if (termIndexInfo.getFieldNamesAndValues().values().size() > 0) { proceed = true; } else if (orsAllIndexed) { proceed = true; } else { proceed = false; } if (log.isDebugEnabled()) { log.debug("Proceed with optimized query: " + proceed); if (null != termIndexInfo) log.debug("termIndexInfo.getTermsFound().size(): " + termIndexInfo.getFieldNamesAndValues().values().size() + " indexedTerms.size: " + indexedTerms.size() + " fields.size: " + fields.size()); } if (proceed) { if (log.isDebugEnabled()) { log.debug(hash + " Performing optimized query"); } // Use the scan ranges from the GlobalIndexRanges object as the ranges for the batch scanner ranges = termIndexInfo.getRanges(); if (log.isDebugEnabled()) { log.info(hash + " Ranges: count: " + ranges.size() + ", " + ranges.toString()); } // Create BatchScanner, set the ranges, and setup the iterators. optimizedEventQuery.start(); BatchScanner bs = null; try { bs = connector.createBatchScanner(this.getTableName(), auths, queryThreads); bs.setRanges(ranges); IteratorSetting si = new IteratorSetting(21, "eval", OptimizedQueryIterator.class); if (log.isDebugEnabled()) { log.debug("Setting scan option: " + EvaluatingIterator.QUERY_OPTION + " to " + queryString); } // Set the query option si.addOption(EvaluatingIterator.QUERY_OPTION, queryString); // Set the Indexed Terms List option. This is the field name and normalized field value pair separated // by a comma. StringBuilder buf = new StringBuilder(); String sep = ""; for (Entry<String, String> entry : termIndexInfo.getFieldNamesAndValues().entries()) { buf.append(sep); buf.append(entry.getKey()); buf.append(":"); buf.append(termIndexInfo.getIndexValuesToOriginalValues().get(entry.getValue())); buf.append(":"); buf.append(entry.getValue()); if (sep.equals("")) { sep = ";"; } } if (log.isDebugEnabled()) { log.debug("Setting scan option: " + FieldIndexQueryReWriter.INDEXED_TERMS_LIST + " to " + buf.toString()); } FieldIndexQueryReWriter rewriter = new FieldIndexQueryReWriter(); String q = ""; try { q = queryString; q = rewriter.applyCaseSensitivity(q, true, false);// Set upper/lower case for fieldname/fieldvalue Map<String, String> opts = new HashMap<String, String>(); opts.put(FieldIndexQueryReWriter.INDEXED_TERMS_LIST, buf.toString()); q = rewriter.removeNonIndexedTermsAndInvalidRanges(q, opts); q = rewriter.applyNormalizedTerms(q, opts); if (log.isDebugEnabled()) { log.debug("runServerQuery, FieldIndex Query: " + q); } } catch (org.apache.commons.jexl2.parser.ParseException ex) { log.error("Could not parse query, Jexl ParseException: " + ex); } catch (Exception ex) { log.error("Problem rewriting query, Exception: " + ex.getMessage()); } si.addOption(BooleanLogicIterator.FIELD_INDEX_QUERY, q); // Set the term cardinality option sep = ""; buf.delete(0, buf.length()); for (Entry<String, Long> entry : termIndexInfo.getTermCardinality().entrySet()) { buf.append(sep); buf.append(entry.getKey()); buf.append(":"); buf.append(entry.getValue()); sep = ","; } if (log.isDebugEnabled()) log.debug("Setting scan option: " + BooleanLogicIterator.TERM_CARDINALITIES + " to " + buf.toString()); si.addOption(BooleanLogicIterator.TERM_CARDINALITIES, buf.toString()); if (this.useReadAheadIterator) { if (log.isDebugEnabled()) { log.debug("Enabling read ahead iterator with queue size: " + this.readAheadQueueSize + " and timeout: " + this.readAheadTimeOut); } si.addOption(ReadAheadIterator.QUEUE_SIZE, this.readAheadQueueSize); si.addOption(ReadAheadIterator.TIMEOUT, this.readAheadTimeOut); } if (null != unevaluatedExpressions) { StringBuilder unevaluatedExpressionList = new StringBuilder(); String sep2 = ""; for (String exp : unevaluatedExpressions) { unevaluatedExpressionList.append(sep2).append(exp); sep2 = ","; } if (log.isDebugEnabled()) log.debug("Setting scan option: " + EvaluatingIterator.UNEVALUTED_EXPRESSIONS + " to " + unevaluatedExpressionList.toString()); si.addOption(EvaluatingIterator.UNEVALUTED_EXPRESSIONS, unevaluatedExpressionList.toString()); } bs.addScanIterator(si); processResults.start(); processResults.suspend(); long count = 0; for (Entry<Key, Value> entry : bs) { count++; // The key that is returned by the EvaluatingIterator is not the same key that is in // the table. The value that is returned by the EvaluatingIterator is a kryo // serialized EventFields object. processResults.resume(); Document d = this.createDocument(entry.getKey(), entry.getValue()); results.getResults().add(d); processResults.suspend(); } log.info(count + " matching entries found in optimized query."); optimizationSucceeded = true; processResults.stop(); } catch (TableNotFoundException e) { log.error(this.getTableName() + "not found", e); throw new RuntimeException(this.getIndexTableName() + "not found", e); } finally { if (bs != null) { bs.close(); } } optimizedEventQuery.stop(); } optimizedQuery.stop(); } // WE should look into finding a better way to handle whether we do an optimized query or not. // We are not setting up an else condition here because we may have aborted the logic early in the if statement. if (!optimizationSucceeded || ((null != orTerms && orTerms.size() > 0) && (indexedTerms.size() != fields.size()) && !orsAllIndexed)) { // if (!optimizationSucceeded || ((null != orTerms && orTerms.size() > 0) && (indexedTerms.size() != fields.size()))) { fullScanQuery.start(); if (log.isDebugEnabled()) { log.debug(hash + " Performing full scan query"); } // Set up a full scan using the date ranges from the query // Create BatchScanner, set the ranges, and setup the iterators. BatchScanner bs = null; try { // The ranges are the start and end dates Collection<Range> r = getFullScanRange(beginDate, endDate, terms); ranges.addAll(r); if (log.isDebugEnabled()) { log.debug(hash + " Ranges: count: " + ranges.size() + ", " + ranges.toString()); } bs = connector.createBatchScanner(this.getTableName(), auths, queryThreads); bs.setRanges(ranges); IteratorSetting si = new IteratorSetting(22, "eval", EvaluatingIterator.class); // Create datatype regex if needed if (null != typeFilter) { StringBuilder buf = new StringBuilder(); String s = ""; for (String type : typeFilter) { buf.append(s).append(type).append(".*"); s = "|"; } if (log.isDebugEnabled()) log.debug("Setting colf regex iterator to: " + buf.toString()); IteratorSetting ri = new IteratorSetting(21, "typeFilter", RegExFilter.class); RegExFilter.setRegexs(ri, null, buf.toString(), null, null, false); bs.addScanIterator(ri); } if (log.isDebugEnabled()) { log.debug("Setting scan option: " + EvaluatingIterator.QUERY_OPTION + " to " + queryString); } si.addOption(EvaluatingIterator.QUERY_OPTION, queryString); if (null != unevaluatedExpressions) { StringBuilder unevaluatedExpressionList = new StringBuilder(); String sep2 = ""; for (String exp : unevaluatedExpressions) { unevaluatedExpressionList.append(sep2).append(exp); sep2 = ","; } if (log.isDebugEnabled()) log.debug("Setting scan option: " + EvaluatingIterator.UNEVALUTED_EXPRESSIONS + " to " + unevaluatedExpressionList.toString()); si.addOption(EvaluatingIterator.UNEVALUTED_EXPRESSIONS, unevaluatedExpressionList.toString()); } bs.addScanIterator(si); long count = 0; processResults.start(); processResults.suspend(); for (Entry<Key, Value> entry : bs) { count++; // The key that is returned by the EvaluatingIterator is not the same key that is in // the partition table. The value that is returned by the EvaluatingIterator is a kryo // serialized EventFields object. processResults.resume(); Document d = this.createDocument(entry.getKey(), entry.getValue()); results.getResults().add(d); processResults.suspend(); } processResults.stop(); log.info(count + " matching entries found in full scan query."); } catch (TableNotFoundException e) { log.error(this.getTableName() + "not found", e); } finally { if (bs != null) { bs.close(); } } fullScanQuery.stop(); } log.info("AbstractQueryLogic: " + queryString + " " + timeString(abstractQueryLogic.getTime())); log.info(" 1) parse query " + timeString(parseQuery.getTime())); log.info(" 2) query metadata " + timeString(queryMetadata.getTime())); log.info(" 3) full scan query " + timeString(fullScanQuery.getTime())); log.info(" 3) optimized query " + timeString(optimizedQuery.getTime())); log.info(" 1) process results " + timeString(processResults.getTime())); log.info(" 1) query global index " + timeString(queryGlobalIndex.getTime())); log.info(hash + " Query completed."); return results; }