List of usage examples for org.apache.commons.lang3.mutable MutableObject MutableObject
public MutableObject(final T value)
From source file:org.apache.asterix.optimizer.rules.am.BTreeAccessMethod.java
@Override public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException { Dataset dataset = indexSubTree.getDataset(); ARecordType recordType = indexSubTree.getRecordType(); ARecordType metaRecordType = indexSubTree.getMetaRecordType(); // we made sure indexSubTree has datasource scan AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef() .getValue();/* w w w. j av a 2 s . c o m*/ List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.indexExprsAndVars.get(chosenIndex); List<IOptimizableFuncExpr> matchedFuncExprs = analysisCtx.matchedFuncExprs; int numSecondaryKeys = analysisCtx.indexNumMatchedKeys.get(chosenIndex); // List of function expressions that will be replaced by the secondary-index search. // These func exprs will be removed from the select condition at the very end of this method. Set<ILogicalExpression> replacedFuncExprs = new HashSet<>(); // Info on high and low keys for the BTree search predicate. ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys]; ILogicalExpression[] highKeyExprs = new ILogicalExpression[numSecondaryKeys]; LimitType[] lowKeyLimits = new LimitType[numSecondaryKeys]; LimitType[] highKeyLimits = new LimitType[numSecondaryKeys]; boolean[] lowKeyInclusive = new boolean[numSecondaryKeys]; boolean[] highKeyInclusive = new boolean[numSecondaryKeys]; ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys]; LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys]; /* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range * predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate, * we just bail by setting this flag.*/ boolean couldntFigureOut = false; boolean doneWithExprs = false; boolean isEqCondition = false; BitSet setLowKeys = new BitSet(numSecondaryKeys); BitSet setHighKeys = new BitSet(numSecondaryKeys); // Go through the func exprs listed as optimizable by the chosen index, // and formulate a range predicate on the secondary-index keys. // checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value // since we have a round issues when dealing with LT(<) OR GT(>) operator. boolean realTypeConvertedToIntegerType; for (Pair<Integer, Integer> exprIndex : exprAndVarList) { // Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs. IOptimizableFuncExpr optFuncExpr = matchedFuncExprs.get(exprIndex.first); int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames()); if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) { // If we are optimizing a join, the matching field may be the second field name. keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames()); } if (keyPos < 0) { throw new AlgebricksException( "Could not match optimizable function expression to any index field name."); } Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils .createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree); ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first; if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { constantAtRuntimeExpressions[keyPos] = searchKeyExpr; constAtRuntimeExprVars[keyPos] = context.newVar(); searchKeyExpr = new VariableReferenceExpression(constAtRuntimeExprVars[keyPos]); } realTypeConvertedToIntegerType = returnedSearchKeyExpr.second; LimitType limit = getLimitType(optFuncExpr, probeSubTree); // If a DOUBLE or FLOAT constant is converted to an INT type value, // we need to check a corner case where two real values are located between an INT value. // For example, for the following query, // // for $emp in dataset empDataset // where $emp.age > double("2.3") and $emp.age < double("3.3") // return $emp.id // // It should generate a result if there is a tuple that satisfies the condition, which is 3, // however, it does not generate the desired result since finding candidates // fail after truncating the fraction part (there is no INT whose value is greater than 2 and less than 3.) // // Therefore, we convert LT(<) to LE(<=) and GT(>) to GE(>=) to find candidates. // This does not change the result of an actual comparison since this conversion is only applied // for finding candidates from an index. // if (realTypeConvertedToIntegerType) { if (limit == LimitType.HIGH_EXCLUSIVE) { limit = LimitType.HIGH_INCLUSIVE; } else if (limit == LimitType.LOW_EXCLUSIVE) { limit = LimitType.LOW_INCLUSIVE; } } switch (limit) { case EQUAL: { if (lowKeyLimits[keyPos] == null && highKeyLimits[keyPos] == null) { lowKeyLimits[keyPos] = highKeyLimits[keyPos] = limit; lowKeyInclusive[keyPos] = highKeyInclusive[keyPos] = true; lowKeyExprs[keyPos] = highKeyExprs[keyPos] = searchKeyExpr; setLowKeys.set(keyPos); setHighKeys.set(keyPos); isEqCondition = true; } else { // Has already been set to the identical values. // When optimizing join we may encounter the same optimizable expression twice // (once from analyzing each side of the join) if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) { isEqCondition = true; break; } couldntFigureOut = true; } // TODO: For now don't consider prefix searches. // If high and low keys are set, we exit for now. if (setLowKeys.cardinality() == numSecondaryKeys && setHighKeys.cardinality() == numSecondaryKeys) { doneWithExprs = true; } break; } case HIGH_EXCLUSIVE: { if (highKeyLimits[keyPos] == null || (highKeyLimits[keyPos] != null && highKeyInclusive[keyPos])) { highKeyLimits[keyPos] = limit; highKeyExprs[keyPos] = searchKeyExpr; highKeyInclusive[keyPos] = false; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false && highKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } case HIGH_INCLUSIVE: { if (highKeyLimits[keyPos] == null) { highKeyLimits[keyPos] = limit; highKeyExprs[keyPos] = searchKeyExpr; highKeyInclusive[keyPos] = true; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } case LOW_EXCLUSIVE: { if (lowKeyLimits[keyPos] == null || (lowKeyLimits[keyPos] != null && lowKeyInclusive[keyPos])) { lowKeyLimits[keyPos] = limit; lowKeyExprs[keyPos] = searchKeyExpr; lowKeyInclusive[keyPos] = false; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false && lowKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } case LOW_INCLUSIVE: { if (lowKeyLimits[keyPos] == null) { lowKeyLimits[keyPos] = limit; lowKeyExprs[keyPos] = searchKeyExpr; lowKeyInclusive[keyPos] = true; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } default: { throw new IllegalStateException(); } } if (!couldntFigureOut) { // Remember to remove this funcExpr later. replacedFuncExprs.add(matchedFuncExprs.get(exprIndex.first).getFuncExpr()); } if (doneWithExprs) { break; } } if (couldntFigureOut) { return null; } // If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals // closed to obtain a superset of answers and leave the original selection in place. boolean primaryIndexPostProccessingIsNeeded = false; for (int i = 1; i < numSecondaryKeys; ++i) { if (lowKeyInclusive[i] != lowKeyInclusive[0]) { Arrays.fill(lowKeyInclusive, true); primaryIndexPostProccessingIsNeeded = true; break; } } for (int i = 1; i < numSecondaryKeys; ++i) { if (highKeyInclusive[i] != highKeyInclusive[0]) { Arrays.fill(highKeyInclusive, true); primaryIndexPostProccessingIsNeeded = true; break; } } // determine cases when prefix search could be applied for (int i = 1; i < lowKeyExprs.length; i++) { if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null || highKeyLimits[0] == null && highKeyLimits[i] != null || highKeyLimits[0] != null && highKeyLimits[i] == null) { numSecondaryKeys--; primaryIndexPostProccessingIsNeeded = true; } } if (lowKeyLimits[0] == null) { lowKeyInclusive[0] = true; } if (highKeyLimits[0] == null) { highKeyInclusive[0] = true; } // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index // search. // List of variables for the assign. ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>(); // List of variables and expressions for the assign. ArrayList<LogicalVariable> assignKeyVarList = new ArrayList<LogicalVariable>(); ArrayList<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<Mutable<ILogicalExpression>>(); int numLowKeys = createKeyVarsAndExprs(numSecondaryKeys, lowKeyLimits, lowKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars); int numHighKeys = createKeyVarsAndExprs(numSecondaryKeys, highKeyLimits, highKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars); BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast); jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]); jobGenParams.setHighKeyInclusive(highKeyInclusive[0]); jobGenParams.setIsEqCondition(isEqCondition); jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys); jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys); ILogicalOperator inputOp = null; if (!assignKeyVarList.isEmpty()) { // Assign operator that sets the constant secondary-index search-key fields if necessary. AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList); // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). assignConstantSearchKeys.getInputs().add(new MutableObject<ILogicalOperator>( OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue()))); assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode()); inputOp = assignConstantSearchKeys; } else { // All index search keys are variables. inputOp = probeSubTree.getRoot(); } ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput, retainNull); // Generate the rest of the upstream plan which feeds the search results into the primary index. AbstractUnnestMapOperator primaryIndexUnnestOp = null; boolean isPrimaryIndex = chosenIndex.isPrimaryIndex(); if (dataset.getDatasetType() == DatasetType.EXTERNAL) { // External dataset UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap( dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull); indexSubTree.getDataSourceRef().setValue(externalDataAccessOp); return externalDataAccessOp; } else if (!isPrimaryIndex) { primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false); // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context); } else { List<Object> primaryIndexOutputTypes = new ArrayList<Object>(); AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes); List<LogicalVariable> scanVariables = dataSourceOp.getVariables(); // Checks whether the primary index search can replace the given // SELECT condition. // If so, condition will be set to null and eventually the SELECT // operator will be removed. // If not, we create a new condition based on remaining ones. if (!primaryIndexPostProccessingIsNeeded) { List<Mutable<ILogicalExpression>> remainingFuncExprs = new ArrayList<Mutable<ILogicalExpression>>(); getNewConditionExprs(conditionRef, replacedFuncExprs, remainingFuncExprs); // Generate new condition. if (!remainingFuncExprs.isEmpty()) { ILogicalExpression pulledCond = createSelectCondition(remainingFuncExprs); conditionRef.setValue(pulledCond); } else { conditionRef.setValue(null); } } // Checks whether LEFT_OUTER_UNNESTMAP operator is required. boolean leftOuterUnnestMapRequired = false; if (retainNull && retainInput) { leftOuterUnnestMapRequired = true; } else { leftOuterUnnestMapRequired = false; } if (conditionRef.getValue() != null) { // The job gen parameters are transferred to the actual job gen // via the UnnestMapOperator's function arguments. List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>(); jobGenParams.writeToFuncArgs(primaryIndexFuncArgs); // An index search is expressed as an unnest-map over an // index-search function. IFunctionInfo primaryIndexSearch = FunctionUtil .getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH); UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression( primaryIndexSearch, primaryIndexFuncArgs); primaryIndexSearchFunc.setReturnsUniqueValues(true); if (!leftOuterUnnestMapRequired) { primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput); } else { primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, true); } } else { if (!leftOuterUnnestMapRequired) { primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, ((UnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, retainInput); } else { primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, ((LeftOuterUnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, true); } } primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, scanVariables, recordType, metaRecordType, dataset, context); } return primaryIndexUnnestOp; }
From source file:org.apache.asterix.optimizer.rules.am.BTreeAccessMethod.java
private int createKeyVarsAndExprs(int numKeys, LimitType[] keyLimits, ILogicalExpression[] searchKeyExprs, ArrayList<LogicalVariable> assignKeyVarList, ArrayList<Mutable<ILogicalExpression>> assignKeyExprList, ArrayList<LogicalVariable> keyVarList, IOptimizationContext context, ILogicalExpression[] constExpressions, LogicalVariable[] constExprVars) { if (keyLimits[0] == null) { return 0; }//from w w w .ja v a 2 s. com for (int i = 0; i < numKeys; i++) { ILogicalExpression searchKeyExpr = searchKeyExprs[i]; ILogicalExpression constExpression = constExpressions[i]; LogicalVariable keyVar = null; if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) { keyVar = context.newVar(); assignKeyExprList.add(new MutableObject<ILogicalExpression>(searchKeyExpr)); assignKeyVarList.add(keyVar); } else { keyVar = ((VariableReferenceExpression) searchKeyExpr).getVariableReference(); if (constExpression != null) { assignKeyExprList.add(new MutableObject<ILogicalExpression>(constExpression)); assignKeyVarList.add(constExprVars[i]); } } keyVarList.add(keyVar); } return numKeys; }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceLSMComponentFilterRule.java
private AssignOperator createAssignOperator(List<IOptimizableFuncExpr> optFuncExprs, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOptimizationContext context) {/* ww w .j a va 2s. c om*/ List<LogicalVariable> assignKeyVarList = new ArrayList<>(); List<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>(); for (IOptimizableFuncExpr optFuncExpr : optFuncExprs) { ComparisonKind ck = AlgebricksBuiltinFunctions .getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier()); ILogicalExpression searchKeyExpr = optFuncExpr.getConstantAtRuntimeExpr(0); LogicalVariable var = context.newVar(); assignKeyExprList.add(new MutableObject<ILogicalExpression>(searchKeyExpr)); assignKeyVarList.add(var); if (ck == ComparisonKind.GE || ck == ComparisonKind.GT) { minFilterVars.add(var); } else if (ck == ComparisonKind.LE || ck == ComparisonKind.LT) { maxFilterVars.add(var); } else if (ck == ComparisonKind.EQ) { minFilterVars.add(var); maxFilterVars.add(var); } } return new AssignOperator(assignKeyVarList, assignKeyExprList); }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceLSMComponentFilterRule.java
private void changePlan(List<IOptimizableFuncExpr> optFuncExprs, AbstractLogicalOperator op, Dataset dataset, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue(); while (descendantOp != null) { if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) { DataSourceScanOperator dataSourceScanOp = (DataSourceScanOperator) descendantOp; AqlDataSource ds = (AqlDataSource) dataSourceScanOp.getDataSource(); if (dataset.getDatasetName() .compareTo(((DatasetDataSource) ds).getDataset().getDatasetName()) == 0) { List<LogicalVariable> minFilterVars = new ArrayList<>(); List<LogicalVariable> maxFilterVars = new ArrayList<>(); AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context);/*from w w w . ja v a 2s . c o m*/ dataSourceScanOp.setMinFilterVars(minFilterVars); dataSourceScanOp.setMaxFilterVars(maxFilterVars); List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<>(); for (LogicalVariable var : assignOp.getVariables()) { additionalFilteringExpressions .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var))); } dataSourceScanOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs().add( new MutableObject<ILogicalOperator>(dataSourceScanOp.getInputs().get(0).getValue())); dataSourceScanOp.getInputs().get(0).setValue(assignOp); } } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) { UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp; ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue(); if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr; FunctionIdentifier fid = f.getFunctionIdentifier(); if (!fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) { throw new IllegalStateException(); } AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams(); jobGenParams.readFromFuncArgs(f.getArguments()); if (dataset.getDatasetName().compareTo(jobGenParams.datasetName) == 0) { List<LogicalVariable> minFilterVars = new ArrayList<>(); List<LogicalVariable> maxFilterVars = new ArrayList<>(); AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context); unnestMapOp.setMinFilterVars(minFilterVars); unnestMapOp.setMaxFilterVars(maxFilterVars); List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<>(); for (LogicalVariable var : assignOp.getVariables()) { additionalFilteringExpressions.add( new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var))); } unnestMapOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs().add( new MutableObject<ILogicalOperator>(unnestMapOp.getInputs().get(0).getValue())); unnestMapOp.getInputs().get(0).setValue(assignOp); } } } if (descendantOp.getInputs().isEmpty()) { break; } descendantOp = (AbstractLogicalOperator) descendantOp.getInputs().get(0).getValue(); } }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceSelectAccessMethodRule.java
private ILogicalOperator connectAll2ndarySearchPlanWithIntersect(List<ILogicalOperator> subRoots, IOptimizationContext context) throws AlgebricksException { ILogicalOperator lop = subRoots.get(0); List<List<LogicalVariable>> inputVars = new ArrayList<>(subRoots.size()); for (int i = 0; i < subRoots.size(); i++) { if (lop.getOperatorTag() != subRoots.get(i).getOperatorTag()) { throw new AlgebricksException("The data source root should have the same operator type"); }/* w w w . ja va 2s. c o m*/ if (lop.getInputs().size() != 1) { throw new AlgebricksException("The primary search has multiple input"); } ILogicalOperator curRoot = subRoots.get(i); OrderOperator order = (OrderOperator) curRoot.getInputs().get(0).getValue(); List<LogicalVariable> orderedColumn = new ArrayList<>(order.getOrderExpressions().size()); for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> orderExpression : order .getOrderExpressions()) { if (orderExpression.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) { throw new AlgebricksException("It should not happen, the order by expression is not variables"); } VariableReferenceExpression orderedVar = (VariableReferenceExpression) orderExpression.second .getValue(); orderedColumn.add(orderedVar.getVariableReference()); } inputVars.add(orderedColumn); } List<LogicalVariable> outputVar = inputVars.get(0); IntersectOperator intersect = new IntersectOperator(outputVar, inputVars); for (ILogicalOperator secondarySearch : subRoots) { intersect.getInputs().add(secondarySearch.getInputs().get(0)); } context.computeAndSetTypeEnvironmentForOperator(intersect); lop.getInputs().set(0, new MutableObject<>(intersect)); return lop; }
From source file:org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.java
@Override public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException { IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); Dataset dataset = indexSubTree.getDataset(); ARecordType recordType = indexSubTree.getRecordType(); ARecordType metaRecordType = indexSubTree.getMetaRecordType(); // we made sure indexSubTree has datasource scan DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.getDataSourceRef().getValue(); InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams(chosenIndex.getIndexName(), chosenIndex.getIndexType(), dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);//from w ww. jav a 2 s . c om // Add function-specific args such as search modifier, and possibly a similarity threshold. addFunctionSpecificArgs(optFuncExpr, jobGenParams); // Add the type of search key from the optFuncExpr. addSearchKeyType(optFuncExpr, indexSubTree, context, jobGenParams); // Operator that feeds the secondary-index search. AbstractLogicalOperator inputOp = null; // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index search. // List of variables for the assign. ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>(); // probeSubTree is null if we are dealing with a selection query, and non-null for join queries. if (probeSubTree == null) { // List of expressions for the assign. ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>(); // Add key vars and exprs to argument list. addKeyVarsAndExprs(optFuncExpr, keyVarList, keyExprList, context); // Assign operator that sets the secondary-index search-key fields. inputOp = new AssignOperator(keyVarList, keyExprList); // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). inputOp.getInputs().add(new MutableObject<>( OperatorManipulationUtil.deepCopy(dataSourceScan.getInputs().get(0).getValue()))); inputOp.setExecutionMode(dataSourceScan.getExecutionMode()); } else { // We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs. LogicalVariable inputSearchVariable = getInputSearchVar(optFuncExpr, indexSubTree); keyVarList.add(inputSearchVariable); inputOp = (AbstractLogicalOperator) probeSubTree.getRoot(); } jobGenParams.setKeyVarList(keyVarList); ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput, retainNull); // Generate the rest of the upstream plan which feeds the search results into the primary index. AbstractUnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap( dataSourceScan, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false); return primaryIndexUnnestOp; }
From source file:org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.java
@Override public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException { // Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side). Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex); OptimizableOperatorSubTree indexSubTree; OptimizableOperatorSubTree probeSubTree; // We assume that the left subtree is the outer branch and the right subtree is the inner branch. // This assumption holds true since we only use an index from the right subtree. // The following is just a sanity check. if (rightSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) { indexSubTree = rightSubTree;/*from ww w . ja va 2 s .c o m*/ probeSubTree = leftSubTree; } else { return false; } IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); // The arguments of edit-distance-contains() function are asymmetrical, we can only use index // if the dataset of index subtree and the dataset of first argument's subtree is the same if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS && optFuncExpr.getOperatorSubTree(0).getDataset() != null && !optFuncExpr.getOperatorSubTree(0) .getDataset().getDatasetName().equals(indexSubTree.getDataset().getDatasetName())) { return false; } //if LOJ, reset null place holder variable LogicalVariable newNullPlaceHolderVar = null; if (isLeftOuterJoin && hasGroupBy) { //get a new null place holder variable that is the first field variable of the primary key //from the indexSubTree's datasourceScanOp newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); //reset the null place holder variable AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context); } AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) joinRef.getValue(); // Remember the original probe subtree, and its primary-key variables, // so we can later retrieve the missing attributes via an equi join. List<LogicalVariable> originalSubTreePKs = new ArrayList<>(); // Remember the primary-keys of the new probe subtree for the top-level equi join. List<LogicalVariable> surrogateSubTreePKs = new ArrayList<>(); // Copy probe subtree, replacing their variables with new ones. We will use the original variables // to stitch together a top-level equi join. Mutable<ILogicalOperator> originalProbeSubTreeRootRef = copyAndReinitProbeSubTree(probeSubTree, join.getCondition().getValue(), optFuncExpr, originalSubTreePKs, surrogateSubTreePKs, context); // Remember original live variables from the index sub tree. List<LogicalVariable> indexSubTreeLiveVars = new ArrayList<>(); VariableUtilities.getLiveVariables(indexSubTree.getRoot(), indexSubTreeLiveVars); // Clone the original join condition because we may have to modify it (and we also need the original). ILogicalExpression joinCond = join.getCondition().getValue().cloneExpression(); // Create "panic" (non indexed) nested-loop join path if necessary. Mutable<ILogicalOperator> panicJoinRef = null; Map<LogicalVariable, LogicalVariable> panicVarMap = null; if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK || optFuncExpr.getFuncExpr() .getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) { panicJoinRef = new MutableObject<>(joinRef.getValue()); panicVarMap = new HashMap<>(); Mutable<ILogicalOperator> newProbeRootRef = createPanicNestedLoopJoinPlan(panicJoinRef, indexSubTree, probeSubTree, optFuncExpr, chosenIndex, panicVarMap, context); probeSubTree.getRootRef().setValue(newProbeRootRef.getValue()); probeSubTree.setRoot(newProbeRootRef.getValue()); } // Create regular indexed-nested loop join path. ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context); indexSubTree.getDataSourceRef().setValue(indexPlanRootOp); // Change join into a select with the same condition. SelectOperator topSelect = new SelectOperator(new MutableObject<ILogicalExpression>(joinCond), isLeftOuterJoin, newNullPlaceHolderVar); topSelect.getInputs().add(indexSubTree.getRootRef()); topSelect.setExecutionMode(ExecutionMode.LOCAL); context.computeAndSetTypeEnvironmentForOperator(topSelect); ILogicalOperator topOp = topSelect; // Hook up the indexed-nested loop join path with the "panic" (non indexed) nested-loop join path by putting a union all on top. if (panicJoinRef != null) { LogicalVariable inputSearchVar = getInputSearchVar(optFuncExpr, indexSubTree); indexSubTreeLiveVars.addAll(originalSubTreePKs); indexSubTreeLiveVars.add(inputSearchVar); List<LogicalVariable> panicPlanLiveVars = new ArrayList<>(); VariableUtilities.getLiveVariables(panicJoinRef.getValue(), panicPlanLiveVars); // Create variable mapping for union all operator. List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<>(); for (int i = 0; i < indexSubTreeLiveVars.size(); i++) { LogicalVariable indexSubTreeVar = indexSubTreeLiveVars.get(i); LogicalVariable panicPlanVar = panicVarMap.get(indexSubTreeVar); if (panicPlanVar == null) { panicPlanVar = indexSubTreeVar; } varMap.add(new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(indexSubTreeVar, panicPlanVar, indexSubTreeVar)); } UnionAllOperator unionAllOp = new UnionAllOperator(varMap); unionAllOp.getInputs().add(new MutableObject<ILogicalOperator>(topOp)); unionAllOp.getInputs().add(panicJoinRef); unionAllOp.setExecutionMode(ExecutionMode.PARTITIONED); context.computeAndSetTypeEnvironmentForOperator(unionAllOp); topOp = unionAllOp; } // Place a top-level equi-join on top to retrieve the missing variables from the original probe subtree. // The inner (build) branch of the join is the subtree with the data scan, since the result of the similarity join could potentially be big. // This choice may not always be the most efficient, but it seems more robust than the alternative. Mutable<ILogicalExpression> eqJoinConditionRef = createPrimaryKeysEqJoinCondition(originalSubTreePKs, surrogateSubTreePKs); InnerJoinOperator topEqJoin = new InnerJoinOperator(eqJoinConditionRef, originalProbeSubTreeRootRef, new MutableObject<ILogicalOperator>(topOp)); topEqJoin.setExecutionMode(ExecutionMode.PARTITIONED); joinRef.setValue(topEqJoin); context.computeAndSetTypeEnvironmentForOperator(topEqJoin); return true; }
From source file:org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.java
/** * Copies the probeSubTree (using new variables), and reinitializes the probeSubTree to it. * Accordingly replaces the variables in the given joinCond, and the optFuncExpr. * Returns a reference to the original plan root. *//*from w w w .j av a 2 s. c om*/ private Mutable<ILogicalOperator> copyAndReinitProbeSubTree(OptimizableOperatorSubTree probeSubTree, ILogicalExpression joinCond, IOptimizableFuncExpr optFuncExpr, List<LogicalVariable> originalSubTreePKs, List<LogicalVariable> surrogateSubTreePKs, IOptimizationContext context) throws AlgebricksException { probeSubTree.getPrimaryKeyVars(originalSubTreePKs); // Create two copies of the original probe subtree. // The first copy, which becomes the new probe subtree, will retain the primary-key and secondary-search key variables, // but have all other variables replaced with new ones. // The second copy, which will become an input to the top-level equi-join to resolve the surrogates, // will have all primary-key and secondary-search keys replaced, but retains all other original variables. // Variable replacement map for the first copy. LinkedHashMap<LogicalVariable, LogicalVariable> newProbeSubTreeVarMap = new LinkedHashMap<>(); // Variable replacement map for the second copy. LinkedHashMap<LogicalVariable, LogicalVariable> joinInputSubTreeVarMap = new LinkedHashMap<>(); // Init with all live vars. List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>(); VariableUtilities.getLiveVariables(probeSubTree.getRoot(), liveVars); for (LogicalVariable var : liveVars) { joinInputSubTreeVarMap.put(var, var); } // Fill variable replacement maps. for (int i = 0; i < optFuncExpr.getNumLogicalVars(); i++) { joinInputSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), context.newVar()); newProbeSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), optFuncExpr.getLogicalVar(i)); } for (int i = 0; i < originalSubTreePKs.size(); i++) { LogicalVariable newPKVar = context.newVar(); surrogateSubTreePKs.add(newPKVar); joinInputSubTreeVarMap.put(originalSubTreePKs.get(i), newPKVar); newProbeSubTreeVarMap.put(originalSubTreePKs.get(i), originalSubTreePKs.get(i)); } // Create first copy. LogicalOperatorDeepCopyWithNewVariablesVisitor firstDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor( context, context, newProbeSubTreeVarMap); ILogicalOperator newProbeSubTree = firstDeepCopyVisitor.deepCopy(probeSubTree.getRoot()); inferTypes(newProbeSubTree, context); Mutable<ILogicalOperator> newProbeSubTreeRootRef = new MutableObject<ILogicalOperator>(newProbeSubTree); // Create second copy. LogicalOperatorDeepCopyWithNewVariablesVisitor secondDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor( context, context, joinInputSubTreeVarMap); ILogicalOperator joinInputSubTree = secondDeepCopyVisitor.deepCopy(probeSubTree.getRoot()); inferTypes(joinInputSubTree, context); probeSubTree.getRootRef().setValue(joinInputSubTree); // Remember the original probe subtree reference so we can return it. Mutable<ILogicalOperator> originalProbeSubTreeRootRef = probeSubTree.getRootRef(); // Replace the original probe subtree with its copy. Dataset origDataset = probeSubTree.getDataset(); ARecordType origRecordType = probeSubTree.getRecordType(); probeSubTree.initFromSubTree(newProbeSubTreeRootRef); probeSubTree.setDataset(origDataset); probeSubTree.setRecordType(origRecordType); // Replace the variables in the join condition based on the mapping of variables // in the new probe subtree. Map<LogicalVariable, LogicalVariable> varMapping = firstDeepCopyVisitor.getInputToOutputVariableMapping(); for (Map.Entry<LogicalVariable, LogicalVariable> varMapEntry : varMapping.entrySet()) { if (varMapEntry.getKey() != varMapEntry.getValue()) { joinCond.substituteVar(varMapEntry.getKey(), varMapEntry.getValue()); } } return originalProbeSubTreeRootRef; }
From source file:org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.java
private Mutable<ILogicalExpression> createPrimaryKeysEqJoinCondition(List<LogicalVariable> originalSubTreePKs, List<LogicalVariable> surrogateSubTreePKs) { List<Mutable<ILogicalExpression>> eqExprs = new ArrayList<Mutable<ILogicalExpression>>(); int numPKVars = originalSubTreePKs.size(); for (int i = 0; i < numPKVars; i++) { List<Mutable<ILogicalExpression>> args = new ArrayList<Mutable<ILogicalExpression>>(); args.add(new MutableObject<ILogicalExpression>( new VariableReferenceExpression(surrogateSubTreePKs.get(i)))); args.add(new MutableObject<ILogicalExpression>( new VariableReferenceExpression(originalSubTreePKs.get(i)))); ILogicalExpression eqFunc = new ScalarFunctionCallExpression( FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), args); eqExprs.add(new MutableObject<ILogicalExpression>(eqFunc)); }//from w w w .ja v a 2s .c o m if (eqExprs.size() == 1) { return eqExprs.get(0); } else { ILogicalExpression andFunc = new ScalarFunctionCallExpression( FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.AND), eqExprs); return new MutableObject<ILogicalExpression>(andFunc); } }
From source file:org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.java
private Mutable<ILogicalOperator> createPanicNestedLoopJoinPlan(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, IOptimizableFuncExpr optFuncExpr, Index chosenIndex, Map<LogicalVariable, LogicalVariable> panicVarMap, IOptimizationContext context) throws AlgebricksException { LogicalVariable inputSearchVar = getInputSearchVar(optFuncExpr, indexSubTree); // We split the plan into two "branches", and add selections on each side. AbstractLogicalOperator replicateOp = new ReplicateOperator(2); replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(probeSubTree.getRoot())); replicateOp.setExecutionMode(ExecutionMode.PARTITIONED); context.computeAndSetTypeEnvironmentForOperator(replicateOp); // Create select ops for removing tuples that are filterable and not filterable, respectively. IVariableTypeEnvironment probeTypeEnv = context.getOutputTypeEnvironment(probeSubTree.getRoot()); IAType inputSearchVarType;// w w w . j a va 2 s . com if (chosenIndex.isEnforcingKeyFileds()) { inputSearchVarType = optFuncExpr.getFieldType(optFuncExpr.findLogicalVar(inputSearchVar)); } else { inputSearchVarType = (IAType) probeTypeEnv.getVarType(inputSearchVar); } Mutable<ILogicalOperator> isFilterableSelectOpRef = new MutableObject<ILogicalOperator>(); Mutable<ILogicalOperator> isNotFilterableSelectOpRef = new MutableObject<ILogicalOperator>(); createIsFilterableSelectOps(replicateOp, inputSearchVar, inputSearchVarType, optFuncExpr, chosenIndex, context, isFilterableSelectOpRef, isNotFilterableSelectOpRef); List<LogicalVariable> originalLiveVars = new ArrayList<LogicalVariable>(); VariableUtilities.getLiveVariables(indexSubTree.getRoot(), originalLiveVars); // Copy the scan subtree in indexSubTree. LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor( context, context); ILogicalOperator scanSubTree = deepCopyVisitor.deepCopy(indexSubTree.getRoot()); Map<LogicalVariable, LogicalVariable> copyVarMap = deepCopyVisitor.getInputToOutputVariableMapping(); panicVarMap.putAll(copyVarMap); List<LogicalVariable> copyLiveVars = new ArrayList<LogicalVariable>(); VariableUtilities.getLiveVariables(scanSubTree, copyLiveVars); // Replace the inputs of the given join op, and replace variables in its // condition since we deep-copied one of the scanner subtrees which // changed variables. AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue(); for (Map.Entry<LogicalVariable, LogicalVariable> entry : copyVarMap.entrySet()) { joinOp.getCondition().getValue().substituteVar(entry.getKey(), entry.getValue()); } joinOp.getInputs().clear(); joinOp.getInputs().add(new MutableObject<ILogicalOperator>(scanSubTree)); // Make sure that the build input (which may be materialized causing blocking) comes from // the split+select, otherwise the plan will have a deadlock. joinOp.getInputs().add(isNotFilterableSelectOpRef); context.computeAndSetTypeEnvironmentForOperator(joinOp); // Return the new root of the probeSubTree. return isFilterableSelectOpRef; }