List of usage examples for org.apache.commons.lang3.mutable Mutable getValue
T getValue();
From source file:org.apache.asterix.optimizer.rules.am.AccessMethodUtils.java
public static ScalarFunctionCallExpression findLOJIsMissingFuncInGroupBy(GroupByOperator lojGroupbyOp) throws AlgebricksException { //find IS_NULL function of which argument has the nullPlaceholder variable in the nested plan of groupby. ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0); Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0); AbstractLogicalOperator subPlanRootOp = (AbstractLogicalOperator) subPlanRootOpRef.getValue(); boolean foundSelectNonNull = false; ScalarFunctionCallExpression isNullFuncExpr = null; AbstractLogicalOperator inputOp = subPlanRootOp; while (inputOp != null) { if (inputOp.getOperatorTag() == LogicalOperatorTag.SELECT) { SelectOperator selectOp = (SelectOperator) inputOp; if (selectOp.getCondition().getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) selectOp.getCondition().getValue()) .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) { ScalarFunctionCallExpression notFuncExpr = (ScalarFunctionCallExpression) selectOp .getCondition().getValue(); if (notFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue()) .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_MISSING)) { isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0) .getValue(); if (isNullFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.VARIABLE) { foundSelectNonNull = true; break; }//w w w.j av a2 s . c o m } } } } } inputOp = inputOp.getInputs().size() > 0 ? (AbstractLogicalOperator) inputOp.getInputs().get(0).getValue() : null; } if (!foundSelectNonNull) { throw new AlgebricksException( "Could not find the non-null select operator in GroupByOperator for LEFTOUTERJOIN plan optimization."); } return isNullFuncExpr; }
From source file:org.apache.asterix.optimizer.rules.am.BTreeAccessMethod.java
@Override public boolean applySelectPlanTransformation(Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException { SelectOperator select = (SelectOperator) selectRef.getValue(); Mutable<ILogicalExpression> conditionRef = select.getCondition(); ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, subTree, null, chosenIndex, analysisCtx, false, false, false, context); if (primaryIndexUnnestOp == null) { return false; }//ww w.j a v a 2 s. c om Mutable<ILogicalOperator> opRef = subTree.getAssignsAndUnnestsRefs().isEmpty() ? null : subTree.getAssignsAndUnnestsRefs().get(0); ILogicalOperator op = null; if (opRef != null) { op = opRef.getValue(); } // Generate new select using the new condition. if (conditionRef.getValue() != null) { select.getInputs().clear(); if (op != null) { subTree.getDataSourceRef().setValue(primaryIndexUnnestOp); select.getInputs().add(new MutableObject<ILogicalOperator>(op)); } else { select.getInputs().add(new MutableObject<ILogicalOperator>(primaryIndexUnnestOp)); } } else { ((AbstractLogicalOperator) primaryIndexUnnestOp).setExecutionMode(ExecutionMode.PARTITIONED); if (op != null) { subTree.getDataSourceRef().setValue(primaryIndexUnnestOp); selectRef.setValue(op); } else { selectRef.setValue(primaryIndexUnnestOp); } } return true; }
From source file:org.apache.asterix.optimizer.rules.am.BTreeAccessMethod.java
@Override public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException { AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue(); Mutable<ILogicalExpression> conditionRef = joinOp.getCondition(); // Determine if the index is applicable on the left or right side // (if both, we arbitrarily prefer the left side). Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex); OptimizableOperatorSubTree indexSubTree; OptimizableOperatorSubTree probeSubTree; // We assume that the left subtree is the outer branch and the right subtree is the inner branch. // This assumption holds true since we only use an index from the right subtree. // The following is just a sanity check. if (rightSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) { indexSubTree = rightSubTree;//from ww w.j a v a2s .co m probeSubTree = leftSubTree; } else { return false; } LogicalVariable newNullPlaceHolderVar = null; if (isLeftOuterJoin) { //get a new null place holder variable that is the first field variable of the primary key //from the indexSubTree's datasourceScanOp newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); } ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context); if (primaryIndexUnnestOp == null) { return false; } if (isLeftOuterJoin && hasGroupBy) { //reset the null place holder variable AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context); } // If there are conditions left, add a new select operator on top. indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp); if (conditionRef.getValue() != null) { SelectOperator topSelect = new SelectOperator(conditionRef, isLeftOuterJoin, newNullPlaceHolderVar); topSelect.getInputs().add(indexSubTree.getRootRef()); topSelect.setExecutionMode(ExecutionMode.LOCAL); context.computeAndSetTypeEnvironmentForOperator(topSelect); // Replace the original join with the new subtree rooted at the select op. joinRef.setValue(topSelect); } else { joinRef.setValue(indexSubTree.getRootRef().getValue()); } return true; }
From source file:org.apache.asterix.optimizer.rules.am.BTreeAccessMethod.java
@Override public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException { Dataset dataset = indexSubTree.getDataset(); ARecordType recordType = indexSubTree.getRecordType(); ARecordType metaRecordType = indexSubTree.getMetaRecordType(); // we made sure indexSubTree has datasource scan AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef() .getValue();// w w w .j a v a2 s. c om List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.indexExprsAndVars.get(chosenIndex); List<IOptimizableFuncExpr> matchedFuncExprs = analysisCtx.matchedFuncExprs; int numSecondaryKeys = analysisCtx.indexNumMatchedKeys.get(chosenIndex); // List of function expressions that will be replaced by the secondary-index search. // These func exprs will be removed from the select condition at the very end of this method. Set<ILogicalExpression> replacedFuncExprs = new HashSet<>(); // Info on high and low keys for the BTree search predicate. ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys]; ILogicalExpression[] highKeyExprs = new ILogicalExpression[numSecondaryKeys]; LimitType[] lowKeyLimits = new LimitType[numSecondaryKeys]; LimitType[] highKeyLimits = new LimitType[numSecondaryKeys]; boolean[] lowKeyInclusive = new boolean[numSecondaryKeys]; boolean[] highKeyInclusive = new boolean[numSecondaryKeys]; ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys]; LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys]; /* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range * predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate, * we just bail by setting this flag.*/ boolean couldntFigureOut = false; boolean doneWithExprs = false; boolean isEqCondition = false; BitSet setLowKeys = new BitSet(numSecondaryKeys); BitSet setHighKeys = new BitSet(numSecondaryKeys); // Go through the func exprs listed as optimizable by the chosen index, // and formulate a range predicate on the secondary-index keys. // checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value // since we have a round issues when dealing with LT(<) OR GT(>) operator. boolean realTypeConvertedToIntegerType; for (Pair<Integer, Integer> exprIndex : exprAndVarList) { // Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs. IOptimizableFuncExpr optFuncExpr = matchedFuncExprs.get(exprIndex.first); int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames()); if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) { // If we are optimizing a join, the matching field may be the second field name. keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames()); } if (keyPos < 0) { throw new AlgebricksException( "Could not match optimizable function expression to any index field name."); } Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils .createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree); ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first; if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { constantAtRuntimeExpressions[keyPos] = searchKeyExpr; constAtRuntimeExprVars[keyPos] = context.newVar(); searchKeyExpr = new VariableReferenceExpression(constAtRuntimeExprVars[keyPos]); } realTypeConvertedToIntegerType = returnedSearchKeyExpr.second; LimitType limit = getLimitType(optFuncExpr, probeSubTree); // If a DOUBLE or FLOAT constant is converted to an INT type value, // we need to check a corner case where two real values are located between an INT value. // For example, for the following query, // // for $emp in dataset empDataset // where $emp.age > double("2.3") and $emp.age < double("3.3") // return $emp.id // // It should generate a result if there is a tuple that satisfies the condition, which is 3, // however, it does not generate the desired result since finding candidates // fail after truncating the fraction part (there is no INT whose value is greater than 2 and less than 3.) // // Therefore, we convert LT(<) to LE(<=) and GT(>) to GE(>=) to find candidates. // This does not change the result of an actual comparison since this conversion is only applied // for finding candidates from an index. // if (realTypeConvertedToIntegerType) { if (limit == LimitType.HIGH_EXCLUSIVE) { limit = LimitType.HIGH_INCLUSIVE; } else if (limit == LimitType.LOW_EXCLUSIVE) { limit = LimitType.LOW_INCLUSIVE; } } switch (limit) { case EQUAL: { if (lowKeyLimits[keyPos] == null && highKeyLimits[keyPos] == null) { lowKeyLimits[keyPos] = highKeyLimits[keyPos] = limit; lowKeyInclusive[keyPos] = highKeyInclusive[keyPos] = true; lowKeyExprs[keyPos] = highKeyExprs[keyPos] = searchKeyExpr; setLowKeys.set(keyPos); setHighKeys.set(keyPos); isEqCondition = true; } else { // Has already been set to the identical values. // When optimizing join we may encounter the same optimizable expression twice // (once from analyzing each side of the join) if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) { isEqCondition = true; break; } couldntFigureOut = true; } // TODO: For now don't consider prefix searches. // If high and low keys are set, we exit for now. if (setLowKeys.cardinality() == numSecondaryKeys && setHighKeys.cardinality() == numSecondaryKeys) { doneWithExprs = true; } break; } case HIGH_EXCLUSIVE: { if (highKeyLimits[keyPos] == null || (highKeyLimits[keyPos] != null && highKeyInclusive[keyPos])) { highKeyLimits[keyPos] = limit; highKeyExprs[keyPos] = searchKeyExpr; highKeyInclusive[keyPos] = false; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false && highKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } case HIGH_INCLUSIVE: { if (highKeyLimits[keyPos] == null) { highKeyLimits[keyPos] = limit; highKeyExprs[keyPos] = searchKeyExpr; highKeyInclusive[keyPos] = true; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } case LOW_EXCLUSIVE: { if (lowKeyLimits[keyPos] == null || (lowKeyLimits[keyPos] != null && lowKeyInclusive[keyPos])) { lowKeyLimits[keyPos] = limit; lowKeyExprs[keyPos] = searchKeyExpr; lowKeyInclusive[keyPos] = false; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false && lowKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } case LOW_INCLUSIVE: { if (lowKeyLimits[keyPos] == null) { lowKeyLimits[keyPos] = limit; lowKeyExprs[keyPos] = searchKeyExpr; lowKeyInclusive[keyPos] = true; } else { // Has already been set to the identical values. When optimizing join we may encounter the // same optimizable expression twice // (once from analyzing each side of the join) if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr)) { break; } couldntFigureOut = true; doneWithExprs = true; } break; } default: { throw new IllegalStateException(); } } if (!couldntFigureOut) { // Remember to remove this funcExpr later. replacedFuncExprs.add(matchedFuncExprs.get(exprIndex.first).getFuncExpr()); } if (doneWithExprs) { break; } } if (couldntFigureOut) { return null; } // If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals // closed to obtain a superset of answers and leave the original selection in place. boolean primaryIndexPostProccessingIsNeeded = false; for (int i = 1; i < numSecondaryKeys; ++i) { if (lowKeyInclusive[i] != lowKeyInclusive[0]) { Arrays.fill(lowKeyInclusive, true); primaryIndexPostProccessingIsNeeded = true; break; } } for (int i = 1; i < numSecondaryKeys; ++i) { if (highKeyInclusive[i] != highKeyInclusive[0]) { Arrays.fill(highKeyInclusive, true); primaryIndexPostProccessingIsNeeded = true; break; } } // determine cases when prefix search could be applied for (int i = 1; i < lowKeyExprs.length; i++) { if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null || highKeyLimits[0] == null && highKeyLimits[i] != null || highKeyLimits[0] != null && highKeyLimits[i] == null) { numSecondaryKeys--; primaryIndexPostProccessingIsNeeded = true; } } if (lowKeyLimits[0] == null) { lowKeyInclusive[0] = true; } if (highKeyLimits[0] == null) { highKeyInclusive[0] = true; } // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index // search. // List of variables for the assign. ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>(); // List of variables and expressions for the assign. ArrayList<LogicalVariable> assignKeyVarList = new ArrayList<LogicalVariable>(); ArrayList<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<Mutable<ILogicalExpression>>(); int numLowKeys = createKeyVarsAndExprs(numSecondaryKeys, lowKeyLimits, lowKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars); int numHighKeys = createKeyVarsAndExprs(numSecondaryKeys, highKeyLimits, highKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars); BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast); jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]); jobGenParams.setHighKeyInclusive(highKeyInclusive[0]); jobGenParams.setIsEqCondition(isEqCondition); jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys); jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys); ILogicalOperator inputOp = null; if (!assignKeyVarList.isEmpty()) { // Assign operator that sets the constant secondary-index search-key fields if necessary. AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList); // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). assignConstantSearchKeys.getInputs().add(new MutableObject<ILogicalOperator>( OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue()))); assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode()); inputOp = assignConstantSearchKeys; } else { // All index search keys are variables. inputOp = probeSubTree.getRoot(); } ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput, retainNull); // Generate the rest of the upstream plan which feeds the search results into the primary index. AbstractUnnestMapOperator primaryIndexUnnestOp = null; boolean isPrimaryIndex = chosenIndex.isPrimaryIndex(); if (dataset.getDatasetType() == DatasetType.EXTERNAL) { // External dataset UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap( dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull); indexSubTree.getDataSourceRef().setValue(externalDataAccessOp); return externalDataAccessOp; } else if (!isPrimaryIndex) { primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false); // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context); } else { List<Object> primaryIndexOutputTypes = new ArrayList<Object>(); AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes); List<LogicalVariable> scanVariables = dataSourceOp.getVariables(); // Checks whether the primary index search can replace the given // SELECT condition. // If so, condition will be set to null and eventually the SELECT // operator will be removed. // If not, we create a new condition based on remaining ones. if (!primaryIndexPostProccessingIsNeeded) { List<Mutable<ILogicalExpression>> remainingFuncExprs = new ArrayList<Mutable<ILogicalExpression>>(); getNewConditionExprs(conditionRef, replacedFuncExprs, remainingFuncExprs); // Generate new condition. if (!remainingFuncExprs.isEmpty()) { ILogicalExpression pulledCond = createSelectCondition(remainingFuncExprs); conditionRef.setValue(pulledCond); } else { conditionRef.setValue(null); } } // Checks whether LEFT_OUTER_UNNESTMAP operator is required. boolean leftOuterUnnestMapRequired = false; if (retainNull && retainInput) { leftOuterUnnestMapRequired = true; } else { leftOuterUnnestMapRequired = false; } if (conditionRef.getValue() != null) { // The job gen parameters are transferred to the actual job gen // via the UnnestMapOperator's function arguments. List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>(); jobGenParams.writeToFuncArgs(primaryIndexFuncArgs); // An index search is expressed as an unnest-map over an // index-search function. IFunctionInfo primaryIndexSearch = FunctionUtil .getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH); UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression( primaryIndexSearch, primaryIndexFuncArgs); primaryIndexSearchFunc.setReturnsUniqueValues(true); if (!leftOuterUnnestMapRequired) { primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput); } else { primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, true); } } else { if (!leftOuterUnnestMapRequired) { primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, ((UnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, retainInput); } else { primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, ((LeftOuterUnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, true); } } primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, scanVariables, recordType, metaRecordType, dataset, context); } return primaryIndexUnnestOp; }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceJoinAccessMethodRule.java
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { clear();//from w ww . ja va 2 s . com setMetadataDeclarations(context); // Match operator pattern and initialize optimizable sub trees. if (!matchesOperatorPattern(opRef, context)) { return false; } // Analyze condition on those optimizable subtrees that have a datasource scan. Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new HashMap<IAccessMethod, AccessMethodAnalysisContext>(); boolean matchInLeftSubTree = false; boolean matchInRightSubTree = false; if (leftSubTree.hasDataSource()) { matchInLeftSubTree = analyzeCondition(joinCond, leftSubTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment); } if (rightSubTree.hasDataSource()) { matchInRightSubTree = analyzeCondition(joinCond, rightSubTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment); } if (!matchInLeftSubTree && !matchInRightSubTree) { return false; } // Set dataset and type metadata. AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider(); boolean checkLeftSubTreeMetadata = false; boolean checkRightSubTreeMetadata = false; if (matchInLeftSubTree) { checkLeftSubTreeMetadata = leftSubTree.setDatasetAndTypeMetadata(metadataProvider); } if (matchInRightSubTree) { checkRightSubTreeMetadata = rightSubTree.setDatasetAndTypeMetadata(metadataProvider); } if (!checkLeftSubTreeMetadata && !checkRightSubTreeMetadata) { return false; } if (checkLeftSubTreeMetadata) { fillSubTreeIndexExprs(leftSubTree, analyzedAMs, context); } if (checkRightSubTreeMetadata) { fillSubTreeIndexExprs(rightSubTree, analyzedAMs, context); } pruneIndexCandidates(analyzedAMs, context, typeEnvironment); // We only consider indexes from the inner branch (right subTree). // If no index is available, then we stop this optimization. removeIndexCandidatesFromOuterBranch(analyzedAMs); // Choose an index from the inner branch that will be used. Pair<IAccessMethod, Index> chosenIndex = chooseBestIndex(analyzedAMs); if (chosenIndex == null) { context.addToDontApplySet(this, join); return false; } // Apply plan transformation using chosen index. AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first); //For LOJ with GroupBy, prepare objects to reset LOJ nullPlaceHolderVariable in GroupByOp if (isLeftOuterJoin && hasGroupBy) { analysisCtx.setLOJGroupbyOpRef(opRef); ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils .findLOJIsMissingFuncInGroupBy((GroupByOperator) opRef.getValue()); analysisCtx.setLOJIsNullFuncInGroupBy(isNullFuncExpr); } // At this point, we are sure that only an index from the inner branch is going to be used. // So, the left subtree is the outer branch and the right subtree is the inner branch. boolean res = chosenIndex.first.applyJoinPlanTransformation(joinRef, leftSubTree, rightSubTree, chosenIndex.second, analysisCtx, context, isLeftOuterJoin, hasGroupBy); if (res) { OperatorPropertiesUtil.typeOpRec(opRef, context); } context.addToDontApplySet(this, join); return res; }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceJoinAccessMethodRule.java
protected boolean matchesOperatorPattern(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { // First check that the operator is a join and its condition is a function call. AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); if (context.checkIfInDontApplySet(this, op1)) { return false; }//from w ww . j a va 2 s . com boolean isInnerJoin = isInnerJoin(op1); isLeftOuterJoin = isLeftOuterJoin(op1); if (!isInnerJoin && !isLeftOuterJoin) { return false; } // Set and analyze select. if (isInnerJoin) { joinRef = opRef; join = (InnerJoinOperator) op1; } else { joinRef = op1.getInputs().get(0); join = (LeftOuterJoinOperator) joinRef.getValue(); } typeEnvironment = context.getOutputTypeEnvironment(join); // Check that the select's condition is a function call. ILogicalExpression condExpr = join.getCondition().getValue(); if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { return false; } joinCond = (AbstractFunctionCallExpression) condExpr; boolean leftSubTreeInitialized = leftSubTree.initFromSubTree(join.getInputs().get(0)); boolean rightSubTreeInitialized = rightSubTree.initFromSubTree(join.getInputs().get(1)); if (!leftSubTreeInitialized || !rightSubTreeInitialized) { return false; } // One of the subtrees must have a datasource scan. if (leftSubTree.hasDataSourceScan() || rightSubTree.hasDataSourceScan()) { return true; } return false; }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceLSMComponentFilterRule.java
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { if (!checkIfRuleIsApplicable(opRef, context)) { return false; }//w ww . j av a2s . com AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); typeEnvironment = context.getOutputTypeEnvironment(op); ILogicalExpression condExpr = ((SelectOperator) op).getCondition().getValue(); AccessMethodAnalysisContext analysisCtx = analyzeCondition(condExpr, context, typeEnvironment); if (analysisCtx.matchedFuncExprs.isEmpty()) { return false; } Dataset dataset = getDataset(op, context); List<String> filterFieldName = null; ARecordType recType = null; if (dataset != null && dataset.getDatasetType() == DatasetType.INTERNAL) { filterFieldName = DatasetUtils.getFilterField(dataset); IAType itemType = ((AqlMetadataProvider) context.getMetadataProvider()) .findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); if (itemType.getTypeTag() == ATypeTag.RECORD) { recType = (ARecordType) itemType; } } if (filterFieldName == null || recType == null) { return false; } List<Index> datasetIndexes = ((AqlMetadataProvider) context.getMetadataProvider()) .getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName()); List<IOptimizableFuncExpr> optFuncExprs = new ArrayList<>(); for (int i = 0; i < analysisCtx.matchedFuncExprs.size(); i++) { IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(i); boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes, context); if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) { optFuncExprs.add(optFuncExpr); } } if (optFuncExprs.isEmpty()) { return false; } changePlan(optFuncExprs, op, dataset, context); OperatorPropertiesUtil.typeOpRec(opRef, context); context.addToDontApplySet(this, op); return true; }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceLSMComponentFilterRule.java
private AccessMethodAnalysisContext analyzeCondition(ILogicalExpression cond, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { AccessMethodAnalysisContext analysisCtx = new AccessMethodAnalysisContext(); AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) cond; FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier(); if (funcIdent != AlgebricksBuiltinFunctions.OR) { analyzeFunctionExpr(funcExpr, analysisCtx, context, typeEnvironment); for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) { ILogicalExpression argExpr = arg.getValue(); if (argExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { continue; }// w w w .java 2 s. com analyzeFunctionExpr((AbstractFunctionCallExpression) argExpr, analysisCtx, context, typeEnvironment); } } return analysisCtx; }
From source file:org.apache.asterix.optimizer.rules.am.IntroduceSelectAccessMethodRule.java
protected boolean matchesOperatorPattern(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { // First check that the operator is a select and its condition is a function call. AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); if (context.checkIfInDontApplySet(this, op1)) { return false; }// ww w . j a v a2s . c o m if (op1.getOperatorTag() != LogicalOperatorTag.SELECT) { return false; } // Set and analyze select. selectRef = opRef; select = (SelectOperator) op1; typeEnvironment = context.getOutputTypeEnvironment(op1); // Check that the select's condition is a function call. ILogicalExpression condExpr = select.getCondition().getValue(); if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { return false; } selectCond = (AbstractFunctionCallExpression) condExpr; boolean res = subTree.initFromSubTree(op1.getInputs().get(0)); return res && subTree.hasDataSourceScan(); }
From source file:org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.java
@Override public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException { // Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side). Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex); OptimizableOperatorSubTree indexSubTree; OptimizableOperatorSubTree probeSubTree; // We assume that the left subtree is the outer branch and the right subtree is the inner branch. // This assumption holds true since we only use an index from the right subtree. // The following is just a sanity check. if (rightSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) { indexSubTree = rightSubTree;//from ww w . j av a 2 s .com probeSubTree = leftSubTree; } else { return false; } IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); // The arguments of edit-distance-contains() function are asymmetrical, we can only use index // if the dataset of index subtree and the dataset of first argument's subtree is the same if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS && optFuncExpr.getOperatorSubTree(0).getDataset() != null && !optFuncExpr.getOperatorSubTree(0) .getDataset().getDatasetName().equals(indexSubTree.getDataset().getDatasetName())) { return false; } //if LOJ, reset null place holder variable LogicalVariable newNullPlaceHolderVar = null; if (isLeftOuterJoin && hasGroupBy) { //get a new null place holder variable that is the first field variable of the primary key //from the indexSubTree's datasourceScanOp newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); //reset the null place holder variable AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context); } AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) joinRef.getValue(); // Remember the original probe subtree, and its primary-key variables, // so we can later retrieve the missing attributes via an equi join. List<LogicalVariable> originalSubTreePKs = new ArrayList<>(); // Remember the primary-keys of the new probe subtree for the top-level equi join. List<LogicalVariable> surrogateSubTreePKs = new ArrayList<>(); // Copy probe subtree, replacing their variables with new ones. We will use the original variables // to stitch together a top-level equi join. Mutable<ILogicalOperator> originalProbeSubTreeRootRef = copyAndReinitProbeSubTree(probeSubTree, join.getCondition().getValue(), optFuncExpr, originalSubTreePKs, surrogateSubTreePKs, context); // Remember original live variables from the index sub tree. List<LogicalVariable> indexSubTreeLiveVars = new ArrayList<>(); VariableUtilities.getLiveVariables(indexSubTree.getRoot(), indexSubTreeLiveVars); // Clone the original join condition because we may have to modify it (and we also need the original). ILogicalExpression joinCond = join.getCondition().getValue().cloneExpression(); // Create "panic" (non indexed) nested-loop join path if necessary. Mutable<ILogicalOperator> panicJoinRef = null; Map<LogicalVariable, LogicalVariable> panicVarMap = null; if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK || optFuncExpr.getFuncExpr() .getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) { panicJoinRef = new MutableObject<>(joinRef.getValue()); panicVarMap = new HashMap<>(); Mutable<ILogicalOperator> newProbeRootRef = createPanicNestedLoopJoinPlan(panicJoinRef, indexSubTree, probeSubTree, optFuncExpr, chosenIndex, panicVarMap, context); probeSubTree.getRootRef().setValue(newProbeRootRef.getValue()); probeSubTree.setRoot(newProbeRootRef.getValue()); } // Create regular indexed-nested loop join path. ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context); indexSubTree.getDataSourceRef().setValue(indexPlanRootOp); // Change join into a select with the same condition. SelectOperator topSelect = new SelectOperator(new MutableObject<ILogicalExpression>(joinCond), isLeftOuterJoin, newNullPlaceHolderVar); topSelect.getInputs().add(indexSubTree.getRootRef()); topSelect.setExecutionMode(ExecutionMode.LOCAL); context.computeAndSetTypeEnvironmentForOperator(topSelect); ILogicalOperator topOp = topSelect; // Hook up the indexed-nested loop join path with the "panic" (non indexed) nested-loop join path by putting a union all on top. if (panicJoinRef != null) { LogicalVariable inputSearchVar = getInputSearchVar(optFuncExpr, indexSubTree); indexSubTreeLiveVars.addAll(originalSubTreePKs); indexSubTreeLiveVars.add(inputSearchVar); List<LogicalVariable> panicPlanLiveVars = new ArrayList<>(); VariableUtilities.getLiveVariables(panicJoinRef.getValue(), panicPlanLiveVars); // Create variable mapping for union all operator. List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<>(); for (int i = 0; i < indexSubTreeLiveVars.size(); i++) { LogicalVariable indexSubTreeVar = indexSubTreeLiveVars.get(i); LogicalVariable panicPlanVar = panicVarMap.get(indexSubTreeVar); if (panicPlanVar == null) { panicPlanVar = indexSubTreeVar; } varMap.add(new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(indexSubTreeVar, panicPlanVar, indexSubTreeVar)); } UnionAllOperator unionAllOp = new UnionAllOperator(varMap); unionAllOp.getInputs().add(new MutableObject<ILogicalOperator>(topOp)); unionAllOp.getInputs().add(panicJoinRef); unionAllOp.setExecutionMode(ExecutionMode.PARTITIONED); context.computeAndSetTypeEnvironmentForOperator(unionAllOp); topOp = unionAllOp; } // Place a top-level equi-join on top to retrieve the missing variables from the original probe subtree. // The inner (build) branch of the join is the subtree with the data scan, since the result of the similarity join could potentially be big. // This choice may not always be the most efficient, but it seems more robust than the alternative. Mutable<ILogicalExpression> eqJoinConditionRef = createPrimaryKeysEqJoinCondition(originalSubTreePKs, surrogateSubTreePKs); InnerJoinOperator topEqJoin = new InnerJoinOperator(eqJoinConditionRef, originalProbeSubTreeRootRef, new MutableObject<ILogicalOperator>(topOp)); topEqJoin.setExecutionMode(ExecutionMode.PARTITIONED); joinRef.setValue(topEqJoin); context.computeAndSetTypeEnvironmentForOperator(topEqJoin); return true; }