| package edu.uci.ics.asterix.optimizer.rules.am; |
| |
| import java.util.ArrayList; |
| import java.util.List; |
| |
| import org.apache.commons.lang3.mutable.Mutable; |
| import org.apache.commons.lang3.mutable.MutableObject; |
| |
| import edu.uci.ics.asterix.aql.util.FunctionUtils; |
| import edu.uci.ics.asterix.metadata.declared.AqlCompiledDatasetDecl; |
| import edu.uci.ics.asterix.metadata.declared.AqlCompiledIndexDecl; |
| import edu.uci.ics.asterix.metadata.declared.AqlCompiledIndexDecl.IndexKind; |
| import edu.uci.ics.asterix.metadata.utils.DatasetUtils; |
| import edu.uci.ics.asterix.om.base.ABoolean; |
| import edu.uci.ics.asterix.om.base.AInt32; |
| import edu.uci.ics.asterix.om.base.AString; |
| import edu.uci.ics.asterix.om.base.IAObject; |
| import edu.uci.ics.asterix.om.constants.AsterixConstantValue; |
| import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions; |
| import edu.uci.ics.asterix.om.types.ARecordType; |
| import edu.uci.ics.asterix.om.types.IAType; |
| import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil; |
| import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; |
| import edu.uci.ics.hyracks.algebricks.common.utils.Pair; |
| import edu.uci.ics.hyracks.algebricks.common.utils.Triple; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder; |
| import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; |
| import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory; |
| |
| /** |
| * Static helper functions for rewriting plans using indexes. |
| */ |
| public class AccessMethodUtils { |
| public static void appendPrimaryIndexTypes(AqlCompiledDatasetDecl datasetDecl, IAType itemType, List<Object> target) { |
| List<Triple<ICopyEvaluatorFactory, ScalarFunctionCallExpression, IAType>> partitioningFunctions = DatasetUtils |
| .getPartitioningFunctions(datasetDecl); |
| for (Triple<ICopyEvaluatorFactory, ScalarFunctionCallExpression, IAType> t : partitioningFunctions) { |
| target.add(t.third); |
| } |
| target.add(itemType); |
| } |
| |
| public static ConstantExpression createStringConstant(String str) { |
| return new ConstantExpression(new AsterixConstantValue(new AString(str))); |
| } |
| |
| public static ConstantExpression createInt32Constant(int i) { |
| return new ConstantExpression(new AsterixConstantValue(new AInt32(i))); |
| } |
| |
| public static ConstantExpression createBooleanConstant(boolean b) { |
| if (b) { |
| return new ConstantExpression(new AsterixConstantValue(ABoolean.TRUE)); |
| } else { |
| return new ConstantExpression(new AsterixConstantValue(ABoolean.FALSE)); |
| } |
| } |
| |
| public static String getStringConstant(Mutable<ILogicalExpression> expr) { |
| IAObject obj = ((AsterixConstantValue)((ConstantExpression) expr.getValue()) |
| .getValue()).getObject(); |
| return ((AString)obj).getStringValue(); |
| } |
| |
| public static int getInt32Constant(Mutable<ILogicalExpression> expr) { |
| IAObject obj = ((AsterixConstantValue)((ConstantExpression) expr.getValue()) |
| .getValue()).getObject(); |
| return ((AInt32)obj).getIntegerValue(); |
| } |
| |
| public static boolean getBooleanConstant(Mutable<ILogicalExpression> expr) { |
| IAObject obj = ((AsterixConstantValue)((ConstantExpression) expr.getValue()) |
| .getValue()).getObject(); |
| return ((ABoolean)obj).getBoolean(); |
| } |
| |
| public static boolean analyzeFuncExprArgsForOneConstAndVar(AbstractFunctionCallExpression funcExpr, |
| AccessMethodAnalysisContext analysisCtx) { |
| IAlgebricksConstantValue constFilterVal = null; |
| LogicalVariable fieldVar = null; |
| ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue(); |
| ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue(); |
| // One of the args must be a constant, and the other arg must be a variable. |
| if (arg1.getExpressionTag() == LogicalExpressionTag.CONSTANT |
| && arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) { |
| ConstantExpression constExpr = (ConstantExpression) arg1; |
| constFilterVal = constExpr.getValue(); |
| VariableReferenceExpression varExpr = (VariableReferenceExpression) arg2; |
| fieldVar = varExpr.getVariableReference(); |
| } else if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE |
| && arg2.getExpressionTag() == LogicalExpressionTag.CONSTANT) { |
| ConstantExpression constExpr = (ConstantExpression) arg2; |
| constFilterVal = constExpr.getValue(); |
| VariableReferenceExpression varExpr = (VariableReferenceExpression) arg1; |
| fieldVar = varExpr.getVariableReference(); |
| } else { |
| return false; |
| } |
| analysisCtx.matchedFuncExprs.add(new OptimizableFuncExpr(funcExpr, fieldVar, constFilterVal)); |
| return true; |
| } |
| |
| public static int getNumSecondaryKeys(AqlCompiledDatasetDecl datasetDecl, AqlCompiledIndexDecl indexDecl, |
| ARecordType recordType) throws AlgebricksException { |
| switch (indexDecl.getKind()) { |
| case BTREE: |
| case WORD_INVIX: |
| case NGRAM_INVIX: { |
| return indexDecl.getFieldExprs().size(); |
| } |
| case RTREE: { |
| Pair<IAType, Boolean> keyPairType = AqlCompiledIndexDecl.getNonNullableKeyFieldType(indexDecl.getFieldExprs().get(0), recordType); |
| IAType keyType = keyPairType.first; |
| int numDimensions = NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag()); |
| return numDimensions * 2; |
| } |
| default: { |
| throw new AlgebricksException("Unknown index kind: " + indexDecl.getKind()); |
| } |
| } |
| } |
| |
| /** |
| * Appends the types of the fields produced by the given secondary index to dest. |
| */ |
| public static void appendSecondaryIndexTypes(AqlCompiledDatasetDecl datasetDecl, ARecordType recordType, |
| AqlCompiledIndexDecl indexDecl, boolean primaryKeysOnly, List<Object> dest) throws AlgebricksException { |
| if (!primaryKeysOnly) { |
| switch (indexDecl.getKind()) { |
| case BTREE: |
| case WORD_INVIX: |
| case NGRAM_INVIX: { |
| for (String sk : indexDecl.getFieldExprs()) { |
| Pair<IAType, Boolean> keyPairType = AqlCompiledIndexDecl.getNonNullableKeyFieldType(sk, recordType); |
| dest.add(keyPairType.first); |
| } |
| break; |
| } |
| case RTREE: { |
| Pair<IAType, Boolean> keyPairType = AqlCompiledIndexDecl.getNonNullableKeyFieldType(indexDecl.getFieldExprs().get(0), recordType); |
| IAType keyType = keyPairType.first; |
| IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag()); |
| int numKeys = getNumSecondaryKeys(datasetDecl, indexDecl, recordType); |
| for (int i = 0; i < numKeys; i++) { |
| dest.add(nestedKeyType); |
| } |
| break; |
| } |
| } |
| } |
| // Primary keys. |
| for (Triple<ICopyEvaluatorFactory, ScalarFunctionCallExpression, IAType> t : DatasetUtils |
| .getPartitioningFunctions(datasetDecl)) { |
| dest.add(t.third); |
| } |
| } |
| |
| public static void appendSecondaryIndexOutputVars(AqlCompiledDatasetDecl datasetDecl, ARecordType recordType, |
| AqlCompiledIndexDecl indexDecl, boolean primaryKeysOnly, IOptimizationContext context, |
| List<LogicalVariable> dest) throws AlgebricksException { |
| int numPrimaryKeys = DatasetUtils.getPartitioningFunctions(datasetDecl).size(); |
| int numSecondaryKeys = getNumSecondaryKeys(datasetDecl, indexDecl, recordType); |
| int numVars = (primaryKeysOnly) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys; |
| for (int i = 0; i < numVars; i++) { |
| dest.add(context.newVar()); |
| } |
| } |
| |
| public static List<LogicalVariable> getPrimaryKeyVarsFromUnnestMap(AqlCompiledDatasetDecl datasetDecl, |
| ILogicalOperator unnestMapOp) { |
| int numPrimaryKeys = DatasetUtils.getPartitioningFunctions(datasetDecl).size(); |
| List<LogicalVariable> primaryKeyVars = new ArrayList<LogicalVariable>(); |
| List<LogicalVariable> sourceVars = ((UnnestMapOperator) unnestMapOp).getVariables(); |
| // Assumes the primary keys are located at the end. |
| int start = sourceVars.size() - numPrimaryKeys; |
| int stop = sourceVars.size(); |
| for (int i = start; i < stop; i++) { |
| primaryKeyVars.add(sourceVars.get(i)); |
| } |
| return primaryKeyVars; |
| } |
| |
| public static UnnestMapOperator createSecondaryIndexUnnestMap(AqlCompiledDatasetDecl datasetDecl, |
| ARecordType recordType, AqlCompiledIndexDecl indexDecl, ILogicalOperator inputOp, |
| AccessMethodJobGenParams jobGenParams, IOptimizationContext context, boolean outputPrimaryKeysOnly, |
| boolean retainInput) throws AlgebricksException { |
| // The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments. |
| ArrayList<Mutable<ILogicalExpression>> secondaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>(); |
| jobGenParams.writeToFuncArgs(secondaryIndexFuncArgs); |
| // Variables and types coming out of the secondary-index search. |
| List<LogicalVariable> secondaryIndexUnnestVars = new ArrayList<LogicalVariable>(); |
| List<Object> secondaryIndexOutputTypes = new ArrayList<Object>(); |
| // Append output variables/types generated by the secondary-index search (not forwarded from input). |
| appendSecondaryIndexOutputVars(datasetDecl, recordType, indexDecl, outputPrimaryKeysOnly, context, secondaryIndexUnnestVars); |
| appendSecondaryIndexTypes(datasetDecl, recordType, indexDecl, outputPrimaryKeysOnly, secondaryIndexOutputTypes); |
| // An index search is expressed as an unnest over an index-search function. |
| IFunctionInfo secondaryIndexSearch = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH); |
| UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression(secondaryIndexSearch, secondaryIndexFuncArgs); |
| secondaryIndexSearchFunc.setReturnsUniqueValues(true); |
| // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine |
| // which index to use, which variables contain the index-search keys, what is the original dataset, etc. |
| UnnestMapOperator secondaryIndexUnnestOp = new UnnestMapOperator(secondaryIndexUnnestVars, new MutableObject<ILogicalExpression>( |
| secondaryIndexSearchFunc), secondaryIndexOutputTypes, retainInput); |
| secondaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); |
| context.computeAndSetTypeEnvironmentForOperator(secondaryIndexUnnestOp); |
| secondaryIndexUnnestOp.setExecutionMode(ExecutionMode.PARTITIONED); |
| return secondaryIndexUnnestOp; |
| } |
| |
| public static UnnestMapOperator createPrimaryIndexUnnestMap(DataSourceScanOperator dataSourceScan, AqlCompiledDatasetDecl datasetDecl, |
| ARecordType recordType, ILogicalOperator inputOp, |
| IOptimizationContext context, boolean sortPrimaryKeys, boolean retainInput, boolean requiresBroadcast) throws AlgebricksException { |
| List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromUnnestMap(datasetDecl, inputOp); |
| // Optionally add a sort on the primary-index keys before searching the primary index. |
| OrderOperator order = null; |
| if (sortPrimaryKeys) { |
| order = new OrderOperator(); |
| for (LogicalVariable pkVar : primaryKeyVars) { |
| Mutable<ILogicalExpression> vRef = new MutableObject<ILogicalExpression>(new VariableReferenceExpression(pkVar)); |
| order.getOrderExpressions().add( |
| new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, vRef)); |
| } |
| // The secondary-index search feeds into the sort. |
| order.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); |
| order.setExecutionMode(ExecutionMode.LOCAL); |
| context.computeAndSetTypeEnvironmentForOperator(order); |
| } |
| // The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments. |
| List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>(); |
| BTreeJobGenParams jobGenParams = new BTreeJobGenParams(datasetDecl.getName(), IndexKind.BTREE, datasetDecl.getName(), retainInput, requiresBroadcast); |
| // Set low/high inclusive to true for a point lookup. |
| jobGenParams.setLowKeyInclusive(true); |
| jobGenParams.setHighKeyInclusive(true); |
| jobGenParams.setLowKeyVarList(primaryKeyVars, 0, primaryKeyVars.size()); |
| jobGenParams.setHighKeyVarList(primaryKeyVars, 0, primaryKeyVars.size()); |
| jobGenParams.writeToFuncArgs(primaryIndexFuncArgs); |
| // Variables and types coming out of the primary-index search. |
| List<LogicalVariable> primaryIndexUnnestVars = new ArrayList<LogicalVariable>(); |
| List<Object> primaryIndexOutputTypes = new ArrayList<Object>(); |
| // Append output variables/types generated by the primary-index search (not forwarded from input). |
| primaryIndexUnnestVars.addAll(dataSourceScan.getVariables()); |
| appendPrimaryIndexTypes(datasetDecl, recordType, primaryIndexOutputTypes); |
| // An index search is expressed as an unnest over an index-search function. |
| IFunctionInfo primaryIndexSearch = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH); |
| AbstractFunctionCallExpression primaryIndexSearchFunc = new ScalarFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs); |
| // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine |
| // which index to use, which variables contain the index-search keys, what is the original dataset, etc. |
| UnnestMapOperator primaryIndexUnnestOp = new UnnestMapOperator(primaryIndexUnnestVars, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), |
| primaryIndexOutputTypes, retainInput); |
| // Fed by the order operator or the secondaryIndexUnnestOp. |
| if (sortPrimaryKeys) { |
| primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(order)); |
| } else { |
| primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); |
| } |
| context.computeAndSetTypeEnvironmentForOperator(primaryIndexUnnestOp); |
| primaryIndexUnnestOp.setExecutionMode(ExecutionMode.PARTITIONED); |
| return primaryIndexUnnestOp; |
| } |
| |
| } |