| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.pig.newplan.logical; |
| |
| import java.util.ArrayList; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| |
| import org.apache.pig.impl.logicalLayer.ExpressionOperator; |
| import org.apache.pig.impl.logicalLayer.FrontendException; |
| import org.apache.pig.impl.logicalLayer.LOAdd; |
| import org.apache.pig.impl.logicalLayer.LOAnd; |
| import org.apache.pig.impl.logicalLayer.LOBinCond; |
| import org.apache.pig.impl.logicalLayer.LOCast; |
| import org.apache.pig.impl.logicalLayer.LOConst; |
| import org.apache.pig.impl.logicalLayer.LODivide; |
| import org.apache.pig.impl.logicalLayer.LOEqual; |
| import org.apache.pig.impl.logicalLayer.LOGreaterThan; |
| import org.apache.pig.impl.logicalLayer.LOGreaterThanEqual; |
| import org.apache.pig.impl.logicalLayer.LOIsNull; |
| import org.apache.pig.impl.logicalLayer.LOLesserThan; |
| import org.apache.pig.impl.logicalLayer.LOLesserThanEqual; |
| import org.apache.pig.impl.logicalLayer.LOMapLookup; |
| import org.apache.pig.impl.logicalLayer.LOMod; |
| import org.apache.pig.impl.logicalLayer.LOMultiply; |
| import org.apache.pig.impl.logicalLayer.LONegative; |
| import org.apache.pig.impl.logicalLayer.LONot; |
| import org.apache.pig.impl.logicalLayer.LONotEqual; |
| import org.apache.pig.impl.logicalLayer.LOOr; |
| import org.apache.pig.impl.logicalLayer.LOProject; |
| import org.apache.pig.impl.logicalLayer.LORegexp; |
| import org.apache.pig.impl.logicalLayer.LOSplit; |
| import org.apache.pig.impl.logicalLayer.LOSplitOutput; |
| import org.apache.pig.impl.logicalLayer.LOSubtract; |
| import org.apache.pig.impl.logicalLayer.LOUserFunc; |
| import org.apache.pig.impl.logicalLayer.LOVisitor; |
| import org.apache.pig.impl.logicalLayer.LogicalOperator; |
| import org.apache.pig.impl.logicalLayer.LogicalPlan; |
| import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; |
| import org.apache.pig.impl.plan.DependencyOrderWalkerWOSeenChk; |
| import org.apache.pig.impl.plan.VisitorException; |
| import org.apache.pig.newplan.logical.expression.AddExpression; |
| import org.apache.pig.newplan.logical.expression.AndExpression; |
| import org.apache.pig.newplan.logical.expression.BinCondExpression; |
| import org.apache.pig.newplan.logical.expression.CastExpression; |
| import org.apache.pig.newplan.logical.expression.ConstantExpression; |
| import org.apache.pig.newplan.logical.expression.DereferenceExpression; |
| import org.apache.pig.newplan.logical.expression.DivideExpression; |
| import org.apache.pig.newplan.logical.expression.EqualExpression; |
| import org.apache.pig.newplan.logical.expression.GreaterThanEqualExpression; |
| import org.apache.pig.newplan.logical.expression.GreaterThanExpression; |
| import org.apache.pig.newplan.logical.expression.IsNullExpression; |
| import org.apache.pig.newplan.logical.expression.LessThanEqualExpression; |
| import org.apache.pig.newplan.logical.expression.LessThanExpression; |
| import org.apache.pig.newplan.logical.expression.LogicalExpression; |
| import org.apache.pig.newplan.logical.expression.MapLookupExpression; |
| import org.apache.pig.newplan.logical.expression.ModExpression; |
| import org.apache.pig.newplan.logical.expression.MultiplyExpression; |
| import org.apache.pig.newplan.logical.expression.NegativeExpression; |
| import org.apache.pig.newplan.logical.expression.NotEqualExpression; |
| import org.apache.pig.newplan.logical.expression.NotExpression; |
| import org.apache.pig.newplan.logical.expression.OrExpression; |
| import org.apache.pig.newplan.logical.expression.ProjectExpression; |
| import org.apache.pig.newplan.logical.expression.RegexExpression; |
| import org.apache.pig.newplan.logical.expression.SubtractExpression; |
| import org.apache.pig.newplan.logical.expression.UserFuncExpression; |
| import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator; |
| import org.apache.pig.newplan.logical.relational.LogicalSchema; |
| |
| //visitor to translate expressions |
| public class LogicalExpPlanMigrationVistor extends LOVisitor { |
| |
| protected org.apache.pig.newplan.logical.expression.LogicalExpressionPlan exprPlan; |
| protected HashMap<LogicalOperator, LogicalExpression> exprOpsMap; |
| protected LogicalRelationalOperator attachedRelationalOp; |
| protected LogicalOperator oldAttachedRelationalOp; |
| protected LogicalPlan outerPlan; |
| private Map<LogicalOperator, LogicalRelationalOperator> outerOpsMap; |
| |
| public LogicalExpPlanMigrationVistor(LogicalPlan expressionPlan, LogicalOperator oldAttachedOperator, |
| LogicalRelationalOperator attachedOperator, LogicalPlan outerPlan, |
| Map<LogicalOperator, LogicalRelationalOperator> outerOpsMap) { |
| super(expressionPlan, new DependencyOrderWalkerWOSeenChk<LogicalOperator, LogicalPlan>(expressionPlan)); |
| exprPlan = new org.apache.pig.newplan.logical.expression.LogicalExpressionPlan(); |
| exprOpsMap = new HashMap<LogicalOperator, LogicalExpression>(); |
| attachedRelationalOp = attachedOperator; |
| oldAttachedRelationalOp = oldAttachedOperator; |
| this.outerPlan = outerPlan; |
| this.outerOpsMap = outerOpsMap; |
| } |
| |
| private void translateConnection(LogicalOperator oldOp, org.apache.pig.newplan.Operator newOp) { |
| List<LogicalOperator> preds = mPlan.getPredecessors(oldOp); |
| |
| // the dependency relationship of new expression plan is opposite to the old logical plan |
| // for example, a+b, in old plan, "add" is a leave, and "a" and "b" are roots |
| // in new plan, "add" is root, and "a" and "b" are leaves. |
| if(preds != null) { |
| for(LogicalOperator pred: preds) { |
| org.apache.pig.newplan.Operator newPred = exprOpsMap.get(pred); |
| newOp.getPlan().connect(newOp, newPred); |
| } |
| } |
| |
| List<LogicalOperator> softPreds = mPlan.getSoftLinkPredecessors(oldOp); |
| |
| if(softPreds != null) { |
| for(LogicalOperator softPred: softPreds) { |
| org.apache.pig.newplan.Operator newSoftPred = exprOpsMap.get(softPred); |
| newOp.getPlan().createSoftLink(newOp, newSoftPred); |
| } |
| } |
| } |
| |
| public void visit(LOProject project) throws VisitorException { |
| int col = project.getCol(); |
| |
| LogicalExpression pe; |
| if (project.getPlan().getPredecessors(project)!=null && project.getPlan().getPredecessors(project).get(0) |
| instanceof ExpressionOperator) { |
| List<Integer> columnNums = new ArrayList<Integer>(); |
| columnNums.add(col); |
| pe = new DereferenceExpression(exprPlan, columnNums); |
| } |
| else { |
| LogicalOperator lg = project.getExpression(); |
| int input; |
| if (oldAttachedRelationalOp instanceof LOSplitOutput) { |
| LOSplit split = (LOSplit)outerPlan.getPredecessors(oldAttachedRelationalOp).get(0); |
| input = outerPlan.getPredecessors(split).indexOf(lg); |
| } |
| else { |
| input = outerPlan.getPredecessors(oldAttachedRelationalOp).indexOf(lg); |
| } |
| pe = new ProjectExpression(exprPlan, input, project.isStar()?-1:col, attachedRelationalOp); |
| } |
| |
| exprPlan.add(pe); |
| exprOpsMap.put(project, pe); |
| translateConnection(project, pe); |
| } |
| |
| public void visit(LOConst con) throws VisitorException{ |
| ConstantExpression ce = null; |
| try { |
| ce = new ConstantExpression(exprPlan, con.getValue(), Util.translateFieldSchema(con.getFieldSchema())); |
| } catch (FrontendException e) { |
| throw new VisitorException(e); |
| } |
| |
| exprPlan.add(ce); |
| exprOpsMap.put(con, ce); |
| translateConnection(con, ce); |
| } |
| |
| public void visit(LOGreaterThan op) throws VisitorException { |
| ExpressionOperator left = op.getLhsOperand(); |
| ExpressionOperator right = op.getRhsOperand(); |
| |
| GreaterThanExpression eq = new GreaterThanExpression |
| (exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, eq); |
| } |
| |
| public void visit(LOLesserThan op) throws VisitorException { |
| ExpressionOperator left = op.getLhsOperand(); |
| ExpressionOperator right = op.getRhsOperand(); |
| |
| LessThanExpression eq = new LessThanExpression |
| (exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, eq); |
| } |
| |
| public void visit(LOGreaterThanEqual op) throws VisitorException { |
| ExpressionOperator left = op.getLhsOperand(); |
| ExpressionOperator right = op.getRhsOperand(); |
| |
| GreaterThanEqualExpression eq = new GreaterThanEqualExpression |
| (exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, eq); |
| } |
| |
| public void visit(LOLesserThanEqual op) throws VisitorException { |
| ExpressionOperator left = op.getLhsOperand(); |
| ExpressionOperator right = op.getRhsOperand(); |
| |
| LessThanEqualExpression eq = new LessThanEqualExpression |
| (exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, eq); |
| } |
| |
| public void visit(LOEqual op) throws VisitorException { |
| ExpressionOperator left = op.getLhsOperand(); |
| ExpressionOperator right = op.getRhsOperand(); |
| |
| EqualExpression eq = new EqualExpression(exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, eq); |
| } |
| |
| public void visit(LOUserFunc op) throws VisitorException { |
| UserFuncExpression exp = new UserFuncExpression(exprPlan, op.getFuncSpec()); |
| |
| List<ExpressionOperator> args = op.getArguments(); |
| |
| for( ExpressionOperator arg : args ) { |
| LogicalExpression expArg = exprOpsMap.get(arg); |
| exprPlan.connect(exp, expArg); |
| } |
| |
| exprOpsMap.put(op, exp); |
| // We need to track all the scalars |
| if(op.getImplicitReferencedOperator() != null) { |
| exp.setImplicitReferencedOperator(outerOpsMap.get(op.getImplicitReferencedOperator())); |
| } |
| |
| } |
| |
| public void visit(LOBinCond op) throws VisitorException { |
| ExpressionOperator condition = op.getCond(); |
| ExpressionOperator left = op.getLhsOp(); |
| ExpressionOperator right = op.getRhsOp(); |
| |
| BinCondExpression exp = new BinCondExpression(exprPlan, |
| exprOpsMap.get(condition), exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, exp); |
| } |
| |
| public void visit(LOCast cast) throws VisitorException { |
| ExpressionOperator exp = cast.getExpression(); |
| |
| CastExpression c = null; |
| try { |
| c = new CastExpression(exprPlan, exprOpsMap.get(exp), Util.translateFieldSchema(cast.getFieldSchema())); |
| } catch (FrontendException e) { |
| throw new VisitorException(e); |
| } |
| c.setFuncSpec(cast.getLoadFuncSpec()); |
| exprOpsMap.put(cast, c); |
| } |
| |
| public void visit(LORegexp binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| RegexExpression ae = new RegexExpression(exprPlan |
| , exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LONotEqual op) throws VisitorException { |
| ExpressionOperator left = op.getLhsOperand(); |
| ExpressionOperator right = op.getRhsOperand(); |
| |
| NotEqualExpression eq = new NotEqualExpression(exprPlan, |
| exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(op, eq); |
| } |
| |
| public void visit(LOAdd binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| AddExpression ae = new AddExpression(exprPlan, |
| exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LOSubtract binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| SubtractExpression ae = new SubtractExpression(exprPlan, |
| exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LOMultiply binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| MultiplyExpression ae = new MultiplyExpression(exprPlan, |
| exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LODivide binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| DivideExpression ae = new DivideExpression(exprPlan, |
| exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LOMod binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| ModExpression ae = new ModExpression(exprPlan, |
| exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| |
| public void visit(LONegative uniOp) throws VisitorException { |
| ExpressionOperator exp = uniOp.getOperand(); |
| NegativeExpression op = new NegativeExpression(exprPlan, exprOpsMap.get(exp)); |
| exprOpsMap.put(uniOp, op); |
| } |
| |
| public void visit(LOMapLookup colOp) throws VisitorException { |
| FieldSchema fieldSchema; |
| try { |
| fieldSchema = colOp.getFieldSchema(); |
| } catch (FrontendException e) { |
| throw new VisitorException( e.getMessage() ); |
| } |
| |
| LogicalSchema.LogicalFieldSchema logfieldSchema = |
| new LogicalSchema.LogicalFieldSchema( fieldSchema.alias, |
| Util.translateSchema(fieldSchema.schema), fieldSchema.type); |
| |
| LogicalExpression map = exprOpsMap.get( colOp.getMap() ); |
| |
| MapLookupExpression op = new MapLookupExpression(exprPlan, |
| colOp.getLookUpKey(), logfieldSchema); |
| |
| exprPlan.connect(op, map); |
| |
| exprOpsMap.put(colOp, op); |
| } |
| |
| public void visit(LOAnd binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| AndExpression ae = new AndExpression(exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LOOr binOp) throws VisitorException { |
| ExpressionOperator left = binOp.getLhsOperand(); |
| ExpressionOperator right = binOp.getRhsOperand(); |
| |
| OrExpression ae = new OrExpression(exprPlan, exprOpsMap.get(left), exprOpsMap.get(right)); |
| exprOpsMap.put(binOp, ae); |
| } |
| |
| public void visit(LONot uniOp) throws VisitorException { |
| ExpressionOperator exp = uniOp.getOperand(); |
| NotExpression not = new NotExpression(exprPlan, exprOpsMap.get(exp)); |
| exprOpsMap.put(uniOp, not); |
| } |
| |
| public void visit(LOIsNull uniOp) throws VisitorException { |
| ExpressionOperator exp = uniOp.getOperand(); |
| IsNullExpression isNull = new IsNullExpression(exprPlan, exprOpsMap.get(exp)); |
| exprOpsMap.put(uniOp, isNull); |
| } |
| } |
