src/org/apache/pig/newplan/logical/visitor/UnionOnSchemaSetter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.pig.newplan.logical.visitor;

import java.util.ArrayList;
import java.util.List;

import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.Pair;
import org.apache.pig.newplan.DependencyOrderWalker;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.logical.expression.CastExpression;
import org.apache.pig.newplan.logical.expression.ConstantExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOGenerate;
import org.apache.pig.newplan.logical.relational.LOInnerLoad;
import org.apache.pig.newplan.logical.relational.LOUnion;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;

/**
 * A visitor that modifies the logical plan (if necessary) for union-onschema
 * functionality. It runs logical plan validator so that the correct schema
 * of its inputs is available. It inserts foreach statements in its input
 * if the input operator schema does not match the schema created by 
 * merging all input schemas.
 * 
 * Migrated from the old UnionOnSchemaSetter class.
 * 
 */
public class UnionOnSchemaSetter extends LogicalRelationalNodesVisitor{

    public UnionOnSchemaSetter(OperatorPlan plan)
            throws FrontendException {
        super(plan, new DependencyOrderWalker(plan));
    }

    @Override
    public void visit(LOUnion union) throws FrontendException {
        if( !union.isOnSchema() )
            return;
        
        LogicalSchema outputSchema = union.getSchema();
        int fieldCount = outputSchema.size();
        OperatorPlan plan = union.getPlan();
        List<Operator> preds = new ArrayList<Operator>();
        preds.addAll( plan.getPredecessors( union ) );

        List<LogicalSchema> fieldSchemas = new ArrayList<LogicalSchema>( fieldCount );
        for( LogicalFieldSchema fs : outputSchema.getFields() ) {
            LogicalSchema ls = new LogicalSchema();
            ls.addField( new LogicalFieldSchema( fs.alias, null, DataType.NULL ) );
            fieldSchemas.add( ls );
        }
        
        for( Operator pred : preds ) {
            LogicalRelationalOperator op = (LogicalRelationalOperator)pred;
            LogicalSchema opSchema = op.getSchema();
            if( opSchema.isEqual( outputSchema , true) )
                continue;
            
            LOForEach foreach = new LOForEach( plan );
            LogicalPlan innerPlan = new LogicalPlan();

            LOGenerate gen = new LOGenerate( innerPlan );
            boolean[] flattenFlags = new boolean[fieldCount];
            List<LogicalExpressionPlan> exprPlans = new ArrayList<LogicalExpressionPlan>( fieldCount );
            List<Operator> genInputs = new ArrayList<Operator>();
            
            // Get exprPlans, and genInputs
            for( LogicalFieldSchema fs : outputSchema.getFields() ) {
                LogicalExpressionPlan exprPlan = new LogicalExpressionPlan();
                exprPlans.add( exprPlan );
                int pos = -1;
                //do a match with subname also
                LogicalFieldSchema matchFS = opSchema.getFieldSubNameMatch(fs.alias);
                if(matchFS != null){
                    pos = opSchema.getFieldPosition(matchFS.alias);
                }
                if( pos == -1 ) {
                    ConstantExpression constExp = new ConstantExpression( exprPlan, null);
                    if(fs.type != DataType.BYTEARRAY){
                        LogicalSchema.LogicalFieldSchema constFs = fs.deepCopy();
                        constFs.resetUid();
                        new CastExpression(exprPlan, constExp, constFs);
                    }
                } else {
                    ProjectExpression projExpr = 
                        new ProjectExpression( exprPlan, genInputs.size(), 0, gen );
                    if( fs.type != DataType.BYTEARRAY
                        && opSchema.getField( pos ).type != fs.type ) {
                        new CastExpression( exprPlan, projExpr, fs );
                    }
                    genInputs.add( new LOInnerLoad( innerPlan, foreach, pos ) );
                }
            }
            
            gen.setFlattenFlags( flattenFlags );
            gen.setOutputPlans( exprPlans );
            gen.setUserDefinedSchema( fieldSchemas );
            innerPlan.add( gen );
            for( Operator input : genInputs ) {
                innerPlan.add(input);
                innerPlan.connect( input, gen );
            }
            
            foreach.setInnerPlan( innerPlan );
            foreach.setAlias(union.getAlias());
            Pair<Integer, Integer> pair = plan.disconnect( pred, union );
            plan.add( foreach );
            plan.connect( pred, pair.first, foreach, 0 );
            plan.connect( foreach, 0, union, pair.second );
        }
        
        union.setUnionOnSchema(false);
    }

}