src/org/apache/pig/builtin/TOBAG.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.pig.builtin;


import java.io.IOException;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.NonSpillableDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;

/**
 * This class takes a list of items and puts them into a bag
 * T = foreach U generate TOBAG($0, $1, $2);
 * It's like saying this:
 * T = foreach U generate {($0), ($1), ($2)}
 * 
 * All arguments that are not of tuple type are inserted into a tuple before
 * being added to the bag. This is because bag is always a bag of tuples.
 * 
 * Output schema:
 * The output schema for this udf depends on the schema of its arguments.
 * If all the arguments have same type and same inner 
 * schema (for bags/tuple columns), then the udf output schema would be a bag 
 * of tuples having a column of the type and inner-schema (if any) of the 
 * arguments. 
 * If the arguments are of type tuple/bag, then their inner schemas should match,
 * though schema field aliases may differ.
 * If these conditions are not met the output schema will be a bag with null 
 * inner schema.
 *  
 *  example 1 
 *  grunt> describe a;
 *  a: {a0: int,a1: int}
 *  grunt> b = foreach a generate TOBAG(a0,a1);
 *  grunt> describe b;
 *  b: {{int}}
 *  
 *  example 2
 *  grunt> describe a;
 *  a: {a0: (x: int),a1: (x: int)}
 *  grunt> b = foreach a generate TOBAG(a0,a1);                                    
 *  grunt> describe b;                                                             
 *  b: {{(x: int)}}
 *  
 *  example 3
 *  grunt> describe a;                                                             
 *  a: {a0: (x: int),a1: (y: int)}
 *  -- note that the inner schemas have matching types but different field aliases.
 *  -- the aliases of the first argument (a0) will be used in output schema:
 *  grunt> b = foreach a generate TOBAG(a0,a1);
 *  grunt> describe b;
 *  b: {{(x: int)}}
 *
 *  example 4
 *  grunt> describe a;
 *  a: {a0: (x: int),a1: (x: chararray)}
 *  -- here the inner schemas do not match, so output schema is not well defined:
 *  grunt> b = foreach a generate TOBAG(a0,a1);                                    
 *  grunt> describe b;                                                             
 *  b: {{NULL}}
 *  
 *  
 * 
 */
public class TOBAG extends EvalFunc<DataBag> {

    @Override
    public DataBag exec(Tuple input) throws IOException {
        try {
            // The assumption is that if the bag contents fits into
            // an input tuple, it will not need to be spilled.
            DataBag bag = new NonSpillableDataBag(input.size());

            for (int i = 0; i < input.size(); ++i) {
                final Object object = input.get(i);
                if (object instanceof Tuple) {
                    bag.add( (Tuple) object);
                } else {
                    Tuple tp2 = TupleFactory.getInstance().newTuple(1);
                    tp2.set(0, object);
                    bag.add(tp2);
                }
            }

            return bag;
        } catch (Exception ee) {
            throw new RuntimeException("Error while creating a bag", ee);
        }
    }

    /* (non-Javadoc)
     * @see org.apache.pig.EvalFunc#outputSchema(org.apache.pig.impl.logicalLayer.schema.Schema)
     * If all the columns in the tuple are of same type, then set the bag schema 
     * to bag of tuple with column of this type
     * 
     */
    @Override
    public Schema outputSchema(Schema inputSch) {
        byte type = DataType.ERROR;
        Schema innerSchema = null;
        if(inputSch != null){
            for(FieldSchema fs : inputSch.getFields()){
                if(type == DataType.ERROR){
                    type = fs.type;
                    innerSchema = fs.schema;
                }else{
                    if( type != fs.type || !nullEquals(innerSchema, fs.schema)){
                        // invalidate the type
                        type = DataType.ERROR;
                        break;
                    }
                }
            }
        }
        try {
            if(type == DataType.ERROR){
                return Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
            }
            FieldSchema innerFs = new Schema.FieldSchema(null, innerSchema, type);
            Schema innerSch = new Schema(innerFs);
            Schema bagSchema = new Schema(new FieldSchema(null, innerSch, DataType.BAG));
            return bagSchema;
        } catch (FrontendException e) {
            //This should not happen
            throw new RuntimeException("Bug : exception thrown while " +
                    "creating output schema for TOBAG udf", e);
        }

    }

    private boolean nullEquals(Schema currentSchema, Schema newSchema) {
        if(currentSchema == null){
            if(newSchema != null){
                return false;
            }
            return true;
        }
        return Schema.equals(currentSchema, newSchema, false, true);
    }

    @Override
    public boolean allowCompileTimeCalculation() {
        return true;
    }
}