hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hcatalog.pig;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;

import junit.framework.Assert;

import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestHCatLoaderComplexSchema {

    //private static MiniCluster cluster = MiniCluster.buildCluster();
    private static Driver driver;
    //private static Properties props;
    private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoaderComplexSchema.class);

    private void dropTable(String tablename) throws IOException, CommandNeedRetryException {
        driver.run("drop table " + tablename);
    }

    private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
        String createTable;
        createTable = "create table " + tablename + "(" + schema + ") ";
        if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) {
            createTable = createTable + "partitioned by (" + partitionedBy + ") ";
        }
        createTable = createTable + "stored as RCFILE tblproperties('hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver'," +
            "'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver') ";
        LOG.info("Creating table:\n {}", createTable);
        CommandProcessorResponse result = driver.run(createTable);
        int retCode = result.getResponseCode();
        if (retCode != 0) {
            throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + " " + result.getErrorMessage() + "]");
        }
    }

    private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException {
        createTable(tablename, schema, null);
    }

    @BeforeClass
    public static void setUpBeforeClass() throws Exception {

        HiveConf hiveConf = new HiveConf(TestHCatLoaderComplexSchema.class);
        hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
        hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
        hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
        driver = new Driver(hiveConf);
        SessionState.start(new CliSessionState(hiveConf));
        //props = new Properties();
        //props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));

    }

    private static final TupleFactory tf = TupleFactory.getInstance();
    private static final BagFactory bf = BagFactory.getInstance();

    private Tuple t(Object... objects) {
        return tf.newTuple(Arrays.asList(objects));
    }

    private DataBag b(Tuple... objects) {
        return bf.newDefaultBag(Arrays.asList(objects));
    }

    /**
     * artificially complex nested schema to test nested schema conversion
     * @throws Exception
     */
    @Test
    public void testSyntheticComplexSchema() throws Exception {
        String pigSchema =
            "(" +
                "a: " +
                "(" +
                "aa: chararray, " +
                "ab: long, " +
                "ac: map[], " +
                "ad: { t: (ada: long) }, " +
                "ae: { t: (aea:long, aeb: ( aeba: chararray, aebb: long)) }," +
                "af: (afa: chararray, afb: long) " +
                ")," +
                "b: chararray, " +
                "c: long, " +
                "d:  { t: (da:long, db: ( dba: chararray, dbb: long), dc: { t: (dca: long) } ) } " +
                ")";

        // with extra structs
        String tableSchema =
            "a struct<" +
                "aa: string, " +
                "ab: bigint, " +
                "ac: map<string, string>, " +
                "ad: array<struct<ada:bigint>>, " +
                "ae: array<struct<aea:bigint, aeb: struct<aeba: string, aebb: bigint>>>," +
                "af: struct<afa: string, afb: bigint> " +
                ">, " +
                "b string, " +
                "c bigint, " +
                "d array<struct<da: bigint, db: struct<dba:string, dbb:bigint>, dc: array<struct<dca: bigint>>>>";

        // without extra structs
        String tableSchema2 =
            "a struct<" +
                "aa: string, " +
                "ab: bigint, " +
                "ac: map<string, string>, " +
                "ad: array<bigint>, " +
                "ae: array<struct<aea:bigint, aeb: struct<aeba: string, aebb: bigint>>>," +
                "af: struct<afa: string, afb: bigint> " +
                ">, " +
                "b string, " +
                "c bigint, " +
                "d array<struct<da: bigint, db: struct<dba:string, dbb:bigint>, dc: array<bigint>>>";

        List<Tuple> data = new ArrayList<Tuple>();
        for (int i = 0; i < 10; i++) {
            Tuple t = t(
                t(
                    "aa test",
                    2l,
                    new HashMap<String, String>() {
                        {
                            put("ac test1", "test 1");
                            put("ac test2", "test 2");
                        }
                    },
                    b(t(3l), t(4l)),
                    b(t(5l, t("aeba test", 6l))),
                    t("afa test", 7l)
                ),
                "b test",
                (long) i,
                b(t(8l, t("dba test", 9l), b(t(10l)))));

            data.add(t);
        }
        verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, true);
        verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, false);
        verifyWriteRead("testSyntheticComplexSchema2", pigSchema, tableSchema2, data, true);
        verifyWriteRead("testSyntheticComplexSchema2", pigSchema, tableSchema2, data, false);

    }

    private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, boolean provideSchemaToStorer)
        throws IOException, CommandNeedRetryException, ExecException, FrontendException {
        MockLoader.setData(tablename + "Input", data);
        try {
            createTable(tablename, tableSchema);
            PigServer server = new PigServer(ExecType.LOCAL);
            server.setBatchOn();
            server.registerQuery("A = load '" + tablename + "Input' using org.apache.hcatalog.pig.MockLoader() AS " + pigSchema + ";");
            Schema dumpedASchema = server.dumpSchema("A");
            server.registerQuery("STORE A into '" + tablename + "' using org.apache.hcatalog.pig.HCatStorer("
                + (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
                + ");");

            ExecJob execJob = server.executeBatch().get(0);
            if (!execJob.getStatistics().isSuccessful()) {
                throw new RuntimeException("Import failed", execJob.getException());
            }
            // test that schema was loaded correctly
            server.registerQuery("X = load '" + tablename + "' using org.apache.hcatalog.pig.HCatLoader();");
            server.dumpSchema("X");
            Iterator<Tuple> it = server.openIterator("X");
            int i = 0;
            while (it.hasNext()) {
                Tuple input = data.get(i++);
                Tuple output = it.next();
                Assert.assertEquals(input.toString(), output.toString());
                LOG.info("tuple : {} ", output);
            }
            Schema dumpedXSchema = server.dumpSchema("X");

            Assert.assertEquals(
                "expected " + dumpedASchema + " but was " + dumpedXSchema + " (ignoring field names)",
                "",
                compareIgnoreFiledNames(dumpedASchema, dumpedXSchema));

        } finally {
            dropTable(tablename);
        }
    }

    private String compareIgnoreFiledNames(Schema expected, Schema got) throws FrontendException {
        if (expected == null || got == null) {
            if (expected == got) {
                return "";
            } else {
                return "\nexpected " + expected + " got " + got;
            }
        }
        if (expected.size() != got.size()) {
            return "\nsize expected " + expected.size() + " (" + expected + ") got " + got.size() + " (" + got + ")";
        }
        String message = "";
        for (int i = 0; i < expected.size(); i++) {
            FieldSchema expectedField = expected.getField(i);
            FieldSchema gotField = got.getField(i);
            if (expectedField.type != gotField.type) {
                message += "\ntype expected " + expectedField.type + " (" + expectedField + ") got " + gotField.type + " (" + gotField + ")";
            } else {
                message += compareIgnoreFiledNames(expectedField.schema, gotField.schema);
            }
        }
        return message;
    }

    /**
     * tests that unnecessary tuples are drop while converting schema
     * (Pig requires Tuples in Bags)
     * @throws Exception
     */
    @Test
    public void testTupleInBagInTupleInBag() throws Exception {
        String pigSchema = "(a: { b : ( c: { d: (i : long) } ) })";

        String tableSchema = "a array< array< bigint > >";

        List<Tuple> data = new ArrayList<Tuple>();
        data.add(t(b(t(b(t(100l), t(101l))), t(b(t(110l))))));
        data.add(t(b(t(b(t(200l))), t(b(t(210l))), t(b(t(220l))))));
        data.add(t(b(t(b(t(300l), t(301l))))));
        data.add(t(b(t(b(t(400l))), t(b(t(410l), t(411l), t(412l))))));


        verifyWriteRead("TupleInBagInTupleInBag1", pigSchema, tableSchema, data, true);
        verifyWriteRead("TupleInBagInTupleInBag2", pigSchema, tableSchema, data, false);

        // test that we don't drop the unnecessary tuple if the table has the corresponding Struct
        String tableSchema2 = "a array< struct< c: array< struct< i: bigint > > > >";

        verifyWriteRead("TupleInBagInTupleInBag3", pigSchema, tableSchema2, data, true);
        verifyWriteRead("TupleInBagInTupleInBag4", pigSchema, tableSchema2, data, false);

    }

    @Test
    public void testMapWithComplexData() throws Exception {
        String pigSchema = "(a: long, b: map[])";
        String tableSchema = "a bigint, b map<string, struct<aa:bigint, ab:string>>";

        List<Tuple> data = new ArrayList<Tuple>();
        for (int i = 0; i < 10; i++) {
            Tuple t = t(
                (long) i,
                new HashMap<String, Object>() {
                    {
                        put("b test 1", t(1l, "test 1"));
                        put("b test 2", t(2l, "test 2"));
                    }
                });

            data.add(t);
        }
        verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true);
        verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false);

    }
}