src/test/org/apache/hcatalog/data/TestJsonSerDe.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hcatalog.data;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestJsonSerDe extends TestCase {

    private static final Logger LOG = LoggerFactory.getLogger(TestJsonSerDe.class);

    public List<Pair<Properties, HCatRecord>> getData() {
        List<Pair<Properties, HCatRecord>> data = new ArrayList<Pair<Properties, HCatRecord>>();

        List<Object> rlist = new ArrayList<Object>(13);
        rlist.add(new Byte("123"));
        rlist.add(new Short("456"));
        rlist.add(new Integer(789));
        rlist.add(new Long(1000L));
        rlist.add(new Double(5.3D));
        rlist.add(new Float(2.39F));
        rlist.add(new String("hcat and hadoop"));
        rlist.add(null);

        List<Object> innerStruct = new ArrayList<Object>(2);
        innerStruct.add(new String("abc"));
        innerStruct.add(new String("def"));
        rlist.add(innerStruct);

        List<Integer> innerList = new ArrayList<Integer>();
        innerList.add(314);
        innerList.add(007);
        rlist.add(innerList);

        Map<Short, String> map = new HashMap<Short, String>(3);
        map.put(new Short("2"), "hcat is cool");
        map.put(new Short("3"), "is it?");
        map.put(new Short("4"), "or is it not?");
        rlist.add(map);

        rlist.add(new Boolean(true));

        List<Object> c1 = new ArrayList<Object>();
        List<Object> c1_1 = new ArrayList<Object>();
        c1_1.add(new Integer(12));
        List<Object> i2 = new ArrayList<Object>();
        List<Integer> ii1 = new ArrayList<Integer>();
        ii1.add(new Integer(13));
        ii1.add(new Integer(14));
        i2.add(ii1);
        Map<String, List<?>> ii2 = new HashMap<String, List<?>>();
        List<Integer> iii1 = new ArrayList<Integer>();
        iii1.add(new Integer(15));
        ii2.put("phew", iii1);
        i2.add(ii2);
        c1_1.add(i2);
        c1.add(c1_1);
        rlist.add(c1);

        List<Object> nlist = new ArrayList<Object>(13);
        nlist.add(null); // tinyint
        nlist.add(null); // smallint
        nlist.add(null); // int
        nlist.add(null); // bigint
        nlist.add(null); // double
        nlist.add(null); // float
        nlist.add(null); // string
        nlist.add(null); // string
        nlist.add(null); // struct
        nlist.add(null); // array
        nlist.add(null); // map
        nlist.add(null); // bool
        nlist.add(null); // complex

        String typeString =
                "tinyint,smallint,int,bigint,double,float,string,string,"
                        + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean,"
                        + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>";
        Properties props = new Properties();

        props.put(Constants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1");
        props.put(Constants.LIST_COLUMN_TYPES, typeString);
//    props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
//    props.put(Constants.SERIALIZATION_FORMAT, "1");

        data.add(new Pair(props, new DefaultHCatRecord(rlist)));
        data.add(new Pair(props, new DefaultHCatRecord(nlist)));
        return data;
    }

    public void testRW() throws Exception {

        Configuration conf = new Configuration();

        for (Pair<Properties, HCatRecord> e : getData()) {
            Properties tblProps = e.first;
            HCatRecord r = e.second;

            HCatRecordSerDe hrsd = new HCatRecordSerDe();
            hrsd.initialize(conf, tblProps);

            JsonSerDe jsde = new JsonSerDe();
            jsde.initialize(conf, tblProps);

            LOG.info("ORIG:{}", r);

            Writable s = hrsd.serialize(r, hrsd.getObjectInspector());
            LOG.info("ONE:{}", s);

            Object o1 = hrsd.deserialize(s);
            assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o1));

            Writable s2 = jsde.serialize(o1, hrsd.getObjectInspector());
            LOG.info("TWO:{}", s2);
            Object o2 = jsde.deserialize(s2);
            LOG.info("deserialized TWO : {} ", o2);

            assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2));
        }

    }

    public void testRobustRead() throws Exception {
        /**
         *  This test has been added to account for HCATALOG-436
         *  We write out columns with "internal column names" such
         *  as "_col0", but try to read with retular column names.
         */

        Configuration conf = new Configuration();

        for (Pair<Properties, HCatRecord> e : getData()) {
            Properties tblProps = e.first;
            HCatRecord r = e.second;

            Properties internalTblProps = new Properties();
            for (Map.Entry pe : tblProps.entrySet()) {
                if (!pe.getKey().equals(Constants.LIST_COLUMNS)) {
                    internalTblProps.put(pe.getKey(), pe.getValue());
                } else {
                    internalTblProps.put(pe.getKey(), getInternalNames((String) pe.getValue()));
                }
            }

            LOG.info("orig tbl props:{}", tblProps);
            LOG.info("modif tbl props:{}", internalTblProps);

            JsonSerDe wjsd = new JsonSerDe();
            wjsd.initialize(conf, internalTblProps);

            JsonSerDe rjsd = new JsonSerDe();
            rjsd.initialize(conf, tblProps);

            LOG.info("ORIG:{}", r);

            Writable s = wjsd.serialize(r, wjsd.getObjectInspector());
            LOG.info("ONE:{}", s);

            Object o1 = wjsd.deserialize(s);
            LOG.info("deserialized ONE : {} ", o1);

            Object o2 = rjsd.deserialize(s);
            LOG.info("deserialized TWO : {} ", o2);
            assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2));
        }

    }

    String getInternalNames(String columnNames) {
        if (columnNames == null) {
            return null;
        }
        if (columnNames.isEmpty()) {
            return "";
        }

        StringBuffer sb = new StringBuffer();
        int numStrings = columnNames.split(",").length;
        sb.append("_col0");
        for (int i = 1; i < numStrings; i++) {
            sb.append(",");
            sb.append(HiveConf.getColumnInternalName(i));
        }
        return sb.toString();
    }
}