src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hcatalog.data;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;

import junit.framework.Assert;
import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestHCatRecordSerDe extends TestCase {

    private static final Logger LOG = LoggerFactory.getLogger(TestHCatRecordSerDe.class);

    public Map<Properties, HCatRecord> getData() {
        Map<Properties, HCatRecord> data = new HashMap<Properties, HCatRecord>();

        List<Object> rlist = new ArrayList<Object>(11);
        rlist.add(new Byte("123"));
        rlist.add(new Short("456"));
        rlist.add(new Integer(789));
        rlist.add(new Long(1000L));
        rlist.add(new Double(5.3D));
        rlist.add(new Float(2.39F));
        rlist.add(new String("hcat and hadoop"));
        rlist.add(null);

        List<Object> innerStruct = new ArrayList<Object>(2);
        innerStruct.add(new String("abc"));
        innerStruct.add(new String("def"));
        rlist.add(innerStruct);

        List<Integer> innerList = new ArrayList<Integer>();
        innerList.add(314);
        innerList.add(007);
        rlist.add(innerList);

        Map<Short, String> map = new HashMap<Short, String>(3);
        map.put(new Short("2"), "hcat is cool");
        map.put(new Short("3"), "is it?");
        map.put(new Short("4"), "or is it not?");
        rlist.add(map);

        rlist.add(new Boolean(true));

        List<Object> c1 = new ArrayList<Object>();
        List<Object> c1_1 = new ArrayList<Object>();
        c1_1.add(new Integer(12));
        List<Object> i2 = new ArrayList<Object>();
        List<Integer> ii1 = new ArrayList<Integer>();
        ii1.add(new Integer(13));
        ii1.add(new Integer(14));
        i2.add(ii1);
        Map<String, List<?>> ii2 = new HashMap<String, List<?>>();
        List<Integer> iii1 = new ArrayList<Integer>();
        iii1.add(new Integer(15));
        ii2.put("phew", iii1);
        i2.add(ii2);
        c1_1.add(i2);
        c1.add(c1_1);
        rlist.add(c1);
        List<Object> am = new ArrayList<Object>();
        Map<String, String> am_1 = new HashMap<String, String>();
        am_1.put("noo", "haha");
        am.add(am_1);
        rlist.add(am);
        List<Object> aa = new ArrayList<Object>();
        List<String> aa_1 = new ArrayList<String>();
        aa_1.add("bloo");
        aa_1.add("bwahaha");
        aa.add(aa_1);
        rlist.add(aa);

        String typeString =
                "tinyint,smallint,int,bigint,double,float,string,string,"
                        + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean,"
                        + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>,"
                        + "array<map<string,string>>,array<array<string>>";
        Properties props = new Properties();

        props.put(Constants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1,am,aa");
        props.put(Constants.LIST_COLUMN_TYPES, typeString);
//    props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
//    props.put(Constants.SERIALIZATION_FORMAT, "1");

        data.put(props, new DefaultHCatRecord(rlist));
        return data;
    }

    public void testRW() throws Exception {

        Configuration conf = new Configuration();

        for (Entry<Properties, HCatRecord> e : getData().entrySet()) {
            Properties tblProps = e.getKey();
            HCatRecord r = e.getValue();

            HCatRecordSerDe hrsd = new HCatRecordSerDe();
            hrsd.initialize(conf, tblProps);

            LOG.info("ORIG: {}", r);

            Writable s = hrsd.serialize(r, hrsd.getObjectInspector());
            LOG.info("ONE: {}", s);

            HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
            Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, r2));

            // If it went through correctly, then s is also a HCatRecord,
            // and also equal to the above, and a deepcopy, and this holds
            // through for multiple levels more of serialization as well.

            Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
            LOG.info("TWO: {}", s2);
            Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
            Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));

            // serialize using another serde, and read out that object repr.
            LazySimpleSerDe testSD = new LazySimpleSerDe();
            testSD.initialize(conf, tblProps);

            Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
            LOG.info("THREE: {}", s3);
            Object o3 = testSD.deserialize(s3);
            Assert.assertFalse(r.getClass().equals(o3.getClass()));

            // then serialize again using hrsd, and compare results
            HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
            LOG.info("FOUR: {}", s4);

            // Test LazyHCatRecord init and read
            LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
            LOG.info("FIVE: {}", s5);

            LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
            LOG.info("SIX: {}", s6);

        }

    }

}