src/test/org/apache/hcatalog/common/TestHCatUtil.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hcatalog.common;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.junit.Assert;
import org.junit.Test;

public class TestHCatUtil {

    @Test
    public void testFsPermissionOperation() {

        HashMap<String, Integer> permsCode = new HashMap<String, Integer>();

        for (int i = 0; i < 8; i++) {
            for (int j = 0; j < 8; j++) {
                for (int k = 0; k < 8; k++) {
                    StringBuilder sb = new StringBuilder();
                    sb.append("0");
                    sb.append(i);
                    sb.append(j);
                    sb.append(k);
                    Integer code = (((i * 8) + j) * 8) + k;
                    String perms = (new FsPermission(Short.decode(sb.toString()))).toString();
                    if (permsCode.containsKey(perms)) {
                        Assert.assertEquals("permissions(" + perms + ") mapped to multiple codes", code, permsCode.get(perms));
                    }
                    permsCode.put(perms, code);
                    assertFsPermissionTransformationIsGood(perms);
                }
            }
        }
    }

    private void assertFsPermissionTransformationIsGood(String perms) {
        Assert.assertEquals(perms, FsPermission.valueOf("-" + perms).toString());
    }

    @Test
    public void testValidateMorePermissive() {
        assertConsistentFsPermissionBehaviour(FsAction.ALL, true, true, true, true, true, true, true, true);
        assertConsistentFsPermissionBehaviour(FsAction.READ, false, true, false, true, false, false, false, false);
        assertConsistentFsPermissionBehaviour(FsAction.WRITE, false, true, false, false, true, false, false, false);
        assertConsistentFsPermissionBehaviour(FsAction.EXECUTE, false, true, true, false, false, false, false, false);
        assertConsistentFsPermissionBehaviour(FsAction.READ_EXECUTE, false, true, true, true, false, true, false, false);
        assertConsistentFsPermissionBehaviour(FsAction.READ_WRITE, false, true, false, true, true, false, true, false);
        assertConsistentFsPermissionBehaviour(FsAction.WRITE_EXECUTE, false, true, true, false, true, false, false, true);
        assertConsistentFsPermissionBehaviour(FsAction.NONE, false, true, false, false, false, false, false, false);
    }


    private void assertConsistentFsPermissionBehaviour(
            FsAction base, boolean versusAll, boolean versusNone,
            boolean versusX, boolean versusR, boolean versusW,
            boolean versusRX, boolean versusRW, boolean versusWX) {

        Assert.assertTrue(versusAll == HCatUtil.validateMorePermissive(base, FsAction.ALL));
        Assert.assertTrue(versusX == HCatUtil.validateMorePermissive(base, FsAction.EXECUTE));
        Assert.assertTrue(versusNone == HCatUtil.validateMorePermissive(base, FsAction.NONE));
        Assert.assertTrue(versusR == HCatUtil.validateMorePermissive(base, FsAction.READ));
        Assert.assertTrue(versusRX == HCatUtil.validateMorePermissive(base, FsAction.READ_EXECUTE));
        Assert.assertTrue(versusRW == HCatUtil.validateMorePermissive(base, FsAction.READ_WRITE));
        Assert.assertTrue(versusW == HCatUtil.validateMorePermissive(base, FsAction.WRITE));
        Assert.assertTrue(versusWX == HCatUtil.validateMorePermissive(base, FsAction.WRITE_EXECUTE));
    }

    @Test
    public void testExecutePermissionsCheck() {
        Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.ALL));
        Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.NONE));
        Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.EXECUTE));
        Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_EXECUTE));
        Assert.assertTrue(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE_EXECUTE));

        Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ));
        Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.WRITE));
        Assert.assertFalse(HCatUtil.validateExecuteBitPresentIfReadOrWrite(FsAction.READ_WRITE));

    }

    @Test
    public void testGetTableSchemaWithPtnColsApi() throws IOException {
        // Check the schema of a table with one field & no partition keys.
        StorageDescriptor sd = new StorageDescriptor(
                Lists.newArrayList(new FieldSchema("username", Constants.STRING_TYPE_NAME, null)),
                "location", "org.apache.hadoop.mapred.TextInputFormat",
                "org.apache.hadoop.mapred.TextOutputFormat", false, -1, new SerDeInfo(),
                new ArrayList<String>(), new ArrayList<Order>(), new HashMap<String, String>());
        org.apache.hadoop.hive.metastore.api.Table apiTable =
                new org.apache.hadoop.hive.metastore.api.Table("test_tblname", "test_dbname", "test_owner",
                        0, 0, 0, sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
                        "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
        Table table = new Table(apiTable);

        List<HCatFieldSchema> expectedHCatSchema =
                Lists.newArrayList(new HCatFieldSchema("username", HCatFieldSchema.Type.STRING, null));

        Assert.assertEquals(new HCatSchema(expectedHCatSchema),
                HCatUtil.getTableSchemaWithPtnCols(table));

        // Add a partition key & ensure its reflected in the schema.
        List<FieldSchema> partitionKeys =
                Lists.newArrayList(new FieldSchema("dt", Constants.STRING_TYPE_NAME, null));
        table.getTTable().setPartitionKeys(partitionKeys);
        expectedHCatSchema.add(new HCatFieldSchema("dt", HCatFieldSchema.Type.STRING, null));
        Assert.assertEquals(new HCatSchema(expectedHCatSchema),
                HCatUtil.getTableSchemaWithPtnCols(table));
    }

    /**
     * Hive represents tables in two ways:
     * <ul>
     *   <li>org.apache.hadoop.hive.metastore.api.Table - exactly whats stored in the metastore</li>
     *   <li>org.apache.hadoop.hive.ql.metadata.Table - adds business logic over api.Table</li>
     * </ul>
     * Here we check SerDe-reported fields are included in the table schema.
     */
    @Test
    public void testGetTableSchemaWithPtnColsSerDeReportedFields() throws IOException {
        Map<String, String> parameters = Maps.newHashMap();
        parameters.put(Constants.SERIALIZATION_CLASS,
                "org.apache.hadoop.hive.serde2.thrift.test.IntString");
        parameters.put(Constants.SERIALIZATION_FORMAT, "org.apache.thrift.protocol.TBinaryProtocol");

        SerDeInfo serDeInfo = new SerDeInfo(null,
                "org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer", parameters);

        // StorageDescriptor has an empty list of fields - SerDe will report them.
        StorageDescriptor sd = new StorageDescriptor(new ArrayList<FieldSchema>(), "location",
                "org.apache.hadoop.mapred.TextInputFormat", "org.apache.hadoop.mapred.TextOutputFormat",
                false, -1, serDeInfo, new ArrayList<String>(), new ArrayList<Order>(),
                new HashMap<String, String>());

        org.apache.hadoop.hive.metastore.api.Table apiTable =
                new org.apache.hadoop.hive.metastore.api.Table("test_tblname", "test_dbname", "test_owner",
                        0, 0, 0, sd, new ArrayList<FieldSchema>(), new HashMap<String, String>(),
                        "viewOriginalText", "viewExpandedText", TableType.EXTERNAL_TABLE.name());
        Table table = new Table(apiTable);

        List<HCatFieldSchema> expectedHCatSchema = Lists.newArrayList(
                new HCatFieldSchema("myint", HCatFieldSchema.Type.INT, null),
                new HCatFieldSchema("mystring", HCatFieldSchema.Type.STRING, null),
                new HCatFieldSchema("underscore_int", HCatFieldSchema.Type.INT, null));

        Assert.assertEquals(new HCatSchema(expectedHCatSchema),
                HCatUtil.getTableSchemaWithPtnCols(table));
    }
}