// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.impala.catalog;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.impala.analysis.FunctionName;
import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.analysis.NumericLiteral;
import org.apache.impala.authorization.AuthorizationPolicy;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Reference;
import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.apache.impala.thrift.TGetPartitionStatsRequest;
import org.apache.impala.thrift.TPartitionKeyValue;
import org.apache.impala.thrift.TPartitionStats;
import org.apache.impala.thrift.TPrincipalType;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TPrivilegeLevel;
import org.apache.impala.thrift.TPrivilegeScope;
import org.apache.impala.thrift.TTableName;
import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

public class CatalogTest {
  private CatalogServiceCatalog catalog_;

  @Before
  public void init() {
    catalog_ = CatalogServiceTestCatalog.create();
  }

  @After
  public void cleanUp() { catalog_.close(); }

  public static void checkTableCols(FeDb db, String tblName, int numClusteringCols,
      String[] colNames, Type[] colTypes) throws TableLoadingException {
    FeTable tbl = db.getTable(tblName);
    assertEquals(tbl.getName(), tblName);
    assertEquals(tbl.getNumClusteringCols(), numClusteringCols);
    List<Column> cols = tbl.getColumns();
    assertEquals(colNames.length, colTypes.length);
    assertEquals(cols.size(), colNames.length);
    Iterator<Column> it = cols.iterator();
    int i = 0;
    while (it.hasNext()) {
      Column col = it.next();
      assertEquals(col.getName(), colNames[i]);
      assertTrue(col.getType().equals(colTypes[i]));
      ++i;
    }
  }

  private void checkHBaseTableCols(Db db, String hiveTableName, String hbaseTableName,
      String[] hiveColNames, String[] colFamilies, String[] colQualifiers,
      Type[] colTypes) throws TableLoadingException{
    checkTableCols(db, hiveTableName, 1, hiveColNames, colTypes);
    HBaseTable tbl = (HBaseTable) db.getTable(hiveTableName);
    assertEquals(tbl.getHBaseTableName(), hbaseTableName);
    List<Column> cols = tbl.getColumns();
    assertEquals(colFamilies.length, colTypes.length);
    assertEquals(colQualifiers.length, colTypes.length);
    Iterator<Column> it = cols.iterator();
    int i = 0;
    while (it.hasNext()) {
      HBaseColumn col = (HBaseColumn)it.next();
      assertEquals(col.getColumnFamily(), colFamilies[i]);
      assertEquals(col.getColumnQualifier(), colQualifiers[i]);
      ++i;
    }
  }

  @Test
  public void TestColSchema() throws CatalogException {
    Db functionalDb = catalog_.getDb("functional");
    assertNotNull(functionalDb);
    assertEquals(functionalDb.getName(), "functional");
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypes", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypes_view", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypes_view_sub", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypessmall", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypeserror", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypeserrornonulls", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypesagg", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypesaggnonulls", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypesnopart", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "alltypesinsert", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "complex_view", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "testtbl", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "dimtbl", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "jointbl", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "liketbl", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "greptiny", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "rankingssmall", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "uservisitssmall", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "view_view", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional", "date_tbl", "test"));
    // IMP-163 - table with string partition column does not load if there are partitions
    assertNotNull(catalog_.getOrLoadTable("functional", "StringPartitionKey", "test"));
    // Test non-existent table
    assertNull(catalog_.getOrLoadTable("functional", "nonexistenttable", "test"));

    // functional_seq contains the same tables as functional
    Db testDb = catalog_.getDb("functional_seq");
    assertNotNull(testDb);
    assertEquals(testDb.getName(), "functional_seq");
    assertNotNull(catalog_.getOrLoadTable("functional_seq", "alltypes", "test"));
    assertNotNull(catalog_.getOrLoadTable("functional_seq", "testtbl", "test"));

    Db hbaseDb = catalog_.getDb("functional_hbase");
    assertNotNull(hbaseDb);
    assertEquals(hbaseDb.getName(), "functional_hbase");
    // Loading succeeds for an HBase table that has binary columns and an implicit key
    // column mapping
    assertNotNull(catalog_.getOrLoadTable(hbaseDb.getName(), "alltypessmallbinary",
        "test"));
    assertNotNull(catalog_.getOrLoadTable(hbaseDb.getName(), "alltypessmall", "test"));
    assertNotNull(catalog_.getOrLoadTable(hbaseDb.getName(), "hbasealltypeserror",
        "test"));
    assertNotNull(catalog_.getOrLoadTable(hbaseDb.getName(),
        "hbasealltypeserrornonulls", "test"));
    assertNotNull(catalog_.getOrLoadTable(hbaseDb.getName(), "alltypesagg", "test"));
    assertNotNull(catalog_.getOrLoadTable(hbaseDb.getName(), "stringids", "test"));

    checkTableCols(functionalDb, "alltypes", 2,
        new String[]
          {"year", "month", "id", "bool_col", "tinyint_col", "smallint_col",
           "int_col", "bigint_col", "float_col", "double_col", "date_string_col",
           "string_col", "timestamp_col"},
        new Type[]
          {Type.INT, Type.INT, Type.INT,
           Type.BOOLEAN, Type.TINYINT, Type.SMALLINT,
           Type.INT, Type.BIGINT, Type.FLOAT,
           Type.DOUBLE, Type.STRING, Type.STRING,
           Type.TIMESTAMP});
    checkTableCols(functionalDb, "testtbl", 0,
        new String[] {"id", "name", "zip"},
        new Type[]
          {Type.BIGINT, Type.STRING, Type.INT});
    checkTableCols(testDb, "testtbl", 0,
        new String[] {"id", "name", "zip"},
        new Type[]
          {Type.BIGINT, Type.STRING, Type.INT});
    checkTableCols(functionalDb, "liketbl", 0,
        new String[] {
            "str_col", "match_like_col", "no_match_like_col", "match_regex_col",
            "no_match_regex_col"},
        new Type[]
          {Type.STRING, Type.STRING, Type.STRING,
           Type.STRING, Type.STRING});
    checkTableCols(functionalDb, "dimtbl", 0,
        new String[] {"id", "name", "zip"},
        new Type[]
          {Type.BIGINT, Type.STRING, Type.INT});
    checkTableCols(functionalDb, "jointbl", 0,
        new String[] {"test_id", "test_name", "test_zip", "alltypes_id"},
        new Type[]
          {Type.BIGINT, Type.STRING, Type.INT,
           Type.INT});

    checkHBaseTableCols(hbaseDb, "alltypessmall", "functional_hbase.alltypessmall",
        new String[]
          {"id", "bigint_col", "bool_col", "date_string_col", "double_col", "float_col",
           "int_col", "month", "smallint_col", "string_col", "timestamp_col",
           "tinyint_col", "year"},
        new String[]
          {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[]
          {null, "bigint_col", "bool_col", "date_string_col", "double_col", "float_col",
           "int_col", "month", "smallint_col", "string_col", "timestamp_col",
           "tinyint_col", "year"},
        new Type[]
          {Type.INT, Type.BIGINT, Type.BOOLEAN,
           Type.STRING, Type.DOUBLE, Type.FLOAT,
           Type.INT, Type.INT, Type.SMALLINT,
           Type.STRING, Type.TIMESTAMP,
           Type.TINYINT, Type.INT});

    checkHBaseTableCols(hbaseDb, "hbasealltypeserror",
        "functional_hbase.hbasealltypeserror",
        new String[]
          {"id", "bigint_col", "bool_col","date_string_col", "double_col", "float_col",
           "int_col", "smallint_col", "string_col","timestamp_col", "tinyint_col"},
        new String[]
          {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[]
          {null, "bigint_col", "bool_col","date_string_col", "double_col", "float_col",
           "int_col", "smallint_col", "string_col","timestamp_col", "tinyint_col"},
        new Type[]
          {Type.INT, Type.BIGINT, Type.BOOLEAN,
           Type.STRING, Type.DOUBLE, Type.FLOAT,
           Type.INT, Type.SMALLINT, Type.STRING,
           Type.TIMESTAMP, Type.TINYINT});

    checkHBaseTableCols(hbaseDb, "hbasealltypeserrornonulls",
        "functional_hbase.hbasealltypeserrornonulls",
        new String[]
          {"id", "bigint_col", "bool_col", "date_string_col", "double_col", "float_col",
           "int_col", "smallint_col", "string_col","timestamp_col", "tinyint_col"},
        new String[]
          {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[]
          {null, "bigint_col", "bool_col", "date_string_col", "double_col", "float_col",
           "int_col", "smallint_col", "string_col","timestamp_col", "tinyint_col"},
        new Type[]
          {Type.INT, Type.BIGINT, Type.BOOLEAN,
           Type.STRING, Type.DOUBLE, Type.FLOAT,
           Type.INT, Type.SMALLINT, Type.STRING,
           Type.TIMESTAMP, Type.TINYINT});

    checkHBaseTableCols(hbaseDb, "alltypesagg", "functional_hbase.alltypesagg",
        new String[]
          {"id", "bigint_col", "bool_col", "date_string_col", "day", "double_col",
           "float_col", "int_col", "month", "smallint_col", "string_col",
           "timestamp_col", "tinyint_col", "year"},
        new String[]
          {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[]
          {null, "bigint_col", "bool_col", "date_string_col", "day", "double_col",
           "float_col", "int_col", "month", "smallint_col", "string_col",
           "timestamp_col", "tinyint_col", "year"},
        new Type[]
          {Type.INT, Type.BIGINT, Type.BOOLEAN,
           Type.STRING,Type.INT, Type.DOUBLE,
           Type.FLOAT, Type.INT, Type.INT,
           Type.SMALLINT, Type.STRING, Type.TIMESTAMP,
           Type.TINYINT, Type.INT});

    checkHBaseTableCols(hbaseDb, "stringids", "functional_hbase.alltypesagg",
        new String[]
          {"id", "bigint_col", "bool_col", "date_string_col", "day", "double_col",
           "float_col", "int_col", "month", "smallint_col", "string_col",
           "timestamp_col", "tinyint_col", "year"},
        new String[]
          {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[]
          {null, "bigint_col", "bool_col", "date_string_col", "day", "double_col",
           "float_col", "int_col", "month", "smallint_col", "string_col",
           "timestamp_col", "tinyint_col", "year"},
        new Type[]
          {Type.STRING, Type.BIGINT, Type.BOOLEAN,
           Type.STRING,Type.INT, Type.DOUBLE,
           Type.FLOAT, Type.INT, Type.INT,
           Type.SMALLINT, Type.STRING, Type.TIMESTAMP,
           Type.TINYINT, Type.INT});

    checkTableCols(functionalDb, "greptiny", 0,
        new String[]
          {"field"},
        new Type[]
          {Type.STRING});

    checkTableCols(functionalDb, "rankingssmall", 0,
        new String[]
          {"pagerank", "pageurl", "avgduration"},
        new Type[]
          {Type.INT, Type.STRING, Type.INT});

    checkTableCols(functionalDb, "uservisitssmall", 0,
        new String[]
          {"sourceip", "desturl", "visitdate",  "adrevenue", "useragent",
           "ccode", "lcode", "skeyword", "avgtimeonsite"},
        new Type[]
          {Type.STRING, Type.STRING, Type.STRING,
           Type.FLOAT, Type.STRING, Type.STRING,
           Type.STRING, Type.STRING, Type.INT});

    checkTableCols(functionalDb, "date_tbl", 1,
        new String[] {"date_part", "id_col", "date_col"},
        new Type[] {Type.DATE, Type.INT, Type.DATE});

    // case-insensitive lookup
    assertEquals(catalog_.getOrLoadTable("functional", "alltypes", "test"),
        catalog_.getOrLoadTable("functional", "AllTypes", "test"));
  }

  // Count of listFiles (list status + blocks) calls
  private static final String LIST_LOCATED_STATUS =
      OpType.LIST_LOCATED_STATUS.getSymbol();
  // Count of listStatus calls
  private static final String LIST_STATUS = OpType.LIST_STATUS.getSymbol();
  // Count of getStatus calls
  private static final String GET_FILE_STATUS = OpType.GET_FILE_STATUS.getSymbol();
  // Count of getFileBlockLocations() calls
  private static final String GET_FILE_BLOCK_LOCS =
      OpType.GET_FILE_BLOCK_LOCATIONS.getSymbol();

  /**
   * Regression test for IMPALA-7320 and IMPALA-7047: we should use batch APIs to fetch
   * file permissions for partitions when loading or reloading.
   */
  @Test
  public void testNumberOfGetFileStatusCalls() throws CatalogException, IOException {
    // Reset the filesystem statistics and load the table, ensuring that it's
    // loaded fresh by invalidating it first.
    GlobalStorageStatistics stats = FileSystem.getGlobalStorageStatistics();
    stats.reset();
    catalog_.invalidateTable(new TTableName("functional", "alltypes"),
        /*tblWasRemoved=*/new Reference<Boolean>(),
        /*dbWasAdded=*/new Reference<Boolean>());

    HdfsTable table = (HdfsTable)catalog_.getOrLoadTable("functional", "AllTypes",
        "test");
    StorageStatistics opsCounts = stats.get(DFSOpsCountStatistics.NAME);

    // We expect:
    // - one listLocatedStatus() per partition, to get the file info
    // - one listStatus() for the month=2010/ dir
    // - one listStatus() for the month=2009/ dir
    long expectedCalls = table.getPartitionIds().size() + 2;
    // Due to HDFS-13747, the listStatus calls are incorrectly accounted as
    // op_list_located_status. So, we'll just add up the two to make our
    // assertion resilient to this bug.
    long seenCalls = opsCounts.getLong(LIST_LOCATED_STATUS) +
        opsCounts.getLong(LIST_STATUS);
    assertEquals(expectedCalls, seenCalls);

    // We expect only one getFileStatus call, for the top-level directory.
    assertEquals(1L, (long)opsCounts.getLong(GET_FILE_STATUS));

    // None of the underlying files changed so we should not do any ops for the files.
    assertEquals(0L, (long)opsCounts.getLong(GET_FILE_BLOCK_LOCS));

    // Now test REFRESH on the table...
    stats.reset();
    catalog_.reloadTable(table, "test");

    // Again, we expect only one getFileStatus call, for the top-level directory.
    assertEquals(1L, (long)opsCounts.getLong(GET_FILE_STATUS));
    // REFRESH calls listStatus on each of the partitions, but doesn't re-check
    // the permissions of the partition directories themselves.
    seenCalls = opsCounts.getLong(LIST_LOCATED_STATUS) +
        opsCounts.getLong(LIST_STATUS);
    assertEquals(table.getPartitionIds().size(), seenCalls);
    // None of the underlying files changed so we should not do any ops for the files.
    assertEquals(0L, (long)opsCounts.getLong(GET_FILE_BLOCK_LOCS));

    // Reloading a specific partition should not make an RPC per file
    // (regression test for IMPALA-7047).
    stats.reset();
    List<TPartitionKeyValue> partitionSpec = ImmutableList.of(
        new TPartitionKeyValue("year", "2010"),
        new TPartitionKeyValue("month", "10"));
    catalog_.reloadPartition(table, partitionSpec, "test");
    assertEquals(0L, (long)opsCounts.getLong(GET_FILE_BLOCK_LOCS));

    // Loading or reloading an unpartitioned table with some files in it should not make
    // an RPC per file.
    stats.reset();
    HdfsTable unpartTable = (HdfsTable)catalog_.getOrLoadTable(
        "functional", "alltypesaggmultifilesnopart", "test");
    assertEquals(0L, (long)opsCounts.getLong(GET_FILE_BLOCK_LOCS));
    stats.reset();
    catalog_.reloadTable(unpartTable, "test");
    assertEquals(0L, (long)opsCounts.getLong(GET_FILE_BLOCK_LOCS));

    // Simulate an empty partition, which will trigger the full
    // reload path. Since we can't modify HDFS itself via these tests, we
    // do the next best thing: modify the metadata to revise history as
    // though the partition used above were actually empty.
    HdfsPartition hdfsPartition = table
        .getPartitionFromThriftPartitionSpec(partitionSpec);
    hdfsPartition.setFileDescriptors(new ArrayList<>());
    stats.reset();
    catalog_.reloadPartition(table, partitionSpec, "test");

    // Should not scan the directory file-by-file, should use a single
    // listLocatedStatus() to get the whole directory (partition)
    assertEquals(0L, (long)opsCounts.getLong(GET_FILE_BLOCK_LOCS));
    seenCalls = opsCounts.getLong(LIST_LOCATED_STATUS) +
        opsCounts.getLong(LIST_STATUS);
    assertEquals(1, seenCalls);
  }

  @Test
  public void TestPartitions() throws CatalogException {
    HdfsTable table =
        (HdfsTable) catalog_.getOrLoadTable("functional", "AllTypes", "test");
    checkAllTypesPartitioning(table, true);
  }

  public static void checkAllTypesPartitioning(FeFsTable table,
      boolean checkFileDescriptors) {
    assertEquals(24, table.getPartitionIds().size());
    assertEquals(24, table.getPartitions().size());
    Collection<? extends FeFsPartition> partitions =
        FeCatalogUtils.loadAllPartitions(table);

    // check that partition keys cover the date range 1/1/2009-12/31/2010
    // and that we have one file per partition.
    assertEquals(24, partitions.size());
    Set<HdfsStorageDescriptor> uniqueSds = Collections.newSetFromMap(
        new IdentityHashMap<HdfsStorageDescriptor, Boolean>());
    Set<Long> months = new HashSet<>();
    for (FeFsPartition p: partitions) {
      assertEquals(2, p.getPartitionValues().size());

      LiteralExpr key1Expr = p.getPartitionValues().get(0);
      assertTrue(key1Expr instanceof NumericLiteral);
      long key1 = ((NumericLiteral) key1Expr).getLongValue();
      assertTrue(key1 == 2009 || key1 == 2010);

      LiteralExpr key2Expr = p.getPartitionValues().get(1);
      assertTrue(key2Expr instanceof NumericLiteral);
      long key2 = ((NumericLiteral) key2Expr).getLongValue();
      assertTrue(key2 >= 1 && key2 <= 12);

      months.add(key1 * 100 + key2);

      if (checkFileDescriptors) {
        // TODO(todd): once LocalCatalog supports file descriptors,
        // no need for this boolean anymore.
        assertEquals(p.getFileDescriptors().size(), 1);
      }

      uniqueSds.add(p.getInputFormatDescriptor());
    }
    assertEquals(months.size(), 24);

    // We intern storage descriptors, so we should only have a single instance across
    // all of the partitions.
    assertEquals(1, uniqueSds.size());
  }

  @Test
  public void testStats() throws CatalogException {
    // make sure the stats for functional.alltypesagg look correct
    HdfsTable table = (HdfsTable) catalog_.getOrLoadTable("functional", "AllTypesAgg",
        "test");

    Column idCol = table.getColumn("id");
    assertEquals(idCol.getStats().getAvgSerializedSize(),
        PrimitiveType.INT.getSlotSize(), 0.0001);
    assertEquals(idCol.getStats().getMaxSize(), PrimitiveType.INT.getSlotSize());
    assertFalse(idCol.getStats().hasNulls());

    Column boolCol = table.getColumn("bool_col");
    assertEquals(boolCol.getStats().getAvgSerializedSize(),
        PrimitiveType.BOOLEAN.getSlotSize(), 0.0001);
    assertEquals(boolCol.getStats().getMaxSize(), PrimitiveType.BOOLEAN.getSlotSize());
    assertFalse(boolCol.getStats().hasNulls());

    Column tinyintCol = table.getColumn("tinyint_col");
    assertEquals(tinyintCol.getStats().getAvgSerializedSize(),
        PrimitiveType.TINYINT.getSlotSize(), 0.0001);
    assertEquals(tinyintCol.getStats().getMaxSize(), PrimitiveType.TINYINT.getSlotSize());
    assertTrue(tinyintCol.getStats().hasNulls());

    Column smallintCol = table.getColumn("smallint_col");
    assertEquals(smallintCol.getStats().getAvgSerializedSize(),
        PrimitiveType.SMALLINT.getSlotSize(), 0.0001);
    assertEquals(smallintCol.getStats().getMaxSize(),
        PrimitiveType.SMALLINT.getSlotSize());
    assertTrue(smallintCol.getStats().hasNulls());

    Column intCol = table.getColumn("int_col");
    assertEquals(intCol.getStats().getAvgSerializedSize(),
        PrimitiveType.INT.getSlotSize(), 0.0001);
    assertEquals(intCol.getStats().getMaxSize(), PrimitiveType.INT.getSlotSize());
    assertTrue(intCol.getStats().hasNulls());

    Column bigintCol = table.getColumn("bigint_col");
    assertEquals(bigintCol.getStats().getAvgSerializedSize(),
        PrimitiveType.BIGINT.getSlotSize(), 0.0001);
    assertEquals(bigintCol.getStats().getMaxSize(), PrimitiveType.BIGINT.getSlotSize());
    assertTrue(bigintCol.getStats().hasNulls());

    Column floatCol = table.getColumn("float_col");
    assertEquals(floatCol.getStats().getAvgSerializedSize(),
        PrimitiveType.FLOAT.getSlotSize(), 0.0001);
    assertEquals(floatCol.getStats().getMaxSize(), PrimitiveType.FLOAT.getSlotSize());
    assertTrue(floatCol.getStats().hasNulls());

    Column doubleCol = table.getColumn("double_col");
    assertEquals(doubleCol.getStats().getAvgSerializedSize(),
        PrimitiveType.DOUBLE.getSlotSize(), 0.0001);
    assertEquals(doubleCol.getStats().getMaxSize(), PrimitiveType.DOUBLE.getSlotSize());
    assertTrue(doubleCol.getStats().hasNulls());

    Column timestampCol = table.getColumn("timestamp_col");
    assertEquals(timestampCol.getStats().getAvgSerializedSize(),
        PrimitiveType.TIMESTAMP.getSlotSize(), 0.0001);
    assertEquals(timestampCol.getStats().getMaxSize(),
        PrimitiveType.TIMESTAMP.getSlotSize());
    assertFalse(timestampCol.getStats().hasNulls());

    Column stringCol = table.getColumn("string_col");
    assertTrue(stringCol.getStats().getAvgSerializedSize() > 0);
    assertTrue(stringCol.getStats().getMaxSize() > 0);
    assertFalse(stringCol.getStats().hasNulls());
  }

  /**
   * Verifies that updating column stats data for a type that isn't compatible with
   * the column type results in the stats being set to "unknown". This is a regression
   * test for IMPALA-588, where this used to result in a Preconditions failure.
   */
  @Test
  public void testColStatsColTypeMismatch() throws Exception {
    // First load a table that has column stats.
    //catalog_.refreshTable("functional", "alltypesagg", false);
    HdfsTable table = (HdfsTable) catalog_.getOrLoadTable("functional", "alltypesagg",
        "test");

    // Now attempt to update a column's stats with mismatched stats data and ensure
    // we get the expected results.
    try (MetaStoreClient client = catalog_.getMetaStoreClient()) {
      // Load some string stats data and use it to update the stats of different
      // typed columns.
      ColumnStatisticsData stringColStatsData = MetastoreShim.getTableColumnStatistics(
          client.getHiveClient(), "functional", "alltypesagg",
          Lists.newArrayList("string_col")).get(0).getStatsData();

      assertTrue(!table.getColumn("int_col").updateStats(stringColStatsData));
      assertStatsUnknown(table.getColumn("int_col"));

      assertTrue(!table.getColumn("double_col").updateStats(stringColStatsData));
      assertStatsUnknown(table.getColumn("double_col"));

      assertTrue(!table.getColumn("bool_col").updateStats(stringColStatsData));
      assertStatsUnknown(table.getColumn("bool_col"));

      // Do the same thing, but apply bigint stats to a string column.
      ColumnStatisticsData bigIntCol = MetastoreShim.getTableColumnStatistics(
          client.getHiveClient(), "functional", "alltypes",
          Lists.newArrayList("bigint_col")).get(0).getStatsData();
      assertTrue(!table.getColumn("string_col").updateStats(bigIntCol));
      assertStatsUnknown(table.getColumn("string_col"));

      // Now try to apply a matching column stats data and ensure it succeeds.
      assertTrue(table.getColumn("string_col").updateStats(stringColStatsData));
      assertEquals(963, table.getColumn("string_col").getStats().getNumDistinctValues());
    }
  }

  private void assertStatsUnknown(Column column) {
    assertEquals(-1, column.getStats().getNumDistinctValues());
    assertEquals(-1, column.getStats().getNumNulls());
    double expectedSize = column.getType().isFixedLengthType() ?
        column.getType().getSlotSize() : -1;
    assertEquals(expectedSize, column.getStats().getAvgSerializedSize(), 0.0001);
    assertEquals(expectedSize, column.getStats().getMaxSize(), 0.0001);
  }

  // Fetch partition statistics for dbName.tableName for partitionIds.
  private Map<String, ByteBuffer> getPartitionStatistics(String dbName, String tableName)
      throws CatalogException {
    TGetPartitionStatsRequest req = new TGetPartitionStatsRequest();
    req.setTable_name(new TTableName(dbName, tableName));
    return catalog_.getPartitionStats(req);
  }

  // Expect expCount partitions have statistics (though not incremental statistics).
  private void expectStatistics(String dbName, String tableName, int expCount)
      throws CatalogException {
    Map<String, ByteBuffer> result = getPartitionStatistics(dbName, tableName);
    assertEquals(expCount, result.size());
    for (Map.Entry<String, ByteBuffer> e : result.entrySet()) {
      ByteBuffer compressedBuffer = e.getValue();
      byte[] compressedBytes = new byte[compressedBuffer.remaining()];
      compressedBuffer.get(compressedBytes);
      try {
        TPartitionStats stats =
            PartitionStatsUtil.partStatsFromCompressedBytes(compressedBytes, null);
        assertNotNull(stats);
        assertTrue(!stats.isSetIntermediate_col_stats());
      } catch (ImpalaException ex) {
        throw new CatalogException("Error deserializing partition stats.", ex);
      }
    }
  }

  // Expect an exception whose message prefix-matches msgPrefix when fetching partition
  // statistics.
  private void expectStatisticsException(
      String dbName, String tableName, String msgPrefix) {
    try {
      getPartitionStatistics(dbName, tableName);
      fail("Expected exception.");
    } catch (Exception e) {
      assertTrue(e.getMessage(), e.getMessage().startsWith(msgPrefix));
    }
  }

  @Test
  public void testPullIncrementalStats() throws CatalogException {
    // Partitioned table with stats. Load the table prior to fetching.
    catalog_.getOrLoadTable("functional", "alltypesagg", "test");
    expectStatistics("functional", "alltypesagg", 11);

    // Partitioned table with stats. Invalidate the table prior to fetching.
    Reference<Boolean> tblWasRemoved = new Reference<Boolean>();
    Reference<Boolean> dbWasAdded = new Reference<Boolean>();
    catalog_.invalidateTable(
        new TTableName("functional", "alltypesagg"), tblWasRemoved, dbWasAdded);
    expectStatistics("functional", "alltypesagg", 11);

    // Unpartitioned table with no stats.
    expectStatistics("functional", "table_no_newline", 0);

    // Unpartitioned table with stats.
    expectStatistics("functional", "dimtbl", 0);

    // Bogus table.
    expectStatisticsException("functional", "doesnotexist",
        "Requested partition statistics for table that does not exist");

    // Case of IncompleteTable due to loading error.
    expectStatisticsException("functional", "bad_serde",
        "No statistics available for incompletely loaded table");
  }

  @Test
  public void testInternalHBaseTable() throws CatalogException {
    // Cast will fail if table not an HBaseTable
   HBaseTable table = (HBaseTable)
       catalog_.getOrLoadTable("functional_hbase", "internal_hbase_table", "test");
    assertNotNull("functional_hbase.internal_hbase_table was not found", table);
  }

  @Test
  public void testDatabaseDoesNotExist() {
    Db nonExistentDb = catalog_.getDb("doesnotexist");
    assertNull(nonExistentDb);
  }

  @Test
  public void testCreateTableMetadata() throws CatalogException {
    Table table = catalog_.getOrLoadTable("functional", "alltypes", "test");
    // Tables are created via Impala so the metadata should have been populated properly.
    // alltypes is an external table.
    assertEquals(System.getProperty("user.name"), table.getMetaStoreTable().getOwner());
    assertEquals(TableType.EXTERNAL_TABLE.toString(),
        table.getMetaStoreTable().getTableType());
  }

  /**
   * In Hive-3 the HMS translation layer converts non-transactional managed
   * table definitions to external tables. This test makes sure that such tables
   * are seen as EXTERNAL tables when loaded in catalog
   * @throws CatalogException
   */
  @Test
  public void testCreateTableMetadataHive3() throws CatalogException {
    Assume.assumeTrue(TestUtils.getHiveMajorVersion() > 2);
    // alltypesinsert is created using CREATE TABLE LIKE and is a MANAGED table
    Table table = catalog_.getOrLoadTable("functional", "alltypesinsert", "test");
    assertEquals(System.getProperty("user.name"), table.getMetaStoreTable().getOwner());
    assertEquals(TableType.EXTERNAL_TABLE.toString(),
        table.getMetaStoreTable().getTableType());
    // ACID tables should be loaded as MANAGED tables
    table = catalog_.getOrLoadTable("functional", "insert_only_transactional_table",
        "test");
    assertEquals(System.getProperty("user.name"), table.getMetaStoreTable().getOwner());
    assertEquals(TableType.MANAGED_TABLE.toString(),
        table.getMetaStoreTable().getTableType());
  }

  /**
   * In Hive-2 there is no HMS translation which converts non-transactional managed
   * table definitions to external tables. This test makes sure that the such tables
   * are seen as MANAGED tables in catalog
   * @throws CatalogException
   */
  @Test
  public void testCreateTableMetadataHive2() throws CatalogException {
    Assume.assumeTrue(TestUtils.getHiveMajorVersion() <= 2);
    // alltypesinsert is created using CREATE TABLE LIKE and is a MANAGED table
    Table table = catalog_.getOrLoadTable("functional", "alltypesinsert", "test");
    assertEquals(System.getProperty("user.name"), table.getMetaStoreTable().getOwner());
    assertEquals(TableType.MANAGED_TABLE.toString(),
        table.getMetaStoreTable().getTableType());
  }

  @Test
  public void testLoadingUnsupportedTblTypesOnHive2() throws CatalogException {
    // run the test only when it is running against Hive-2 since index tables are
    // skipped during data-load against Hive-3
    Assume.assumeTrue(
        "Skipping this test since it is only supported when running against Hive-2",
        TestUtils.getHiveMajorVersion() == 2);
    Table table = catalog_.getOrLoadTable("functional", "hive_index_tbl", "test");
    assertTrue(table instanceof IncompleteTable);
    IncompleteTable incompleteTable = (IncompleteTable) table;
    assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
    assertEquals("Unsupported table type 'INDEX_TABLE' for: functional.hive_index_tbl",
        incompleteTable.getCause().getMessage());
  }

  @Test
  public void testLoadingUnsupportedTableTypes() throws CatalogException {
    // Table with unsupported SerDe library.
    Table table = catalog_.getOrLoadTable("functional", "bad_serde", "test");
    assertTrue(table instanceof IncompleteTable);
    IncompleteTable incompleteTable = (IncompleteTable) table;
    assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
    assertEquals("Impala does not support tables of this type. REASON: SerDe" +
        " library 'org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe' " +
        "is not supported.", incompleteTable.getCause().getCause().getMessage());

    // Impala does not yet support Hive's LazyBinaryColumnarSerDe which can be
    // used for RCFILE tables.
    table = catalog_.getOrLoadTable("functional_rc", "rcfile_lazy_binary_serde", "test");
    assertTrue(table instanceof IncompleteTable);
    incompleteTable = (IncompleteTable) table;
    assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
    assertEquals("Impala does not support tables of this type. REASON: SerDe" +
        " library 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' " +
        "is not supported.", incompleteTable.getCause().getCause().getMessage());
  }

  private List<String> getFunctionSignatures(String db) throws DatabaseNotFoundException {
    List<Function> fns = catalog_.getFunctions(db);
    List<String> names = new ArrayList<>();
    for (Function fn: fns) {
      names.add(fn.signatureString());
    }
    return names;
  }

  @Test
  public void TestUdf() throws CatalogException {
    List<String> fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 0);

    List<Type> args1 = new ArrayList<>();
    List<Type> args2 = Lists.<Type>newArrayList(Type.INT);
    List<Type> args3 = Lists.<Type>newArrayList(Type.TINYINT);

    catalog_.removeFunction(
        new Function(new FunctionName("default", "Foo"), args1,
            Type.INVALID, false));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 0);

    ScalarFunction udf1 = ScalarFunction.createForTesting(
        "default", "Foo", args1, Type.INVALID, "/Foo", "Foo.class", null,
        null, TFunctionBinaryType.NATIVE);
    catalog_.addFunction(udf1);
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 1);
    assertTrue(fnNames.contains("foo()"));

    // Same function name, overloaded arguments
    ScalarFunction udf2 = ScalarFunction.createForTesting(
        "default", "Foo", args2, Type.INVALID, "/Foo", "Foo.class", null,
        null, TFunctionBinaryType.NATIVE);
    catalog_.addFunction(udf2);
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 2);
    assertTrue(fnNames.contains("foo()"));
    assertTrue(fnNames.contains("foo(INT)"));

    // Add a function with a new name
    ScalarFunction udf3 = ScalarFunction.createForTesting(
        "default", "Bar", args2, Type.INVALID, "/Foo", "Foo.class", null,
        null, TFunctionBinaryType.NATIVE);
    catalog_.addFunction(udf3);
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 3);
    assertTrue(fnNames.contains("foo()"));
    assertTrue(fnNames.contains("foo(INT)"));
    assertTrue(fnNames.contains("bar(INT)"));

    // Drop Foo()
    catalog_.removeFunction(Function.createFunction("default", "Foo", args1,
          Type.INVALID, false, TFunctionBinaryType.NATIVE));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 2);
    assertTrue(fnNames.contains("foo(INT)"));
    assertTrue(fnNames.contains("bar(INT)"));

    // Drop it again, no-op
    catalog_.removeFunction(Function.createFunction("default", "Foo", args1,
          Type.INVALID, false, TFunctionBinaryType.NATIVE));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 2);
    assertTrue(fnNames.contains("foo(INT)"));
    assertTrue(fnNames.contains("bar(INT)"));

    // Drop bar(), no-op
    catalog_.removeFunction(Function.createFunction("default", "Bar", args1,
          Type.INVALID, false, TFunctionBinaryType.NATIVE));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 2);
    assertTrue(fnNames.contains("foo(INT)"));
    assertTrue(fnNames.contains("bar(INT)"));

    // Drop bar(tinyint), no-op
    catalog_.removeFunction(Function.createFunction("default", "Bar", args3,
          Type.INVALID, false, TFunctionBinaryType.NATIVE));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 2);
    assertTrue(fnNames.contains("foo(INT)"));
    assertTrue(fnNames.contains("bar(INT)"));

    // Drop bar(int)
    catalog_.removeFunction(Function.createFunction("default", "Bar", args2,
          Type.INVALID, false, TFunctionBinaryType.NATIVE));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 1);
    assertTrue(fnNames.contains("foo(INT)"));

    // Drop foo(int)
    catalog_.removeFunction(Function.createFunction("default", "Foo", args2,
          Type.INVALID, false, TFunctionBinaryType.NATIVE));
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 0);

    // Test to check if catalog can handle loading corrupt udfs
    Map<String, String> dbParams = new HashMap<>();
    String badFnKey = "impala_registered_function_badFn";
    String badFnVal = Base64.getEncoder().encodeToString("badFn".getBytes());
    String dbName = "corrupt_udf_test";
    dbParams.put(badFnKey, badFnVal);
    Db db = catalog_.getDb(dbName);
    assertEquals(db, null);
    db = new Db(dbName,
        new org.apache.hadoop.hive.metastore.api.Database(dbName, "", "", dbParams));
    catalog_.addDb(db);
    db = catalog_.getDb(dbName);
    assertTrue(db != null);
    fnNames = getFunctionSignatures(dbName);
    assertEquals(fnNames.size(), 0);

    // Test large functions that exceed HMS 4K param limit. We try to add a sample udf
    // with a very long name, exceeding the hms imposed limit and this is expected to
    // fail.
    ScalarFunction largeUdf = ScalarFunction.createForTesting(
        "default", Strings.repeat("Foo", 5000), args2, Type.INVALID, "/Foo",
        "Foo.class", null, null, TFunctionBinaryType.NATIVE);
    assertTrue(catalog_.addFunction(largeUdf) == false);
    fnNames = getFunctionSignatures("default");
    assertEquals(fnNames.size(), 0);
  }

  @Test
  public void testSentryCatalog() throws CatalogException {
    AuthorizationPolicy authPolicy = catalog_.getAuthPolicy();

    User user = catalog_.addUser("user1");
    TPrivilege userPrivilege = new TPrivilege();
    userPrivilege.setPrincipal_type(TPrincipalType.USER);
    userPrivilege.setPrincipal_id(user.getId());
    userPrivilege.setCreate_time_ms(-1);
    userPrivilege.setServer_name("server1");
    userPrivilege.setScope(TPrivilegeScope.SERVER);
    userPrivilege.setPrivilege_level(TPrivilegeLevel.ALL);
    catalog_.addUserPrivilege("user1", userPrivilege);
    assertSame(user, authPolicy.getPrincipal("user1", TPrincipalType.USER));
    assertNull(authPolicy.getPrincipal("user2", TPrincipalType.USER));
    assertNull(authPolicy.getPrincipal("user1", TPrincipalType.ROLE));
    // Add the same user, the old user will be deleted.
    user = catalog_.addUser("user1");
    assertSame(user, authPolicy.getPrincipal("user1", TPrincipalType.USER));
    // Delete the user.
    assertSame(user, catalog_.removeUser("user1"));
    assertNull(authPolicy.getPrincipal("user1", TPrincipalType.USER));

    Role role = catalog_.addRole("role1", Sets.newHashSet("group1", "group2"));
    TPrivilege rolePrivilege = new TPrivilege();
    rolePrivilege.setPrincipal_type(TPrincipalType.ROLE);
    rolePrivilege.setPrincipal_id(role.getId());
    rolePrivilege.setCreate_time_ms(-1);
    rolePrivilege.setServer_name("server1");
    rolePrivilege.setScope(TPrivilegeScope.SERVER);
    rolePrivilege.setPrivilege_level(TPrivilegeLevel.ALL);
    catalog_.addRolePrivilege("role1", rolePrivilege);
    assertSame(role, catalog_.getAuthPolicy().getPrincipal("role1", TPrincipalType.ROLE));
    assertNull(catalog_.getAuthPolicy().getPrincipal("role1", TPrincipalType.USER));
    assertNull(catalog_.getAuthPolicy().getPrincipal("role2", TPrincipalType.ROLE));
    // Add the same role, the old role will be deleted.
    role = catalog_.addRole("role1", new HashSet<>());
    assertSame(role, authPolicy.getPrincipal("role1", TPrincipalType.ROLE));
    // Delete the role.
    assertSame(role, catalog_.removeRole("role1"));
    assertNull(authPolicy.getPrincipal("role1", TPrincipalType.ROLE));

    // Assert that principal IDs will be unique between roles and users, e.g. no user and
    // role with the same principal ID. The same name can be used for both user and role.
    int size = 10;
    String prefix = "foo";
    for (int i = 0; i < size; i++) {
      String name = prefix + i;
      catalog_.addUser(name);
      catalog_.addRole(name, new HashSet<>());
    }

    for (int i = 0; i < size; i++) {
      String name = prefix + i;
      Principal u = authPolicy.getPrincipal(name, TPrincipalType.USER);
      Principal r = authPolicy.getPrincipal(name, TPrincipalType.ROLE);
      assertEquals(name, u.getName());
      assertEquals(name, r.getName());
      assertNotEquals(u.getId(), r.getId());
    }

    // Validate getAllUsers vs getAllUserNames
    List<User> allUsers = authPolicy.getAllUsers();
    Set<String> allUserNames = authPolicy.getAllUserNames();
    assertEquals(allUsers.size(), allUserNames.size());
    for (Principal principal: allUsers) {
      assertTrue(allUserNames.contains(principal.getName()));
    }

    // Validate getAllRoles and getAllRoleNames work as expected.
    List<Role> allRoles = authPolicy.getAllRoles();
    Set<String> allRoleNames = authPolicy.getAllRoleNames();
    assertEquals(allRoles.size(), allRoleNames.size());
    for (Principal principal: allRoles) {
      assertTrue(allRoleNames.contains(principal.getName()));
    }
  }
}