sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java

package org.apache.sentry.tests.e2e.hdfs;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;

import com.google.common.base.Strings;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.sentry.tests.e2e.hive.fs.DFSFactory.DFSType;
import org.apache.sentry.tests.e2e.hive.fs.TestFSContants;
import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory.HiveServer2Type;
import org.junit.After;
import org.junit.BeforeClass;

import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import static org.junit.Assume.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;

/**
 * A base class for HDFS SynUp tests:
 * The way to run one test could be like the below:
 * mvn test
 -P cluster-hadoop-provider-db \
 -f pom.xml \
 -Dsentry.e2etest.admin.user=hive \
 -Dsentry.e2etest.admin.group=hive \
 -Dhive.server2.thrift.port=10000 \
 -Dhive.server2.authentication.kerberos.keytab=.. \
 -Dhive.server2.authentication.kerberos.principal=.. \
 -Dhive.server2.thrift.bind.host=${HS2_HOST} \
 -Dhive.server2.authentication=kerberos \
 -Dsentry.e2e.hive.keytabs.location=.. \
 -Dsentry.host=${SENTRY_HOST} \
 -Dsentry.service.security.mode=kerberos \
 -Dtest.hdfs.e2e.ext.path=/data
 */

public abstract class TestDbHdfsBase extends AbstractTestWithStaticConfiguration {
  private static final Logger LOGGER = LoggerFactory
      .getLogger(TestDbHdfsBase.class);

  protected static String metastoreDir;
  protected static String scratchLikeDir;
  protected static String authenticationType;
  protected static UserGroupInformation adminUgi;
  protected static UserGroupInformation hiveUgi;
  protected static int NUM_RETRIES_FOR_ACLS = 12;
  protected static int WAIT_SECS_FOR_ACLS = Integer.parseInt(
      System.getProperty(TestFSContants.SENTRY_E2E_TEST_HDFS_ACLS_SYNCUP_SECS, "1000")); // seconds
  protected static String testExtPathDir =
      System.getProperty(TestFSContants.SENTRY_E2E_TEST_HDFS_EXT_PATH);
  protected static final String KEYTAB_LOCATION =
      System.getProperty(TestFSContants.SENTRY_E2E_TEST_HIVE_KEYTAB_LOC, "/cdep/keytabs");
  protected static String DFS_TYPE =
      System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_TYPE, DFSType.MiniDFS.name());

  protected final static String dfsAdmin = System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_ADMIN, "hdfs");
  protected final static String storageUriStr = System.getProperty(TestFSContants.SENTRY_E2E_TEST_STORAGE_URI);

  @BeforeClass
  public static void setupTestStaticConfiguration() throws Exception {
    if (!Strings.isNullOrEmpty(storageUriStr)) {
      LOGGER.warn("Skip HDFS tests if HDFS fileSystem is not configured on hdfs");
      assumeTrue(storageUriStr.toLowerCase().startsWith("hdfs"));
    }
    useSentryService = true;
    enableHDFSAcls = true;
    AbstractTestWithStaticConfiguration.setupTestStaticConfiguration();
    AbstractTestWithStaticConfiguration.setupAdmin();
    scratchLikeDir = context.getProperty(HiveConf.ConfVars.SCRATCHDIR.varname);
    metastoreDir = context.getProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
    authenticationType = System.getProperty(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION.varname);
    assumeNotNull(metastoreDir, scratchLikeDir);
    if (dfsType.equals(DFSType.ClusterDFS.name())) {
      LOGGER.info("Start to run hdfs e2e tests on a real cluster.");
      assumeNotNull(KEYTAB_LOCATION, authenticationType);
      assumeThat(authenticationType, equalToIgnoringCase("kerberos"));
    } else if (dfsType.equals(DFSType.MiniDFS.name())) {
      LOGGER.info("Start to run hdfs e2e tests on a mini cluster.");
      setupMiniCluster();
    } else {
      LOGGER.error("Unknown DFS cluster type: either MiniCluster or ClusterDFS");
      return;
    }
    // Since they are real e2e tests,for now they
    // work on a real cluster managed outside of the tests
    assumeThat(hiveServer2Type, equalTo(HiveServer2Type.UnmanagedHiveServer2));
    assumeThat(dfsType, equalTo(DFSType.ClusterDFS.name()));
  }

  private static void setupMiniCluster() throws Exception {
    createGgis();
  }

  @After
  public void clearAfterPerTest() throws Exception {
    super.clearAfterPerTest();
    // Clean up any extra data created during testing in external path
    LOGGER.info("TestDbHdfsBase clearAfterPerTest");
    kinitFromKeytabFile(dfsAdmin, getKeyTabFileFullPath(dfsAdmin));
    if (!Strings.isNullOrEmpty(testExtPathDir)) {
      Path path = new Path(testExtPathDir);
      FileStatus[] children = fileSystem.listStatus(path);
      for (FileStatus fs : children) {
        LOGGER.info("Deleting " + fs.toString());
        fileSystem.delete(fs.getPath(), true);
      }
    }
  }

  private FileSystem getFS(UserGroupInformation ugi) throws Exception {
    return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
      public FileSystem run() throws Exception {
        Configuration conf = new Configuration();
        return FileSystem.get(conf);
      }
    });
  }

  private static void createGgis() throws Exception {
    if (dfsType.equals(DFSType.MiniDFS.name())) {
      adminUgi = UserGroupInformation.createUserForTesting(
          System.getProperty("user.name"), new String[]{"supergroup"});
      hiveUgi = UserGroupInformation.createUserForTesting(
          "hive", new String[]{"hive"});
    } else if (dfsType.equals(DFSType.ClusterDFS.name())) {
      adminUgi = UserGroupInformation.loginUserFromKeytabAndReturnUGI("hdfs", KEYTAB_LOCATION + "/hdfs.keytab");
      hiveUgi = UserGroupInformation.loginUserFromKeytabAndReturnUGI("hive", KEYTAB_LOCATION + "/hive.keytab");
    }
  }

  protected void verifyAclsRecursive(final List<AclEntry> expectedAcls, final String pathLoc,
                                     final boolean recursive) throws Exception {
    if (DFS_TYPE.equals(DFSType.MiniDFS.name())) {
      fileSystem = getFS(adminUgi);
      adminUgi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          verifyAclsHelper(expectedAcls, pathLoc, recursive);
          return null;
        }
      });
    } else if (DFS_TYPE.equals(DFSType.ClusterDFS.name())) {
      kinitFromKeytabFile(dfsAdmin, getKeyTabFileFullPath(dfsAdmin));
      verifyAclsHelper(expectedAcls, pathLoc, recursive);
    } else {
      fail("Unknown DFS cluster type: " + DFS_TYPE);
    }
  }

  protected void verifyNoAclRecursive(final List<AclEntry> noAcls, final String pathLoc,
                                      final boolean recursive) throws Exception {
    if (DFS_TYPE.equals(DFSType.MiniDFS.name())) {
      fileSystem = getFS(adminUgi);
      adminUgi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          verifyNoAclHelper(noAcls, pathLoc, recursive);
          return null;
        }
      });
    } else if (DFS_TYPE.equals(DFSType.ClusterDFS.name())) {
      kinitFromKeytabFile(dfsAdmin, getKeyTabFileFullPath(dfsAdmin));
      verifyNoAclHelper(noAcls, pathLoc, recursive);
    } else {
      fail("Unknown DFS cluster type: " + DFS_TYPE);
    }
  }

  /**
   * Verify extended acl entries are correctly synced up
   * @param expectedAcls
   * @param pathLoc
   * @param recursive
   * @throws Exception
   */
  private void verifyAclsHelper(List<AclEntry> expectedAcls, String pathLoc,
                                boolean recursive) throws Exception {
    int retry = 0;
    Path path = new Path(pathLoc);
    LOGGER.info("expectedAcls of [" + pathLoc + "] = " + expectedAcls.toString());
    // Syncing up acls takes some time so make validation in a loop
    while (retry < NUM_RETRIES_FOR_ACLS) {
      AclStatus aclStatus = fileSystem.getAclStatus(path);
      List<AclEntry> actualAcls = new ArrayList<>(aclStatus.getEntries());
      LOGGER.info("[" + retry + "] actualAcls of [" + pathLoc + "] = " + actualAcls.toString());
      retry += 1;
      if (!actualAcls.isEmpty() && !actualAcls.contains(expectedAcls.get(expectedAcls.size()-1))) {
        Thread.sleep(WAIT_SECS_FOR_ACLS);
      } else {
          for (AclEntry expected : expectedAcls) {
            assertTrue("Fail to find aclEntry: " + expected.toString(),
                actualAcls.contains(expected));
          }
          break;
      }
    }
    assertThat(retry, lessThan(NUM_RETRIES_FOR_ACLS));
    if (recursive && fileSystem.getFileStatus(path).isDirectory()) {
      FileStatus[] children = fileSystem.listStatus(path);
      for (FileStatus fs : children) {
        verifyAclsRecursive(expectedAcls, fs.getPath().toString(), recursive);
      }
    }
  }

  /**
   * Verify there is no specified acls gotten synced up in the path status
   * @param noAcls
   * @param pathLoc
   * @param recursive
   * @throws Exception
   */
  private void verifyNoAclHelper(List<AclEntry> noAcls, String pathLoc,
                                 boolean recursive) throws Exception {
    int retry = 0;
    // Retry a couple of times in case the incorrect acls take time to be synced up
    while (retry < NUM_RETRIES_FOR_ACLS) {
      Path path = new Path(pathLoc);
      AclStatus aclStatus = fileSystem.getAclStatus(path);
      List<AclEntry> actualAcls = new ArrayList<>(aclStatus.getEntries());
      LOGGER.info("[" + retry + "] actualAcls of [" + pathLoc + "] = " + actualAcls.toString());
      Thread.sleep(1000); // wait for syncup
      retry += 1;
      for (AclEntry acl : actualAcls) {
        if (noAcls.contains(acl)) {
          fail("Path [ " + pathLoc + " ] should not contain " + acl.toString());
        }
      }
    }
    Path path = new Path(pathLoc);
    if (recursive && fileSystem.getFileStatus(path).isDirectory()) {
      FileStatus[] children = fileSystem.listStatus(path);
      for (FileStatus fs : children) {
        verifyNoAclRecursive(noAcls, fs.getPath().toString(), recursive);
      }
    }
  }

  /**
   * Drop and create role, in case the previous
   * tests leave same roles uncleaned up
   * @param statement
   * @param roleName
   * @throws Exception
   */
  protected void dropRecreateRole(Statement statement, String roleName) throws Exception {
    try {
      exec(statement, "DROP ROLE " + roleName);
    } catch (Exception ex) {
      //noop
      LOGGER.info("Role " + roleName  + " does not exist. But it's ok.");
    } finally {
      exec(statement, "CREATE ROLE " + roleName);
    }
  }

  /**
   * Create an internal test database and table
   * @param db
   * @param tbl
   * @throws Exception
   */
  protected void dropRecreateDbTblRl(String db, String tbl) throws Exception {
    dropRecreateDbTblRl(null, db, tbl);
  }

  /**
   * Create test database and table with location pointing to testPathLoc
   * @param testPathLoc
   * @param db
   * @param tbl
   * @throws Exception
   */
  protected void dropRecreateDbTblRl(String testPathLoc, String db, String tbl) throws Exception {
    Connection connection = context.createConnection(ADMIN1);
    Statement statement = connection.createStatement();
    exec(statement, "DROP DATABASE IF EXISTS " + db  + " CASCADE");
    if (testPathLoc != null ) {
      exec(statement, "CREATE DATABASE " + db + " LOCATION \'" + testPathLoc + "\'");
    } else {
      exec(statement, "CREATE DATABASE " + db);
    }
    exec(statement, "USE " + db);
    exec(statement, "CREATE TABLE " + tbl + "(number INT, value STRING) PARTITIONED BY (par INT)");
    exec(statement, "INSERT INTO TABLE " + tbl + " PARTITION(par=1) VALUES (1, 'test1')");
    exec(statement, "SELECT * FROM " + tbl);
    if (statement != null) {
      statement.close();
    }
    if (connection != null ) {
      connection.close();
    }
  }

  /**
   * Create test database and table with location pointing
   * to testPathLoc without partitions
   * @param db
   * @param tbl
   * @throws Exception
   */
  protected void dropRecreateDbTblNoPar(String db, String tbl) throws Exception {
    Connection connection = context.createConnection(ADMIN1);
    Statement statement = connection.createStatement();
    exec(statement, "DROP DATABASE IF EXISTS " + db  + " CASCADE");
    exec(statement, "CREATE DATABASE " + db);
    exec(statement, "USE " + db);
    exec(statement, "CREATE TABLE " + tbl + "(number INT, value STRING)");
    exec(statement, "INSERT INTO TABLE " + tbl + " VALUES (1, 'test1')");
    exec(statement, "SELECT * FROM " + tbl);
    if (statement != null) {
      statement.close();
    }
    if (connection != null ) {
      connection.close();
    }
  }

  protected static void kinitFromKeytabFile (String user, String keyTabFile) throws IOException {
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", authenticationType);
    UserGroupInformation.setConfiguration(conf);
    UserGroupInformation.loginUserFromKeytab(user, keyTabFile);
  }

  protected static String getKeyTabFileFullPath(String user) {
    return KEYTAB_LOCATION + "/" + user + ".keytab";
  }
}