exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.drill.exec.impersonation;

import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.base.Strings;
import org.apache.commons.io.FileUtils;
import org.apache.drill.PlanTestBase;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.dotdrill.DotDrillType;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.dfs.FileSystemConfig;
import org.apache.drill.exec.store.dfs.WorkspaceConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.security.UserGroupInformation;

import java.io.File;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Properties;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

public class BaseTestImpersonation extends PlanTestBase {
  protected static final String MINI_DFS_STORAGE_PLUGIN_NAME = "mini_dfs_plugin";
  protected static final String processUser = System.getProperty("user.name");

  protected static MiniDFSCluster dfsCluster;
  protected static Configuration dfsConf;
  protected static FileSystem fs;
  protected static File miniDfsStoragePath;

  // Test users and groups
  protected static final String[] org1Users = { "user0_1", "user1_1", "user2_1", "user3_1", "user4_1", "user5_1" };
  protected static final String[] org1Groups = { "group0_1", "group1_1", "group2_1", "group3_1", "group4_1", "group5_1" };
  protected static final String[] org2Users = { "user0_2", "user1_2", "user2_2", "user3_2", "user4_2", "user5_2" };
  protected static final String[] org2Groups = { "group0_2", "group1_2", "group2_2", "group3_2", "group4_2", "group5_2" };

  static {
    // "user0_1" belongs to "groups0_1". From "user1_1" onwards each user belongs to corresponding group and the group
    // before it, i.e "user1_1" belongs to "group1_1" and "group0_1" and so on.
    UserGroupInformation.createUserForTesting(org1Users[0], new String[]{org1Groups[0]});
    for(int i=1; i<org1Users.length; i++) {
      UserGroupInformation.createUserForTesting(org1Users[i], new String[] { org1Groups[i], org1Groups[i-1] });
    }

    UserGroupInformation.createUserForTesting(org2Users[0], new String[] { org2Groups[0] });
    for(int i=1; i<org2Users.length; i++) {
      UserGroupInformation.createUserForTesting(org2Users[i], new String[] { org2Groups[i], org2Groups[i-1] });
    }
  }

  /**
   * Start a MiniDFS cluster backed Drillbit cluster with impersonation enabled.
   * @param testClass test class
   * @throws Exception in case of errors during start up
   */
  protected static void startMiniDfsCluster(String testClass) throws Exception {
    startMiniDfsCluster(testClass, true);
  }

  /**
   * Start a MiniDFS cluster backed Drillbit cluster
   * @param testClass test class
   * @param isImpersonationEnabled Enable impersonation in the cluster?
   * @throws Exception in case of errors during start up
   */
  protected static void startMiniDfsCluster(String testClass, boolean isImpersonationEnabled) throws Exception {
    Preconditions.checkArgument(!Strings.isNullOrEmpty(testClass), "Expected a non-null and non-empty test class name");
    dfsConf = new Configuration();

    // Set the MiniDfs base dir to be the temp directory of the test, so that all files created within the MiniDfs
    // are properly cleanup when test exits.
    miniDfsStoragePath = dirTestWatcher.makeRootSubDir(Paths.get("miniDfs"));
    dfsConf.set("hdfs.minidfs.basedir", miniDfsStoragePath.getCanonicalPath());

    if (isImpersonationEnabled) {
      // Set the proxyuser settings so that the user who is running the Drillbits/MiniDfs can impersonate other users.
      dfsConf.set(String.format("hadoop.proxyuser.%s.hosts", processUser), "*");
      dfsConf.set(String.format("hadoop.proxyuser.%s.groups", processUser), "*");
    }

    // Start the MiniDfs cluster
    dfsCluster = new MiniDFSCluster.Builder(dfsConf)
        .numDataNodes(3)
        .format(true)
        .build();

    fs = dfsCluster.getFileSystem();
  }

  protected static void startDrillCluster(final boolean isImpersonationEnabled) throws Exception {
    final Properties props = cloneDefaultTestConfigProperties();
    props.setProperty(ExecConstants.IMPERSONATION_ENABLED, Boolean.toString(isImpersonationEnabled));
    startDrillCluster(props);
  }

  protected static void startDrillCluster(final Properties props) throws Exception {
    updateTestCluster(1, DrillConfig.create(props));
  }

  protected static void addMiniDfsBasedStorage(final Map<String, WorkspaceConfig> workspaces) throws Exception {
    // Create a HDFS based storage plugin based on local storage plugin and add it to plugin registry (connection string
    // for mini dfs is varies for each run).
    StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
    FileSystemConfig lfsPluginConfig = (FileSystemConfig) pluginRegistry.getPlugin("dfs").getConfig();

    String connection = dfsConf.get(FileSystem.FS_DEFAULT_NAME_KEY);
    createAndAddWorkspace("tmp", "/tmp", (short) 0777, processUser, processUser, workspaces);

    FileSystemConfig miniDfsPluginConfig = new FileSystemConfig(connection, null,
        workspaces, lfsPluginConfig.getFormats(), null);
    miniDfsPluginConfig.setEnabled(true);
    pluginRegistry.put(MINI_DFS_STORAGE_PLUGIN_NAME, miniDfsPluginConfig);
  }

  protected static void createAndAddWorkspace(String name, String path, short permissions, String owner,
      String group, final Map<String, WorkspaceConfig> workspaces) throws Exception {

    FsPermission permission = new FsPermission(permissions);

    Path dirPath = new Path(path);
    assertTrue(FileSystem.mkdirs(fs, dirPath, permission));
    fs.setOwner(dirPath, owner, group);

    // create sample file in the workspace to check show files command
    Path sampleFile = new Path(dirPath, String.format("sample_%s.txt", name));
    assertTrue(fs.createNewFile(sampleFile));
    fs.setPermission(sampleFile, permission);
    fs.setOwner(sampleFile, owner, group);

    final WorkspaceConfig ws = new WorkspaceConfig(path, true, "parquet", false);
    workspaces.put(name, ws);
  }

  protected static void stopMiniDfsCluster() {
    if (dfsCluster != null) {
      dfsCluster.shutdown();
      dfsCluster = null;
    }

    if (miniDfsStoragePath != null) {
      FileUtils.deleteQuietly(miniDfsStoragePath);
    }
  }

  // Return the user workspace for given user.
  protected static String getWSSchema(String user) {
    return MINI_DFS_STORAGE_PLUGIN_NAME + "." + user;
  }

  protected static String getUserHome(String user) {
    return "/user/" + user;
  }

  protected static void createView(final String viewOwner, final String viewGroup, final short viewPerms,
                                 final String newViewName, final String fromSourceSchema, final String fromSourceTableName) throws Exception {
    updateClient(viewOwner);
    test(String.format("ALTER SESSION SET `%s`='%o';", ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY, viewPerms));
    test(String.format("CREATE VIEW %s.%s AS SELECT * FROM %s.%s;",
      getWSSchema(viewOwner), newViewName, fromSourceSchema, fromSourceTableName));

    // Verify the view file created has the expected permissions and ownership
    Path viewFilePath = new Path(getUserHome(viewOwner), newViewName + DotDrillType.VIEW.getEnding());
    FileStatus status = fs.getFileStatus(viewFilePath);
    assertEquals(viewGroup, status.getGroup());
    assertEquals(viewOwner, status.getOwner());
    assertEquals(viewPerms, status.getPermission().toShort());
  }

  protected static void createView(final String viewOwner, final String viewGroup, final String viewName,
                                 final String viewDef) throws Exception {
    updateClient(viewOwner);
    test(String.format("ALTER SESSION SET `%s`='%o';", ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY, (short) 0750));
    test("CREATE VIEW %s.%s.%s AS %s", MINI_DFS_STORAGE_PLUGIN_NAME, "tmp", viewName, viewDef);
    final Path viewFilePath = new Path("/tmp/", viewName + DotDrillType.VIEW.getEnding());
    fs.setOwner(viewFilePath, viewOwner, viewGroup);
  }
}