fe/src/test/java/org/apache/impala/testutil/EmbeddedMetastoreClientPool.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.impala.testutil;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.impala.catalog.MetaStoreClientPool;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.log4j.Logger;

import java.nio.file.Path;

/**
 * An implementation of MetastoreClientPool that creates HiveMetastoreClient objects
 * with HMS running in an embedded fashion on the client side. It connects to a Derby
 * database backed by local file system storage.
 *
 * Since local Derby db allows a single connection at any point, there is no use in
 * creating a metastore client pool bigger than that size.
 */
public class EmbeddedMetastoreClientPool extends  MetaStoreClientPool {

  private static final Logger LOG = Logger.getLogger(EmbeddedMetastoreClientPool.class);

  private static final String CONNECTION_URL_TEMPLATE =
      "jdbc:derby:;databaseName=%s;create=true";

  private Path derbyDataStorePath_;

  public EmbeddedMetastoreClientPool(int initialCnxnTimeoutSec, Path dbStorePath) {
    super(1, initialCnxnTimeoutSec, generateEmbeddedHMSConf(dbStorePath));
    derbyDataStorePath_ = dbStorePath;
  }

  // Embedded HMS instantiates partition expression proxy which by default brings in a
  // lot of runtime dependencies from hive-exec. Since we don't depend on this, we
  // should use a DefaultPartitionExpressionProxy which is a no-op implementation of
  // PartitionExpressionProxy interface. It throws UnsupportedOperationException on its
  // APIs so we will find them in tests in case we start using these features when
  // using embedded HMS
  private static final String DEFAULT_PARTITION_EXPRESSION_PROXY_CLASS = "org.apache"
      + ".hadoop.hive.metastore.DefaultPartitionExpressionProxy";
  // In HMS-3 the default value of "metastore.task.thread.always is set to some classes
  // which are present in hive-exec. We don't need this for our tests as of 05/07/2019
  // Setting this to default EventCleanerTask avoid pulling in unnecessary dependencies
  // from hive-exec for running these tests. Note that this config key is not available
  // in HMS-2. But adding this is still okay since it only print a warning of unknown
  // config with no other side-effects
  private static final String METASTORE_TASK_THREAD_ALWAYS_KEY = "metastore.task"
      + ".threads.always";
  private static final String DEFAULT_EVENT_CLEANER_TASK_CLASS = "org.apache.hadoop"
      + ".hive.metastore.events.EventCleanerTask";

  /**
   * Generates the HiveConf required to connect to an embedded metastore backed by
   * derby DB.
   */
  private static HiveConf generateEmbeddedHMSConf(Path dbStorePath) {
    LOG.info("Creating embedded HMS instance at path: " + dbStorePath);
    HiveConf conf = new HiveConf(EmbeddedMetastoreClientPool.class);
    // An embedded HMS with local derby backend requires the following settings
    // hive.metastore.uris - empty
    // javax.jdo.option.ConnectionDriverName - org.apache.derby.jdbc.EmbeddedDriver
    // javax.jdo.option.ConnectionURL - jdbc:derby:;databaseName=<path>;create=true"
    conf.set(ConfVars.METASTOREURIS.varname, "");
    conf.set(ConfVars.METASTORE_CONNECTION_DRIVER.varname,
        "org.apache.derby.jdbc.EmbeddedDriver");
    conf.setBoolean(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, false);
    conf.setBoolean(ConfVars.METASTORE_AUTO_CREATE_ALL.varname, true);
    conf.set(ConfVars.METASTORECONNECTURLKEY.varname,
        String.format(CONNECTION_URL_TEMPLATE, dbStorePath.toString()));
    conf.set(ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname,
        DEFAULT_PARTITION_EXPRESSION_PROXY_CLASS);
    conf.set(METASTORE_TASK_THREAD_ALWAYS_KEY, DEFAULT_EVENT_CLEANER_TASK_CLASS);
    // Disabling notification event listeners
    conf.set(ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS.varname, "");
    return conf;
  }

  @Override
  public void close() {
    super.close();
    // Cleanup the metastore directory.
    FileUtils.deleteQuietly(derbyDataStorePath_.toFile());
  }
}