Google Git
Sign in
apache / hive / 6b095edc31a9baa0ae805e2b5aea88101284d84b / . / standalone-metastore / metastore-server / src / main / java / org / apache / hadoop / hive / metastore / HMSHandler.java
blob: ca7d8681f997f7e9e071723ba17e29c14986b1e4 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.metastore;
import com.codahale.metrics.Counter;
import com.facebook.fb303.FacebookBase;
import com.facebook.fb303.fb_status;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Striped;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.AcidConstants;
import org.apache.hadoop.hive.common.AcidMetaDataFile;
import org.apache.hadoop.hive.common.AcidMetaDataFile.DataFormat;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.repl.ReplConst;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.api.Package;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.dataconnector.DataConnectorProviderFactory;
import org.apache.hadoop.hive.metastore.events.*;
import org.apache.hadoop.hive.metastore.leader.HouseKeepingTasks;
import org.apache.hadoop.hive.metastore.leader.LeaderElectionContext;
import org.apache.hadoop.hive.metastore.messaging.EventMessage;
import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
import org.apache.hadoop.hive.metastore.metrics.Metrics;
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.metrics.PerfLogger;
import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.properties.PropertyException;
import org.apache.hadoop.hive.metastore.properties.PropertyManager;
import org.apache.hadoop.hive.metastore.properties.PropertyMap;
import org.apache.hadoop.hive.metastore.properties.PropertyStore;
import org.apache.hadoop.hive.metastore.txn.*;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.FilterUtils;
import org.apache.hadoop.hive.metastore.utils.HdfsUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jdo.JDOException;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.regex.Pattern;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.join;
import static org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_PATH_SUFFIX;
import static org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_TABLE_PATTERN;
import static org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_TABLE;
import static org.apache.hadoop.hive.common.AcidConstants.DELTA_DIGITS;
import static org.apache.hadoop.hive.common.repl.ReplConst.REPL_RESUME_STARTED_AFTER_FAILOVER;
import static org.apache.hadoop.hive.common.repl.ReplConst.REPL_TARGET_DATABASE_PROPERTY;
import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.RENAME_PARTITION_MAKE_COPY;
import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.TRUNCATE_SKIP_DATA_DELETION;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
import static org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
import static org.apache.hadoop.hive.metastore.ExceptionHandler.newMetaException;
import static org.apache.hadoop.hive.metastore.ExceptionHandler.rethrowException;
import static org.apache.hadoop.hive.metastore.ExceptionHandler.throwMetaException;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedTableName;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTLT;
import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName;
import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
/**
* Default handler for all Hive Metastore methods. Implements methods defined in hive_metastore.thrift.
*/
public class HMSHandler extends FacebookBase implements IHMSHandler {
public static final Logger LOG = LoggerFactory.getLogger(HMSHandler.class);
private final Configuration conf; // stores datastore (jpox) properties,
// right now they come from jpox.properties
// Flag to control that always threads are initialized only once
// instead of multiple times
private final static AtomicBoolean alwaysThreadsInitialized =
new AtomicBoolean(false);
private static String currentUrl;
private FileMetadataManager fileMetadataManager;
private PartitionExpressionProxy expressionProxy;
private StorageSchemaReader storageSchemaReader;
private IMetaStoreMetadataTransformer transformer;
private static DataConnectorProviderFactory dataconnectorFactory = null;
public static final String PARTITION_NUMBER_EXCEED_LIMIT_MSG =
"Number of partitions scanned (=%d) on table '%s' exceeds limit (=%d). This is controlled on the metastore server by %s.";
// Used for testing to simulate method timeout.
@VisibleForTesting
static boolean testTimeoutEnabled = false;
@VisibleForTesting
static long testTimeoutValue = -1;
public static final String ADMIN = "admin";
public static final String PUBLIC = "public";
static final String NO_FILTER_STRING = "";
static final int UNLIMITED_MAX_PARTITIONS = -1;
private Warehouse wh; // hdfs warehouse
private static Striped<Lock> tablelocks;
public static RawStore getRawStore() {
return HMSHandlerContext.getRawStore().orElse(null);
}
static void cleanupHandlerContext() {
AtomicBoolean cleanedRawStore = new AtomicBoolean(false);
try {
HMSHandlerContext.getRawStore().ifPresent(rs -> {
logAndAudit("Cleaning up thread local RawStore...");
rs.shutdown();
cleanedRawStore.set(true);
});
} finally {
HMSHandlerContext.getHMSHandler().ifPresent(HMSHandler::notifyMetaListenersOnShutDown);
if (cleanedRawStore.get()) {
logAndAudit("Done cleaning up thread local RawStore");
}
HMSHandlerContext.clear();
}
}
private static ExecutorService threadPool;
static final Logger auditLog = LoggerFactory.getLogger(
HiveMetaStore.class.getName() + ".audit");
private static void logAuditEvent(String cmd) {
if (cmd == null) {
return;
}
UserGroupInformation ugi;
try {
ugi = SecurityUtils.getUGI();
} catch (Exception ex) {
throw new RuntimeException(ex);
}
String address = getIPAddress();
if (address == null) {
address = "unknown-ip-addr";
}
auditLog.info("ugi={} ip={} cmd={} ", ugi.getUserName(), address, cmd);
}
public static String getIPAddress() {
if (HiveMetaStore.useSasl) {
if (HiveMetaStore.saslServer != null && HiveMetaStore.saslServer.getRemoteAddress() != null) {
return HiveMetaStore.saslServer.getRemoteAddress().getHostAddress();
}
} else {
// if kerberos is not enabled
return getThreadLocalIpAddress();
}
return null;
}
/**
* Internal function to notify listeners for meta config change events
*/
private void notifyMetaListeners(String key, String oldValue, String newValue) throws MetaException {
for (MetaStoreEventListener listener : listeners) {
listener.onConfigChange(new ConfigChangeEvent(this, key, oldValue, newValue));
}
if (transactionalListeners.size() > 0) {
// All the fields of this event are final, so no reason to create a new one for each
// listener
ConfigChangeEvent cce = new ConfigChangeEvent(this, key, oldValue, newValue);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onConfigChange(cce);
}
}
}
/**
* Internal function to notify listeners to revert back to old values of keys
* that were modified during setMetaConf. This would get called from HiveMetaStore#cleanupHandlerContext
*/
private void notifyMetaListenersOnShutDown() {
try {
Map<String, String> modifiedConf = HMSHandlerContext.getModifiedConfig();
Configuration conf = HMSHandlerContext.getConfiguration()
.orElseThrow(() -> new MetaException("Unexpected: modifiedConf is non-null but conf is null"));
// Notify listeners of the changed value
for (Map.Entry<String, String> entry : modifiedConf.entrySet()) {
String key = entry.getKey();
// curr value becomes old and vice-versa
String currVal = entry.getValue();
String oldVal = conf.get(key);
if (!Objects.equals(oldVal, currVal)) {
notifyMetaListeners(key, oldVal, currVal);
}
}
logAndAudit("Meta listeners shutdown notification completed.");
} catch (MetaException e) {
LOG.error("Failed to notify meta listeners on shutdown: ", e);
}
}
static void setThreadLocalIpAddress(String ipAddress) {
HMSHandlerContext.setIpAddress(ipAddress);
}
// This will return null if the metastore is not being accessed from a metastore Thrift server,
// or if the TTransport being used to connect is not an instance of TSocket, or if kereberos
// is used
static String getThreadLocalIpAddress() {
return HMSHandlerContext.getIpAddress().orElse(null);
}
// Make it possible for tests to check that the right type of PartitionExpressionProxy was
// instantiated.
@VisibleForTesting
PartitionExpressionProxy getExpressionProxy() {
return expressionProxy;
}
public HMSHandler(String name, Configuration conf) {
super(name);
this.conf = conf;
isInTest = MetastoreConf.getBoolVar(this.conf, HIVE_IN_TEST);
if (threadPool == null) {
synchronized (HMSHandler.class) {
if (threadPool == null) {
int numThreads = MetastoreConf.getIntVar(conf, ConfVars.FS_HANDLER_THREADS_COUNT);
threadPool = Executors.newFixedThreadPool(numThreads,
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("HMSHandler #%d").build());
int numTableLocks = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_NUM_STRIPED_TABLE_LOCKS);
tablelocks = Striped.lock(numTableLocks);
}
}
}
}
/**
* Use {@link #getConf()} instead.
* @return Configuration object
*/
@Deprecated
public Configuration getHiveConf() {
return conf;
}
private AlterHandler alterHandler;
private List<MetaStorePreEventListener> preListeners;
private List<MetaStoreEventListener> listeners;
private List<TransactionalMetaStoreEventListener> transactionalListeners;
private List<MetaStoreEndFunctionListener> endFunctionListeners;
private List<MetaStoreInitListener> initListeners;
private MetaStoreFilterHook filterHook;
private boolean isServerFilterEnabled = false;
private Pattern partitionValidationPattern;
private final boolean isInTest;
@Override
public List<TransactionalMetaStoreEventListener> getTransactionalListeners() {
return transactionalListeners;
}
@Override
public List<MetaStoreEventListener> getListeners() {
return listeners;
}
@Override
public void init() throws MetaException {
Metrics.initialize(conf);
initListeners = MetaStoreServerUtils.getMetaStoreListeners(
MetaStoreInitListener.class, conf, MetastoreConf.getVar(conf, ConfVars.INIT_HOOKS));
for (MetaStoreInitListener singleInitListener: initListeners) {
MetaStoreInitContext context = new MetaStoreInitContext();
singleInitListener.onInit(context);
}
String alterHandlerName = MetastoreConf.getVar(conf, ConfVars.ALTER_HANDLER);
alterHandler = ReflectionUtils.newInstance(JavaUtils.getClass(
alterHandlerName, AlterHandler.class), conf);
wh = new Warehouse(conf);
synchronized (HMSHandler.class) {
if (currentUrl == null || !currentUrl.equals(MetaStoreInit.getConnectionURL(conf))) {
createDefaultDB();
createDefaultRoles();
addAdminUsers();
currentUrl = MetaStoreInit.getConnectionURL(conf);
updateMetrics();
}
}
preListeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
conf, MetastoreConf.getVar(conf, ConfVars.PRE_EVENT_LISTENERS));
preListeners.add(0, new TransactionalValidationListener(conf));
listeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf,
MetastoreConf.getVar(conf, ConfVars.EVENT_LISTENERS));
listeners.add(new SessionPropertiesListener(conf));
transactionalListeners = new ArrayList() {{
add(new AcidEventListener(conf));
}};
transactionalListeners.addAll(MetaStoreServerUtils.getMetaStoreListeners(
TransactionalMetaStoreEventListener.class, conf,
MetastoreConf.getVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS)));
if (Metrics.getRegistry() != null) {
listeners.add(new HMSMetricsListener(conf));
}
boolean canCachedStoreCanUseEvent = false;
for (MetaStoreEventListener listener : transactionalListeners) {
if (listener.doesAddEventsToNotificationLogTable()) {
canCachedStoreCanUseEvent = true;
break;
}
}
if (conf.getBoolean(ConfVars.METASTORE_CACHE_CAN_USE_EVENT.getVarname(), false) &&
!canCachedStoreCanUseEvent) {
throw new MetaException("CahcedStore can not use events for invalidation as there is no " +
" TransactionalMetaStoreEventListener to add events to notification table");
}
endFunctionListeners = MetaStoreServerUtils.getMetaStoreListeners(
MetaStoreEndFunctionListener.class, conf, MetastoreConf.getVar(conf, ConfVars.END_FUNCTION_LISTENERS));
String partitionValidationRegex =
MetastoreConf.getVar(conf, ConfVars.PARTITION_NAME_WHITELIST_PATTERN);
if (partitionValidationRegex != null && !partitionValidationRegex.isEmpty()) {
partitionValidationPattern = Pattern.compile(partitionValidationRegex);
}
// We only initialize once the tasks that need to be run periodically. For remote metastore
// these threads are started along with the other housekeeping threads only in the leader
// HMS.
if (!HiveMetaStore.isMetaStoreRemote()) {
startAlwaysTaskThreads(conf, this);
}
expressionProxy = PartFilterExprUtil.createExpressionProxy(conf);
fileMetadataManager = new FileMetadataManager(this.getMS(), conf);
isServerFilterEnabled = getIfServerFilterenabled();
filterHook = isServerFilterEnabled ? loadFilterHooks() : null;
String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS);
if (className != null && !className.trim().isEmpty()) {
try {
transformer = JavaUtils.newInstance(JavaUtils.getClass(className.trim(), IMetaStoreMetadataTransformer.class),
new Class[] {IHMSHandler.class}, new Object[] {this});
} catch (Exception e) {
LOG.error("Unable to create instance of class " + className, e);
throw new IllegalArgumentException(e);
}
}
dataconnectorFactory = DataConnectorProviderFactory.getInstance(this);
}
static void startAlwaysTaskThreads(Configuration conf, IHMSHandler handler) throws MetaException {
if (alwaysThreadsInitialized.compareAndSet(false, true)) {
try {
LeaderElectionContext context = new LeaderElectionContext.ContextBuilder(conf)
.setTType(LeaderElectionContext.TTYPE.ALWAYS_TASKS)
.addListener(new HouseKeepingTasks(conf, false))
.setHMSHandler(handler).build();
context.start();
} catch (Exception e) {
throw newMetaException(e);
}
}
}
/**
*
* Filter is actually enabled only when the configured filter hook is configured, not default, and
* enabled in configuration
* @return
*/
private boolean getIfServerFilterenabled() throws MetaException{
boolean isEnabled = MetastoreConf.getBoolVar(conf, ConfVars.METASTORE_SERVER_FILTER_ENABLED);
if (!isEnabled) {
LOG.info("HMS server filtering is disabled by configuration");
return false;
}
String filterHookClassName = MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK);
if (isBlank(filterHookClassName)) {
throw new MetaException("HMS server filtering is enabled but no filter hook is configured");
}
if (filterHookClassName.trim().equalsIgnoreCase(DefaultMetaStoreFilterHookImpl.class.getName())) {
throw new MetaException("HMS server filtering is enabled but the filter hook is DefaultMetaStoreFilterHookImpl, which does no filtering");
}
LOG.info("HMS server filtering is enabled. The filter class is " + filterHookClassName);
return true;
}
private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException {
String errorMsg = "Unable to load filter hook at HMS server. ";
String filterHookClassName = MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK);
Preconditions.checkState(!isBlank(filterHookClassName));
try {
return (MetaStoreFilterHook)Class.forName(
filterHookClassName.trim(), true, JavaUtils.getClassLoader()).getConstructor(
Configuration.class).newInstance(conf);
} catch (Exception e) {
LOG.error(errorMsg, e);
throw new IllegalStateException(errorMsg + e.getMessage(), e);
}
}
/**
* Check if user can access the table associated with the partition. If not, then throw exception
* so user cannot access partitions associated with this table
* We are not calling Pre event listener for authorization because it requires getting the
* table object from DB, more overhead. Instead ,we call filter hook to filter out table if user
* has no access. Filter hook only requires table name, not table object. That saves DB access for
* table object, and still achieve the same purpose: checking if user can access the specified
* table
*
* @param catName catalog name of the table
* @param dbName database name of the table
* @param tblName table name
* @throws NoSuchObjectException
* @throws MetaException
*/
private void authorizeTableForPartitionMetadata(
final String catName, final String dbName, final String tblName)
throws NoSuchObjectException, MetaException {
FilterUtils.checkDbAndTableFilters(
isServerFilterEnabled, filterHook, catName, dbName, tblName);
}
@Override
public void setConf(Configuration conf) {
HMSHandlerContext.setConfiguration(conf);
// reload if DS related configuration is changed
HMSHandlerContext.getRawStore().ifPresent(ms -> ms.setConf(conf));
}
@Override
public Configuration getConf() {
Configuration conf = HMSHandlerContext.getConfiguration()
.orElseGet(() -> {
Configuration configuration = new Configuration(this.conf);
HMSHandlerContext.setConfiguration(configuration);
return configuration;
});
return conf;
}
@Override
public Warehouse getWh() {
return wh;
}
@Override
public void setMetaConf(String key, String value) throws MetaException {
ConfVars confVar = MetastoreConf.getMetaConf(key);
if (confVar == null) {
throw new MetaException("Invalid configuration key " + key);
}
try {
confVar.validate(value);
} catch (IllegalArgumentException e) {
throw new MetaException("Invalid configuration value " + value + " for key " + key +
" by " + e.getMessage());
}
Configuration configuration = getConf();
String oldValue = MetastoreConf.get(configuration, key);
// Save prev val of the key on threadLocal
Map<String, String> modifiedConf = HMSHandlerContext.getModifiedConfig();
if (!modifiedConf.containsKey(key)) {
modifiedConf.put(key, oldValue);
}
// Set invoking HMSHandler on threadLocal, this will be used later to notify
// metaListeners in HiveMetaStore#cleanupHandlerContext
if (!HMSHandlerContext.getHMSHandler().isPresent()) {
HMSHandlerContext.setHMSHandler(this);
}
configuration.set(key, value);
notifyMetaListeners(key, oldValue, value);
if (ConfVars.TRY_DIRECT_SQL == confVar) {
HMSHandler.LOG.info("Direct SQL optimization = {}", value);
}
}
@Override
public String getMetaConf(String key) throws MetaException {
ConfVars confVar = MetastoreConf.getMetaConf(key);
if (confVar == null) {
throw new MetaException("Invalid configuration key " + key);
}
return getConf().get(key, confVar.getDefaultVal().toString());
}
/**
* Get a cached RawStore.
*
* @return the cached RawStore
* @throws MetaException
*/
@Override
public RawStore getMS() throws MetaException {
Configuration conf = getConf();
return getMSForConf(conf);
}
public static synchronized RawStore getMSForConf(Configuration conf) throws MetaException {
RawStore ms = getRawStore();
if (ms == null) {
ms = newRawStoreForConf(conf);
try {
ms.verifySchema();
} catch (MetaException e) {
ms.shutdown();
throw e;
}
HMSHandlerContext.setRawStore(ms);
LOG.info("Created RawStore: {}", ms);
}
return ms;
}
@Override
public TxnStore getTxnHandler() {
return HMSHandlerContext.getTxnStore(conf);
}
static RawStore newRawStoreForConf(Configuration conf) throws MetaException {
Configuration newConf = new Configuration(conf);
String rawStoreClassName = MetastoreConf.getVar(newConf, ConfVars.RAW_STORE_IMPL);
LOG.info("Opening raw store with implementation class: {}", rawStoreClassName);
return RawStoreProxy.getProxy(newConf, conf, rawStoreClassName);
}
@VisibleForTesting
public static void createDefaultCatalog(RawStore ms, Warehouse wh) throws MetaException,
InvalidOperationException {
try {
Catalog defaultCat = ms.getCatalog(DEFAULT_CATALOG_NAME);
// Null check because in some test cases we get a null from ms.getCatalog.
if (defaultCat !=null && defaultCat.getLocationUri().equals("TBD")) {
// One time update issue. When the new 'hive' catalog is created in an upgrade the
// script does not know the location of the warehouse. So we need to update it.
LOG.info("Setting location of default catalog, as it hasn't been done after upgrade");
defaultCat.setLocationUri(wh.getWhRoot().toString());
ms.alterCatalog(defaultCat.getName(), defaultCat);
}
} catch (NoSuchObjectException e) {
Catalog cat = new Catalog(DEFAULT_CATALOG_NAME, wh.getWhRoot().toString());
long time = System.currentTimeMillis() / 1000;
cat.setCreateTime((int) time);
cat.setDescription(Warehouse.DEFAULT_CATALOG_COMMENT);
ms.createCatalog(cat);
}
}
private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException {
try {
ms.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME);
} catch (NoSuchObjectException e) {
LOG.info("Started creating a default database with name: "+DEFAULT_DATABASE_NAME);
Database db = new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT,
wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME, true).toString(), null);
db.setOwnerName(PUBLIC);
db.setOwnerType(PrincipalType.ROLE);
db.setCatalogName(DEFAULT_CATALOG_NAME);
long time = System.currentTimeMillis() / 1000;
db.setCreateTime((int) time);
db.setType(DatabaseType.NATIVE);
ms.createDatabase(db);
LOG.info("Successfully created a default database with name: "+DEFAULT_DATABASE_NAME);
}
}
/**
* create default database if it doesn't exist.
*
* This is a potential contention when HiveServer2 using embedded metastore and Metastore
* Server try to concurrently invoke createDefaultDB. If one failed, JDOException was caught
* for one more time try, if failed again, simply ignored by warning, which meant another
* succeeds.
*
* @throws MetaException
*/
private void createDefaultDB() throws MetaException {
try {
RawStore ms = getMS();
createDefaultCatalog(ms, wh);
createDefaultDB_core(ms);
} catch (JDOException e) {
LOG.warn("Retrying creating default database after error: " + e.getMessage(), e);
try {
RawStore ms = getMS();
createDefaultCatalog(ms, wh);
createDefaultDB_core(ms);
} catch (InvalidObjectException | InvalidOperationException e1) {
throw new MetaException(e1.getMessage());
}
} catch (InvalidObjectException|InvalidOperationException e) {
throw new MetaException(e.getMessage());
}
}
/**
* create default roles if they don't exist.
*
* This is a potential contention when HiveServer2 using embedded metastore and Metastore
* Server try to concurrently invoke createDefaultRoles. If one failed, JDOException was caught
* for one more time try, if failed again, simply ignored by warning, which meant another
* succeeds.
*
* @throws MetaException
*/
private void createDefaultRoles() throws MetaException {
try {
createDefaultRoles_core();
} catch (JDOException e) {
LOG.warn("Retrying creating default roles after error: " + e.getMessage(), e);
createDefaultRoles_core();
}
}
private void createDefaultRoles_core() throws MetaException {
RawStore ms = getMS();
try {
ms.addRole(ADMIN, ADMIN);
} catch (InvalidObjectException e) {
LOG.debug(ADMIN +" role already exists",e);
} catch (NoSuchObjectException e) {
// This should never be thrown.
LOG.warn("Unexpected exception while adding " +ADMIN+" roles" , e);
}
LOG.info("Added "+ ADMIN+ " role in metastore");
try {
ms.addRole(PUBLIC, PUBLIC);
} catch (InvalidObjectException e) {
LOG.debug(PUBLIC + " role already exists",e);
} catch (NoSuchObjectException e) {
// This should never be thrown.
LOG.warn("Unexpected exception while adding "+PUBLIC +" roles" , e);
}
LOG.info("Added "+PUBLIC+ " role in metastore");
// now grant all privs to admin
PrivilegeBag privs = new PrivilegeBag();
privs.addToPrivileges(new HiveObjectPrivilege( new HiveObjectRef(HiveObjectType.GLOBAL, null,
null, null, null), ADMIN, PrincipalType.ROLE, new PrivilegeGrantInfo("All", 0, ADMIN,
PrincipalType.ROLE, true), "SQL"));
try {
ms.grantPrivileges(privs);
} catch (InvalidObjectException e) {
// Surprisingly these privs are already granted.
LOG.debug("Failed while granting global privs to admin", e);
} catch (NoSuchObjectException e) {
// Unlikely to be thrown.
LOG.warn("Failed while granting global privs to admin", e);
}
}
/**
* add admin users if they don't exist.
*
* This is a potential contention when HiveServer2 using embedded metastore and Metastore
* Server try to concurrently invoke addAdminUsers. If one failed, JDOException was caught for
* one more time try, if failed again, simply ignored by warning, which meant another succeeds.
*
* @throws MetaException
*/
private void addAdminUsers() throws MetaException {
try {
addAdminUsers_core();
} catch (JDOException e) {
LOG.warn("Retrying adding admin users after error: " + e.getMessage(), e);
addAdminUsers_core();
}
}
private void addAdminUsers_core() throws MetaException {
// now add pre-configured users to admin role
String userStr = MetastoreConf.getVar(conf,ConfVars.USERS_IN_ADMIN_ROLE,"").trim();
if (userStr.isEmpty()) {
LOG.info("No user is added in admin role, since config is empty");
return;
}
// Since user names need to be valid unix user names, per IEEE Std 1003.1-2001 they cannot
// contain comma, so we can safely split above string on comma.
Iterator<String> users = Splitter.on(",").trimResults().omitEmptyStrings().split(userStr).iterator();
if (!users.hasNext()) {
LOG.info("No user is added in admin role, since config value "+ userStr +
" is in incorrect format. We accept comma separated list of users.");
return;
}
Role adminRole;
RawStore ms = getMS();
try {
adminRole = ms.getRole(ADMIN);
} catch (NoSuchObjectException e) {
LOG.error("Failed to retrieve just added admin role",e);
return;
}
while (users.hasNext()) {
String userName = users.next();
try {
ms.grantRole(adminRole, userName, PrincipalType.USER, ADMIN, PrincipalType.ROLE, true);
LOG.info("Added " + userName + " to admin role");
} catch (NoSuchObjectException e) {
LOG.error("Failed to add "+ userName + " in admin role",e);
} catch (InvalidObjectException e) {
LOG.debug(userName + " already in admin role", e);
}
}
}
private static void logAndAudit(final String m) {
LOG.debug(m);
logAuditEvent(m);
}
private String startFunction(String function, String extraLogInfo) {
incrementCounter(function);
logAndAudit((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
function + extraLogInfo);
com.codahale.metrics.Timer timer =
Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + function);
if (timer != null) {
// Timer will be null we aren't using the metrics
HMSHandlerContext.getTimerContexts().put(function, timer.time());
}
Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
if (counter != null) {
counter.inc();
}
return function;
}
private String startFunction(String function) {
return startFunction(function, "");
}
private void startTableFunction(String function, String catName, String db, String tbl) {
startFunction(function, " : tbl=" +
TableName.getQualified(catName, db, tbl));
}
private void startMultiTableFunction(String function, String db, List<String> tbls) {
String tableNames = join(tbls, ",");
startFunction(function, " : db=" + db + " tbls=" + tableNames);
}
private void startPartitionFunction(String function, String cat, String db, String tbl,
List<String> partVals) {
startFunction(function, " : tbl=" +
TableName.getQualified(cat, db, tbl) + "[" + join(partVals, ",") + "]");
}
private void startPartitionFunction(String function, String catName, String db, String tbl,
Map<String, String> partName) {
startFunction(function, " : tbl=" +
TableName.getQualified(catName, db, tbl) + "partition=" + partName);
}
private void endFunction(String function, boolean successful, Exception e) {
endFunction(function, successful, e, null);
}
private void endFunction(String function, boolean successful, Exception e,
String inputTableName) {
endFunction(function, new MetaStoreEndFunctionContext(successful, e, inputTableName));
}
private void endFunction(String function, MetaStoreEndFunctionContext context) {
com.codahale.metrics.Timer.Context timerContext = HMSHandlerContext.getTimerContexts().remove(function);
if (timerContext != null) {
long timeTaken = timerContext.stop();
LOG.debug((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
function + "time taken(ns): " + timeTaken);
}
Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
if (counter != null) {
counter.dec();
}
for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
listener.onEndFunction(function, context);
}
}
@Override
public fb_status getStatus() {
return fb_status.ALIVE;
}
@Override
public void shutdown() {
cleanupHandlerContext();
PerfLogger.getPerfLogger(false).cleanupPerfLogMetrics();
}
@Override
public AbstractMap<String, Long> getCounters() {
AbstractMap<String, Long> counters = super.getCounters();
// Allow endFunctionListeners to add any counters they have collected
if (endFunctionListeners != null) {
for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
listener.exportCounters(counters);
}
}
return counters;
}
@Override
public void create_catalog(CreateCatalogRequest rqst)
throws AlreadyExistsException, InvalidObjectException, MetaException {
Catalog catalog = rqst.getCatalog();
startFunction("create_catalog", ": " + catalog.toString());
boolean success = false;
Exception ex = null;
try {
try {
getMS().getCatalog(catalog.getName());
throw new AlreadyExistsException("Catalog " + catalog.getName() + " already exists");
} catch (NoSuchObjectException e) {
// expected
}
if (!MetaStoreUtils.validateName(catalog.getName(), null)) {
throw new InvalidObjectException(catalog.getName() + " is not a valid catalog name");
}
if (catalog.getLocationUri() == null) {
throw new InvalidObjectException("You must specify a path for the catalog");
}
RawStore ms = getMS();
Path catPath = new Path(catalog.getLocationUri());
boolean madeDir = false;
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
try {
firePreEvent(new PreCreateCatalogEvent(this, catalog));
if (!wh.isDir(catPath)) {
if (!wh.mkdirs(catPath)) {
throw new MetaException("Unable to create catalog path " + catPath +
", failed to create catalog " + catalog.getName());
}
madeDir = true;
}
// set the create time of catalog
long time = System.currentTimeMillis() / 1000;
catalog.setCreateTime((int) time);
ms.openTransaction();
ms.createCatalog(catalog);
// Create a default database inside the catalog
Database db = new Database(DEFAULT_DATABASE_NAME,
"Default database for catalog " + catalog.getName(), catalog.getLocationUri(),
Collections.emptyMap());
db.setCatalogName(catalog.getName());
create_database_core(ms, db);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.CREATE_CATALOG,
new CreateCatalogEvent(true, this, catalog));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
if (madeDir) {
wh.deleteDir(catPath, true, false, false);
}
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.CREATE_CATALOG,
new CreateCatalogEvent(success, this, catalog),
null,
transactionalListenersResponses, ms);
}
}
success = true;
} catch (AlreadyExistsException|InvalidObjectException|MetaException e) {
ex = e;
throw e;
} finally {
endFunction("create_catalog", success, ex);
}
}
@Override
public void alter_catalog(AlterCatalogRequest rqst) throws TException {
startFunction("alter_catalog " + rqst.getName());
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
GetCatalogResponse oldCat = null;
try {
oldCat = get_catalog(new GetCatalogRequest(rqst.getName()));
// Above should have thrown NoSuchObjectException if there is no such catalog
assert oldCat != null && oldCat.getCatalog() != null;
firePreEvent(new PreAlterCatalogEvent(oldCat.getCatalog(), rqst.getNewCat(), this));
ms.openTransaction();
ms.alterCatalog(rqst.getName(), rqst.getNewCat());
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_CATALOG,
new AlterCatalogEvent(oldCat.getCatalog(), rqst.getNewCat(), true, this));
}
success = ms.commitTransaction();
} catch (MetaException|NoSuchObjectException e) {
ex = e;
throw e;
} finally {
if (!success) {
ms.rollbackTransaction();
}
if ((null != oldCat) && (!listeners.isEmpty())) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_CATALOG,
new AlterCatalogEvent(oldCat.getCatalog(), rqst.getNewCat(), success, this),
null, transactionalListenersResponses, ms);
}
endFunction("alter_catalog", success, ex);
}
}
@Override
public GetCatalogResponse get_catalog(GetCatalogRequest rqst)
throws NoSuchObjectException, TException {
String catName = rqst.getName();
startFunction("get_catalog", ": " + catName);
Catalog cat = null;
Exception ex = null;
try {
cat = getMS().getCatalog(catName);
firePreEvent(new PreReadCatalogEvent(this, cat));
return new GetCatalogResponse(cat);
} catch (MetaException|NoSuchObjectException e) {
ex = e;
throw e;
} finally {
endFunction("get_catalog", cat != null, ex);
}
}
@Override
public GetCatalogsResponse get_catalogs() throws MetaException {
startFunction("get_catalogs");
List<String> ret = null;
Exception ex = null;
try {
ret = getMS().getCatalogs();
} catch (Exception e) {
ex = e;
throw e;
} finally {
endFunction("get_catalog", ret != null, ex);
}
return new GetCatalogsResponse(ret == null ? Collections.emptyList() : ret);
}
@Override
public void drop_catalog(DropCatalogRequest rqst)
throws NoSuchObjectException, InvalidOperationException, MetaException {
String catName = rqst.getName();
startFunction("drop_catalog", ": " + catName);
if (DEFAULT_CATALOG_NAME.equalsIgnoreCase(catName)) {
endFunction("drop_catalog", false, null);
throw new MetaException("Can not drop " + DEFAULT_CATALOG_NAME + " catalog");
}
boolean success = false;
Exception ex = null;
try {
dropCatalogCore(catName);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(NoSuchObjectException.class, InvalidOperationException.class, MetaException.class)
.defaultMetaException();
} finally {
endFunction("drop_catalog", success, ex);
}
}
private void dropCatalogCore(String catName)
throws MetaException, NoSuchObjectException, InvalidOperationException {
boolean success = false;
Catalog cat = null;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
RawStore ms = getMS();
try {
ms.openTransaction();
cat = ms.getCatalog(catName);
firePreEvent(new PreDropCatalogEvent(this, cat));
List<String> allDbs = get_databases(prependNotNullCatToDbName(catName, null));
if (allDbs != null && !allDbs.isEmpty()) {
// It might just be the default, in which case we can drop that one if it's empty
if (allDbs.size() == 1 && allDbs.get(0).equals(DEFAULT_DATABASE_NAME)) {
try {
DropDatabaseRequest req = new DropDatabaseRequest();
req.setName(DEFAULT_DATABASE_NAME);
req.setCatalogName(catName);
req.setDeleteData(true);
req.setCascade(false);
drop_database_core(ms, req);
} catch (InvalidOperationException e) {
// This means there are tables of something in the database
throw new InvalidOperationException("There are still objects in the default " +
"database for catalog " + catName);
} catch (InvalidObjectException|IOException|InvalidInputException e) {
MetaException me = new MetaException("Error attempt to drop default database for " +
"catalog " + catName);
me.initCause(e);
throw me;
}
} else {
throw new InvalidOperationException("There are non-default databases in the catalog " +
catName + " so it cannot be dropped.");
}
}
ms.dropCatalog(catName) ;
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_CATALOG,
new DropCatalogEvent(true, this, cat));
}
success = ms.commitTransaction();
} finally {
if (success) {
wh.deleteDir(wh.getDnsPath(new Path(cat.getLocationUri())), false, false, false);
} else {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_CATALOG,
new DropCatalogEvent(success, this, cat),
null,
transactionalListenerResponses, ms);
}
}
}
static boolean isDbReplicationTarget(Database db) {
String dbCkptStatus = (db.getParameters() == null) ? null : db.getParameters().get(ReplConst.REPL_TARGET_DB_PROPERTY);
return dbCkptStatus != null && !dbCkptStatus.trim().isEmpty();
}
// Assumes that the catalog has already been set.
private void create_database_core(RawStore ms, final Database db)
throws AlreadyExistsException, InvalidObjectException, MetaException {
if (!MetaStoreUtils.validateName(db.getName(), conf)) {
throw new InvalidObjectException(db.getName() + " is not a valid database name");
}
Catalog cat = null;
try {
cat = getMS().getCatalog(db.getCatalogName());
} catch (NoSuchObjectException e) {
LOG.error("No such catalog " + db.getCatalogName());
throw new InvalidObjectException("No such catalog " + db.getCatalogName());
}
boolean skipAuthorization = false;
String passedInURI = db.getLocationUri();
String passedInManagedURI = db.getManagedLocationUri();
if (passedInURI == null && passedInManagedURI == null) {
skipAuthorization = true;
}
final Path defaultDbExtPath = wh.getDefaultDatabasePath(db.getName(), true);
final Path defaultDbMgdPath = wh.getDefaultDatabasePath(db.getName(), false);
final Path dbExtPath = (passedInURI != null) ? wh.getDnsPath(new Path(passedInURI)) : wh.determineDatabasePath(cat, db);
final Path dbMgdPath = (passedInManagedURI != null) ? wh.getDnsPath(new Path(passedInManagedURI)) : null;
if ((defaultDbExtPath.equals(dbExtPath) && defaultDbMgdPath.equals(dbMgdPath)) &&
((dbMgdPath == null) || dbMgdPath.equals(defaultDbMgdPath))) {
skipAuthorization = true;
}
if ( skipAuthorization ) {
//null out to skip authorizer URI check
db.setLocationUri(null);
db.setManagedLocationUri(null);
}else{
db.setLocationUri(dbExtPath.toString());
if (dbMgdPath != null) {
db.setManagedLocationUri(dbMgdPath.toString());
}
}
if (db.getOwnerName() == null){
try {
db.setOwnerName(SecurityUtils.getUGI().getShortUserName());
}catch (Exception e){
LOG.warn("Failed to get owner name for create database operation.", e);
}
}
long time = System.currentTimeMillis()/1000;
db.setCreateTime((int) time);
boolean success = false;
boolean madeManagedDir = false;
boolean madeExternalDir = false;
boolean isReplicated = isDbReplicationTarget(db);
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
try {
firePreEvent(new PreCreateDatabaseEvent(db, this));
//reinstate location uri for metastore db.
if (skipAuthorization == true){
db.setLocationUri(dbExtPath.toString());
if (dbMgdPath != null) {
db.setManagedLocationUri(dbMgdPath.toString());
}
}
if (db.getCatalogName() != null && !db.getCatalogName().
equals(Warehouse.DEFAULT_CATALOG_NAME)) {
if (!wh.isDir(dbExtPath)) {
LOG.debug("Creating database path " + dbExtPath);
if (!wh.mkdirs(dbExtPath)) {
throw new MetaException("Unable to create database path " + dbExtPath +
", failed to create database " + db.getName());
}
madeExternalDir = true;
}
} else {
if (dbMgdPath != null) {
try {
// Since this may be done as random user (if doAs=true) he may not have access
// to the managed directory. We run this as an admin user
madeManagedDir = UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Boolean>() {
@Override public Boolean run() throws MetaException {
if (!wh.isDir(dbMgdPath)) {
LOG.info("Creating database path in managed directory " + dbMgdPath);
if (!wh.mkdirs(dbMgdPath)) {
throw new MetaException("Unable to create database managed path " + dbMgdPath + ", failed to create database " + db.getName());
}
return true;
}
return false;
}
});
if (madeManagedDir) {
LOG.info("Created database path in managed directory " + dbMgdPath);
} else if (!isInTest || !isDbReplicationTarget(db)) { // Hive replication tests doesn't drop the db after each test
throw new MetaException(
"Unable to create database managed directory " + dbMgdPath + ", failed to create database " + db.getName());
}
} catch (IOException | InterruptedException e) {
throw new MetaException(
"Unable to create database managed directory " + dbMgdPath + ", failed to create database " + db.getName() + ":" + e.getMessage());
}
}
if (dbExtPath != null) {
try {
madeExternalDir = UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Boolean>() {
@Override public Boolean run() throws MetaException {
if (!wh.isDir(dbExtPath)) {
LOG.info("Creating database path in external directory " + dbExtPath);
return wh.mkdirs(dbExtPath);
}
return false;
}
});
if (madeExternalDir) {
LOG.info("Created database path in external directory " + dbExtPath);
} else {
LOG.warn("Failed to create external path " + dbExtPath + " for database " + db.getName() + ". This may result in access not being allowed if the "
+ "StorageBasedAuthorizationProvider is enabled");
}
} catch (IOException | InterruptedException | UndeclaredThrowableException e) {
throw new MetaException("Failed to create external path " + dbExtPath + " for database " + db.getName() + ". This may result in access not being allowed if the "
+ "StorageBasedAuthorizationProvider is enabled: " + e.getMessage());
}
} else {
LOG.info("Database external path won't be created since the external warehouse directory is not defined");
}
}
ms.openTransaction();
ms.createDatabase(db);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.CREATE_DATABASE,
new CreateDatabaseEvent(db, true, this, isReplicated));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
if (db.getCatalogName() != null && !db.getCatalogName().
equals(Warehouse.DEFAULT_CATALOG_NAME)) {
if (madeManagedDir && dbMgdPath != null) {
wh.deleteDir(dbMgdPath, true, db);
}
} else {
if (madeManagedDir && dbMgdPath != null) {
try {
UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
@Override public Void run() throws Exception {
wh.deleteDir(dbMgdPath, true, db);
return null;
}
});
} catch (IOException | InterruptedException e) {
LOG.error(
"Couldn't delete managed directory " + dbMgdPath + " after " + "it was created for database " + db.getName() + " " + e.getMessage());
}
}
if (madeExternalDir && dbExtPath != null) {
try {
UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Void>() {
@Override public Void run() throws Exception {
wh.deleteDir(dbExtPath, true, db);
return null;
}
});
} catch (IOException | InterruptedException e) {
LOG.error("Couldn't delete external directory " + dbExtPath + " after " + "it was created for database "
+ db.getName() + " " + e.getMessage());
}
}
}
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.CREATE_DATABASE,
new CreateDatabaseEvent(db, success, this, isReplicated),
null,
transactionalListenersResponses, ms);
}
}
}
@Override
public void create_database(final Database db)
throws AlreadyExistsException, InvalidObjectException, MetaException {
startFunction("create_database", ": " + db.toString());
boolean success = false;
Exception ex = null;
if (!db.isSetCatalogName()) {
db.setCatalogName(getDefaultCatalog(conf));
}
try {
try {
if (null != get_database_core(db.getCatalogName(), db.getName())) {
throw new AlreadyExistsException("Database " + db.getName() + " already exists");
}
} catch (NoSuchObjectException e) {
// expected
}
if (testTimeoutEnabled) {
try {
Thread.sleep(testTimeoutValue);
} catch (InterruptedException e) {
// do nothing
}
Deadline.checkTimeout();
}
create_database_core(getMS(), db);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
.defaultMetaException();
} finally {
endFunction("create_database", success, ex);
}
}
@Override
public Database get_database(final String name)
throws NoSuchObjectException, MetaException {
GetDatabaseRequest request = new GetDatabaseRequest();
String[] parsedDbName = parseDbName(name, conf);
request.setName(parsedDbName[DB_NAME]);
if (parsedDbName[CAT_NAME] != null) {
request.setCatalogName(parsedDbName[CAT_NAME]);
}
return get_database_req(request);
}
@Override
public Database get_database_core(String catName, final String name) throws NoSuchObjectException, MetaException {
Database db = null;
if (name == null) {
throw new MetaException("Database name cannot be null.");
}
try {
db = getMS().getDatabase(catName, name);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
.defaultRuntimeException();
}
return db;
}
@Override
public Database get_database_req(GetDatabaseRequest request) throws NoSuchObjectException, MetaException {
startFunction("get_database", ": " + request.getName());
Database db = null;
Exception ex = null;
if (request.getName() == null) {
throw new MetaException("Database name cannot be null.");
}
List<String> processorCapabilities = request.getProcessorCapabilities();
String processorId = request.getProcessorIdentifier();
try {
db = getMS().getDatabase(request.getCatalogName(), request.getName());
firePreEvent(new PreReadDatabaseEvent(db, this));
if (transformer != null) {
db = transformer.transformDatabase(db, processorCapabilities, processorId);
}
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
.defaultRuntimeException();
} finally {
endFunction("get_database", db != null, ex);
}
return db;
}
@Override
public void alter_database(final String dbName, final Database newDB) throws TException {
startFunction("alter_database " + dbName);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
Database oldDB = null;
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
// Perform the same URI normalization as create_database_core.
if (newDB.getLocationUri() != null) {
newDB.setLocationUri(wh.getDnsPath(new Path(newDB.getLocationUri())).toString());
}
String[] parsedDbName = parseDbName(dbName, conf);
// We can replicate into an empty database, in which case newDB will have indication that
// it's target of replication but not oldDB. But replication flow will never alter a
// database so that oldDB indicates that it's target or replication but not the newDB. So,
// relying solely on newDB to check whether the database is target of replication works.
boolean isReplicated = isDbReplicationTarget(newDB);
try {
oldDB = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
if (oldDB == null) {
throw new MetaException("Could not alter database \"" + parsedDbName[DB_NAME] +
"\". Could not retrieve old definition.");
}
// Add replication target event id.
if (isReplicationEventIdUpdate(oldDB, newDB)) {
Map<String, String> oldParams = new LinkedHashMap<>(newDB.getParameters());
String currentNotificationLogID = Long.toString(ms.getCurrentNotificationEventId().getEventId());
oldParams.put(REPL_TARGET_DATABASE_PROPERTY, currentNotificationLogID);
LOG.debug("Adding the {} property for database {} with event id {}", REPL_TARGET_DATABASE_PROPERTY,
newDB.getName(), currentNotificationLogID);
newDB.setParameters(oldParams);
}
firePreEvent(new PreAlterDatabaseEvent(oldDB, newDB, this));
ms.openTransaction();
ms.alterDatabase(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], newDB);
if (!transactionalListeners.isEmpty()) {
AlterDatabaseEvent event = new AlterDatabaseEvent(oldDB, newDB, true, this, isReplicated);
if (!event.shouldSkipCapturing()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ALTER_DATABASE, event);
}
}
success = ms.commitTransaction();
} catch (MetaException|NoSuchObjectException e) {
ex = e;
throw e;
} finally {
if (!success) {
ms.rollbackTransaction();
}
if ((null != oldDB) && (!listeners.isEmpty())) {
AlterDatabaseEvent event = new AlterDatabaseEvent(oldDB, newDB, success, this, isReplicated);
if (!event.shouldSkipCapturing()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_DATABASE, event, null, transactionalListenersResponses, ms);
}
}
endFunction("alter_database", success, ex);
}
}
/**
* Checks whether the repl.last.id is being updated.
* @param oldDb the old db object
* @param newDb the new db object
* @return true if repl.last.id is being changed.
*/
private boolean isReplicationEventIdUpdate(Database oldDb, Database newDb) {
Map<String, String> oldDbProp = oldDb.getParameters();
Map<String, String> newDbProp = newDb.getParameters();
if (newDbProp == null || newDbProp.isEmpty() ||
Boolean.parseBoolean(newDbProp.get(REPL_RESUME_STARTED_AFTER_FAILOVER))) {
return false;
}
String newReplId = newDbProp.get(ReplConst.REPL_TARGET_TABLE_PROPERTY);
String oldReplId = oldDbProp != null ? oldDbProp.get(ReplConst.REPL_TARGET_TABLE_PROPERTY) : null;
return newReplId != null && !newReplId.equalsIgnoreCase(oldReplId);
}
private void drop_database_core(RawStore ms, DropDatabaseRequest req) throws NoSuchObjectException,
InvalidOperationException, MetaException, IOException, InvalidObjectException, InvalidInputException {
boolean success = false;
Database db = null;
List<Path> tablePaths = new ArrayList<>();
List<Path> partitionPaths = new ArrayList<>();
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
if (req.getName() == null) {
throw new MetaException("Database name cannot be null.");
}
boolean isReplicated = false;
try {
ms.openTransaction();
db = ms.getDatabase(req.getCatalogName(), req.getName());
if (db.getType() == DatabaseType.REMOTE) {
success = drop_remote_database_core(ms, db);
return;
}
isReplicated = isDbReplicationTarget(db);
if (!isInTest && ReplChangeManager.isSourceOfReplication(db)) {
throw new InvalidOperationException("can not drop a database which is a source of replication");
}
firePreEvent(new PreDropDatabaseEvent(db, this));
String catPrependedName = MetaStoreUtils.prependCatalogToDbName(req.getCatalogName(), req.getName(), conf);
Set<String> uniqueTableNames = new HashSet<>(get_all_tables(catPrependedName));
List<String> allFunctions = get_functions(catPrependedName, "*");
ListStoredProcedureRequest request = new ListStoredProcedureRequest(req.getCatalogName());
request.setDbName(req.getName());
List<String> allProcedures = get_all_stored_procedures(request);
ListPackageRequest pkgRequest = new ListPackageRequest(req.getCatalogName());
pkgRequest.setDbName(req.getName());
List<String> allPackages = get_all_packages(pkgRequest);
if (!req.isCascade()) {
if (!uniqueTableNames.isEmpty()) {
throw new InvalidOperationException(
"Database " + db.getName() + " is not empty. One or more tables exist.");
}
if (!allFunctions.isEmpty()) {
throw new InvalidOperationException(
"Database " + db.getName() + " is not empty. One or more functions exist.");
}
if (!allProcedures.isEmpty()) {
throw new InvalidOperationException(
"Database " + db.getName() + " is not empty. One or more stored procedures exist.");
}
if (!allPackages.isEmpty()) {
throw new InvalidOperationException(
"Database " + db.getName() + " is not empty. One or more packages exist.");
}
}
Path path = new Path(db.getLocationUri()).getParent();
if (!wh.isWritable(path)) {
throw new MetaException("Database not dropped since its external warehouse location " + path + " is not writable by " +
SecurityUtils.getUser());
}
path = wh.getDatabaseManagedPath(db).getParent();
if (!wh.isWritable(path)) {
throw new MetaException("Database not dropped since its managed warehouse location " + path + " is not writable by " +
SecurityUtils.getUser());
}
Path databasePath = wh.getDnsPath(wh.getDatabasePath(db));
// drop any functions before dropping db
for (String funcName : allFunctions) {
drop_function(catPrependedName, funcName);
}
for (String procName : allProcedures) {
drop_stored_procedure(new StoredProcedureRequest(req.getCatalogName(), req.getName(), procName));
}
for (String pkgName : allPackages) {
drop_package(new DropPackageRequest(req.getCatalogName(), req.getName(), pkgName));
}
final int tableBatchSize = MetastoreConf.getIntVar(conf,
ConfVars.BATCH_RETRIEVE_MAX);
// First pass will drop the materialized views
List<String> materializedViewNames = getTablesByTypeCore(req.getCatalogName(), req.getName(), ".*",
TableType.MATERIALIZED_VIEW.toString());
int startIndex = 0;
// retrieve the tables from the metastore in batches to alleviate memory constraints
while (startIndex < materializedViewNames.size()) {
int endIndex = Math.min(startIndex + tableBatchSize, materializedViewNames.size());
List<Table> materializedViews;
try {
materializedViews = ms.getTableObjectsByName(req.getCatalogName(), req.getName(), materializedViewNames.subList(startIndex, endIndex));
} catch (UnknownDBException e) {
throw new MetaException(e.getMessage());
}
if (materializedViews != null && !materializedViews.isEmpty()) {
for (Table materializedView : materializedViews) {
boolean isSoftDelete = TxnUtils.isTableSoftDeleteEnabled(materializedView, req.isSoftDelete());
if (materializedView.getSd().getLocation() != null && !isSoftDelete) {
Path materializedViewPath = wh.getDnsPath(new Path(materializedView.getSd().getLocation()));
if (!FileUtils.isSubdirectory(databasePath.toString(), materializedViewPath.toString()) || req.isSoftDelete()) {
if (!wh.isWritable(materializedViewPath.getParent())) {
throw new MetaException("Database metadata not deleted since table: " +
materializedView.getTableName() + " has a parent location " + materializedViewPath.getParent() +
" which is not writable by " + SecurityUtils.getUser());
}
tablePaths.add(materializedViewPath);
}
}
EnvironmentContext context = null;
if (isSoftDelete) {
context = new EnvironmentContext();
context.putToProperties(hive_metastoreConstants.TXN_ID, String.valueOf(req.getTxnId()));
}
// Drop the materialized view but not its data
drop_table_with_environment_context(
req.getName(), materializedView.getTableName(), false, context);
// Remove from all tables
uniqueTableNames.remove(materializedView.getTableName());
}
}
startIndex = endIndex;
}
// drop tables before dropping db
List<String> allTables = new ArrayList<>(uniqueTableNames);
startIndex = 0;
// retrieve the tables from the metastore in batches to alleviate memory constraints
while (startIndex < allTables.size()) {
int endIndex = Math.min(startIndex + tableBatchSize, allTables.size());
List<Table> tables;
try {
tables = ms.getTableObjectsByName(req.getCatalogName(), req.getName(), allTables.subList(startIndex, endIndex));
} catch (UnknownDBException e) {
throw new MetaException(e.getMessage());
}
if (tables != null && !tables.isEmpty()) {
for (Table table : tables) {
// If the table is not external and it might not be in a subdirectory of the database
// add it's locations to the list of paths to delete
boolean isSoftDelete = TxnUtils.isTableSoftDeleteEnabled(table, req.isSoftDelete());
Path tablePath = null;
boolean tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(table, req.isDeleteData())
&& !isSoftDelete;
boolean isManagedTable = table.getTableType().equals(TableType.MANAGED_TABLE.toString());
if (table.getSd().getLocation() != null && tableDataShouldBeDeleted) {
tablePath = wh.getDnsPath(new Path(table.getSd().getLocation()));
if (!isManagedTable || req.isSoftDelete()) {
if (!wh.isWritable(tablePath.getParent())) {
throw new MetaException(
"Database metadata not deleted since table: " + table.getTableName() + " has a parent location "
+ tablePath.getParent() + " which is not writable by " + SecurityUtils.getUser());
}
tablePaths.add(tablePath);
}
}
// For each partition in each table, drop the partitions and get a list of
// partitions' locations which might need to be deleted
partitionPaths = dropPartitionsAndGetLocations(ms, req.getCatalogName(), req.getName(), table.getTableName(),
tablePath, tableDataShouldBeDeleted);
EnvironmentContext context = null;
if (isSoftDelete) {
context = new EnvironmentContext();
context.putToProperties(hive_metastoreConstants.TXN_ID, String.valueOf(req.getTxnId()));
req.setDeleteManagedDir(false);
}
// Drop the table but not its data
drop_table_with_environment_context(
MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf),
table.getTableName(), false, context, false);
}
}
startIndex = endIndex;
}
if (ms.dropDatabase(req.getCatalogName(), req.getName())) {
if (!transactionalListeners.isEmpty()) {
DropDatabaseEvent dropEvent = new DropDatabaseEvent(db, true, this, isReplicated);
EnvironmentContext context = null;
if (!req.isDeleteManagedDir()) {
context = new EnvironmentContext();
context.putToProperties(hive_metastoreConstants.TXN_ID, String.valueOf(req.getTxnId()));
}
dropEvent.setEnvironmentContext(context);
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_DATABASE, dropEvent);
}
success = ms.commitTransaction();
}
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (req.isDeleteData()) {
// Delete the data in the partitions which have other locations
deletePartitionData(partitionPaths, false, db);
// Delete the data in the tables which have other locations or soft-delete is enabled
for (Path tablePath : tablePaths) {
deleteTableData(tablePath, false, db);
}
final Database dbFinal = db;
final Path path = (dbFinal.getManagedLocationUri() != null) ?
new Path(dbFinal.getManagedLocationUri()) : wh.getDatabaseManagedPath(dbFinal);
if (req.isDeleteManagedDir()) {
try {
Boolean deleted = UserGroupInformation.getLoginUser().doAs((PrivilegedExceptionAction<Boolean>)
() -> wh.deleteDir(path, true, dbFinal));
if (!deleted) {
LOG.error("Failed to delete database's managed warehouse directory: " + path);
}
} catch (Exception e) {
LOG.error("Failed to delete database's managed warehouse directory: " + path + " " + e.getMessage());
}
}
try {
Boolean deleted = UserGroupInformation.getCurrentUser().doAs((PrivilegedExceptionAction<Boolean>)
() -> wh.deleteDir(new Path(dbFinal.getLocationUri()), true, dbFinal));
if (!deleted) {
LOG.error("Failed to delete database external warehouse directory " + db.getLocationUri());
}
} catch (IOException | InterruptedException | UndeclaredThrowableException e) {
LOG.error("Failed to delete the database external warehouse directory: " + db.getLocationUri() + " " + e
.getMessage());
}
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_DATABASE,
new DropDatabaseEvent(db, success, this, isReplicated),
null,
transactionalListenerResponses, ms);
}
}
}
private boolean drop_remote_database_core(RawStore ms, final Database db) throws MetaException, NoSuchObjectException {
boolean success = false;
firePreEvent(new PreDropDatabaseEvent(db, this));
if (ms.dropDatabase(db.getCatalogName(), db.getName())) {
success = ms.commitTransaction();
}
return success;
}
@Override
public void drop_database(final String dbName, final boolean deleteData, final boolean cascade)
throws NoSuchObjectException, InvalidOperationException, MetaException {
String[] parsedDbName = parseDbName(dbName, conf);
DropDatabaseRequest req = new DropDatabaseRequest();
req.setName(parsedDbName[DB_NAME]);
req.setCatalogName(parsedDbName[CAT_NAME]);
req.setDeleteData(deleteData);
req.setCascade(cascade);
drop_database_req(req);
}
@Override
public void drop_database_req(final DropDatabaseRequest req)
throws NoSuchObjectException, InvalidOperationException, MetaException {
startFunction("drop_database", ": " + req.getName());
if (DEFAULT_CATALOG_NAME.equalsIgnoreCase(req.getCatalogName())
&& DEFAULT_DATABASE_NAME.equalsIgnoreCase(req.getName())) {
endFunction("drop_database", false, null);
throw new MetaException("Can not drop " + DEFAULT_DATABASE_NAME + " database in catalog "
+ DEFAULT_CATALOG_NAME);
}
boolean success = false;
Exception ex = null;
try {
drop_database_core(getMS(), req);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(NoSuchObjectException.class, InvalidOperationException.class, MetaException.class)
.defaultMetaException();
} finally {
endFunction("drop_database", success, ex);
}
}
@Override
public List<String> get_databases(final String pattern) throws MetaException {
startFunction("get_databases", ": " + pattern);
String[] parsedDbNamed = parseDbName(pattern, conf);
List<String> ret = null;
Exception ex = null;
try {
if (parsedDbNamed[DB_NAME] == null) {
ret = getMS().getAllDatabases(parsedDbNamed[CAT_NAME]);
ret = FilterUtils.filterDbNamesIfEnabled(isServerFilterEnabled, filterHook, ret);
} else {
ret = getMS().getDatabases(parsedDbNamed[CAT_NAME], parsedDbNamed[DB_NAME]);
ret = FilterUtils.filterDbNamesIfEnabled(isServerFilterEnabled, filterHook, ret);
}
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_databases", ret != null, ex);
}
return ret;
}
@Override
public List<String> get_all_databases() throws MetaException {
// get_databases filters results already. No need to filter here
return get_databases(MetaStoreUtils.prependCatalogToDbName(null, null, conf));
}
private void create_dataconnector_core(RawStore ms, final DataConnector connector)
throws AlreadyExistsException, InvalidObjectException, MetaException {
if (!MetaStoreUtils.validateName(connector.getName(), conf)) {
throw new InvalidObjectException(connector.getName() + " is not a valid dataconnector name");
}
if (connector.getOwnerName() == null){
try {
connector.setOwnerName(SecurityUtils.getUGI().getShortUserName());
}catch (Exception e){
LOG.warn("Failed to get owner name for create dataconnector operation.", e);
}
}
long time = System.currentTimeMillis()/1000;
connector.setCreateTime((int) time);
boolean success = false;
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
try {
firePreEvent(new PreCreateDataConnectorEvent(connector, this));
ms.openTransaction();
ms.createDataConnector(connector);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.CREATE_DATACONNECTOR,
new CreateDataConnectorEvent(connector, true, this));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.CREATE_DATACONNECTOR,
new CreateDataConnectorEvent(connector, success, this),
null,
transactionalListenersResponses, ms);
}
}
}
@Override
public void create_dataconnector(final DataConnector connector)
throws AlreadyExistsException, InvalidObjectException, MetaException {
startFunction("create_dataconnector", ": " + connector.toString());
boolean success = false;
Exception ex = null;
try {
try {
if (null != get_dataconnector_core(connector.getName())) {
throw new AlreadyExistsException("DataConnector " + connector.getName() + " already exists");
}
} catch (NoSuchObjectException e) {
// expected
}
if (testTimeoutEnabled) {
try {
Thread.sleep(testTimeoutValue);
} catch (InterruptedException e) {
// do nothing
}
Deadline.checkTimeout();
}
create_dataconnector_core(getMS(), connector);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
.defaultMetaException();
} finally {
endFunction("create_connector", success, ex);
}
}
@Override
public DataConnector get_dataconnector_core(final String name) throws NoSuchObjectException, MetaException {
DataConnector connector = null;
if (name == null) {
throw new MetaException("Data connector name cannot be null.");
}
try {
connector = getMS().getDataConnector(name);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
.defaultRuntimeException();
}
return connector;
}
@Override
public DataConnector get_dataconnector_req(GetDataConnectorRequest request) throws NoSuchObjectException, MetaException {
startFunction("get_dataconnector", ": " + request.getConnectorName());
DataConnector connector = null;
Exception ex = null;
try {
connector = get_dataconnector_core(request.getConnectorName());
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
.defaultRuntimeException();
} finally {
endFunction("get_dataconnector", connector != null, ex);
}
return connector;
}
@Override
public void alter_dataconnector(final String dcName, final DataConnector newDC) throws TException {
startFunction("alter_dataconnector " + dcName);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
DataConnector oldDC = null;
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
try {
oldDC = get_dataconnector_core(dcName);
if (oldDC == null) {
throw new MetaException("Could not alter dataconnector \"" + dcName +
"\". Could not retrieve old definition.");
}
firePreEvent(new PreAlterDataConnectorEvent(oldDC, newDC, this));
ms.openTransaction();
ms.alterDataConnector(dcName, newDC);
/*
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_DATACONNECTOR,
new AlterDataConnectorEvent(oldDC, newDC, true, this));
}
*/
success = ms.commitTransaction();
} catch (MetaException|NoSuchObjectException e) {
ex = e;
throw e;
} finally {
if (!success) {
ms.rollbackTransaction();
}
/*
if ((null != oldDC) && (!listeners.isEmpty())) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_DATACONNECTOR,
new AlterDataConnectorEvent(oldDC, newDC, success, this),
null,
transactionalListenersResponses, ms);
}
*/
endFunction("alter_database", success, ex);
}
}
@Override
public List<String> get_dataconnectors() throws MetaException {
startFunction("get_dataconnectors");
List<String> ret = null;
Exception ex = null;
try {
ret = getMS().getAllDataConnectorNames();
ret = FilterUtils.filterDataConnectorsIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_dataconnectors", ret != null, ex);
}
return ret;
}
@Override
public void drop_dataconnector(final String dcName, boolean ifNotExists, boolean checkReferences) throws NoSuchObjectException, InvalidOperationException, MetaException {
startFunction("drop_dataconnector", ": " + dcName);
boolean success = false;
DataConnector connector = null;
Exception ex = null;
RawStore ms = getMS();
try {
connector = getMS().getDataConnector(dcName);
} catch (NoSuchObjectException e) {
if (!ifNotExists) {
throw new NoSuchObjectException("DataConnector " + dcName + " doesn't exist");
} else {
return;
}
}
try {
ms.openTransaction();
// TODO find DBs with references to this connector
// if any existing references and checkReferences=true, do not drop
firePreEvent(new PreDropDataConnectorEvent(connector, this));
if (!ms.dropDataConnector(dcName)) {
throw new MetaException("Unable to drop dataconnector " + dcName);
} else {
/*
// TODO
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_TABLE,
new DropTableEvent(tbl, true, deleteData,
this, isReplicated),
envContext);
}
*/
success = ms.commitTransaction();
}
} finally {
if (!success) {
ms.rollbackTransaction();
}
/*
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_TABLE,
new DropTableEvent(tbl, success, deleteData, this, isReplicated),
envContext,
transactionalListenerResponses, ms);
}
*/
endFunction("drop_dataconnector", success, ex);
}
}
private void create_type_core(final RawStore ms, final Type type)
throws AlreadyExistsException, MetaException, InvalidObjectException {
if (!MetaStoreUtils.validateName(type.getName(), null)) {
throw new InvalidObjectException("Invalid type name");
}
boolean success = false;
try {
ms.openTransaction();
if (is_type_exists(ms, type.getName())) {
throw new AlreadyExistsException("Type " + type.getName() + " already exists");
}
ms.createType(type);
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
}
}
@Override
public boolean create_type(final Type type) throws AlreadyExistsException,
MetaException, InvalidObjectException {
startFunction("create_type", ": " + type.toString());
boolean success = false;
Exception ex = null;
try {
create_type_core(getMS(), type);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
.defaultMetaException();
} finally {
endFunction("create_type", success, ex);
}
return success;
}
@Override
public Type get_type(final String name) throws MetaException, NoSuchObjectException {
startFunction("get_type", ": " + name);
Type ret = null;
Exception ex = null;
try {
ret = getMS().getType(name);
if (null == ret) {
throw new NoSuchObjectException("Type \"" + name + "\" not found.");
}
} catch (Exception e) {
ex = e;
throwMetaException(e);
} finally {
endFunction("get_type", ret != null, ex);
}
return ret;
}
private boolean is_type_exists(RawStore ms, String typeName)
throws MetaException {
return (ms.getType(typeName) != null);
}
@Override
public boolean drop_type(final String name) throws MetaException, NoSuchObjectException {
startFunction("drop_type", ": " + name);
boolean success = false;
Exception ex = null;
try {
// TODO:pc validate that there are no types that refer to this
success = getMS().dropType(name);
} catch (Exception e) {
ex = e;
throwMetaException(e);
} finally {
endFunction("drop_type", success, ex);
}
return success;
}
@Override
public Map<String, Type> get_type_all(String name) throws MetaException {
// TODO Auto-generated method stub
startFunction("get_type_all", ": " + name);
endFunction("get_type_all", false, null);
throw new MetaException("Not yet implemented");
}
@Override
public Table translate_table_dryrun(final CreateTableRequest req) throws AlreadyExistsException,
MetaException, InvalidObjectException, InvalidInputException {
Table transformedTbl = null;
Table tbl = req.getTable();
List<String> processorCapabilities = req.getProcessorCapabilities();
String processorId = req.getProcessorIdentifier();
if (!tbl.isSetCatName()) {
tbl.setCatName(getDefaultCatalog(conf));
}
if (transformer != null) {
transformedTbl = transformer.transformCreateTable(tbl, processorCapabilities, processorId);
}
return transformedTbl != null ? transformedTbl : tbl;
}
private void create_table_core(final RawStore ms, final Table tbl,
final EnvironmentContext envContext)
throws AlreadyExistsException, MetaException,
InvalidObjectException, NoSuchObjectException, InvalidInputException {
CreateTableRequest req = new CreateTableRequest(tbl);
req.setEnvContext(envContext);
create_table_core(ms, req);
}
private void create_table_core(final RawStore ms, final Table tbl,
final EnvironmentContext envContext, List<SQLPrimaryKey> primaryKeys,
List<SQLForeignKey> foreignKeys, List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints, List<SQLDefaultConstraint> defaultConstraints,
List<SQLCheckConstraint> checkConstraints,
List<String> processorCapabilities, String processorIdentifier)
throws AlreadyExistsException, MetaException,
InvalidObjectException, NoSuchObjectException, InvalidInputException {
CreateTableRequest req = new CreateTableRequest(tbl);
if (envContext != null) {
req.setEnvContext(envContext);
}
if (primaryKeys != null) {
req.setPrimaryKeys(primaryKeys);
}
if (foreignKeys != null) {
req.setForeignKeys(foreignKeys);
}
if (uniqueConstraints != null) {
req.setUniqueConstraints(uniqueConstraints);
}
if (notNullConstraints != null) {
req.setNotNullConstraints(notNullConstraints);
}
if (defaultConstraints != null) {
req.setDefaultConstraints(defaultConstraints);
}
if (checkConstraints != null) {
req.setCheckConstraints(checkConstraints);
}
if (processorCapabilities != null) {
req.setProcessorCapabilities(processorCapabilities);
req.setProcessorIdentifier(processorIdentifier);
}
create_table_core(ms, req);
}
private void create_table_core(final RawStore ms, final CreateTableRequest req)
throws AlreadyExistsException, MetaException,
InvalidObjectException, NoSuchObjectException, InvalidInputException {
ColumnStatistics colStats = null;
Table tbl = req.getTable();
EnvironmentContext envContext = req.getEnvContext();
SQLAllTableConstraints constraints = new SQLAllTableConstraints();
constraints.setPrimaryKeys(req.getPrimaryKeys());
constraints.setForeignKeys(req.getForeignKeys());
constraints.setUniqueConstraints(req.getUniqueConstraints());
constraints.setDefaultConstraints(req.getDefaultConstraints());
constraints.setCheckConstraints(req.getCheckConstraints());
constraints.setNotNullConstraints(req.getNotNullConstraints());
List<String> processorCapabilities = req.getProcessorCapabilities();
String processorId = req.getProcessorIdentifier();
// To preserve backward compatibility throw MetaException in case of null database
if (tbl.getDbName() == null) {
throw new MetaException("Null database name is not allowed");
}
if (!MetaStoreUtils.validateName(tbl.getTableName(), conf)) {
throw new InvalidObjectException(tbl.getTableName()
+ " is not a valid object name");
}
if (!MetaStoreUtils.validateTblStorage(tbl.getSd())) {
throw new InvalidObjectException(tbl.getTableName()
+ " location must not be root path");
}
if (!tbl.isSetCatName()) {
tbl.setCatName(getDefaultCatalog(conf));
}
Database db = get_database_core(tbl.getCatName(), tbl.getDbName());
if (db != null && db.getType().equals(DatabaseType.REMOTE)) {
// HIVE-24425: Create table in REMOTE db should fail
throw new MetaException("Create table in REMOTE database " + db.getName() + " is not allowed");
}
if (is_table_exists(ms, tbl.getCatName(), tbl.getDbName(), tbl.getTableName())) {
throw new AlreadyExistsException("Table " + getCatalogQualifiedTableName(tbl)
+ " already exists");
}
tbl.setDbName(normalizeIdentifier(tbl.getDbName()));
tbl.setTableName(normalizeIdentifier(tbl.getTableName()));
if (transformer != null) {
tbl = transformer.transformCreateTable(tbl, processorCapabilities, processorId);
}
if (tbl.getParameters() != null) {
tbl.getParameters().remove(TABLE_IS_CTAS);
tbl.getParameters().remove(TABLE_IS_CTLT);
}
// If the given table has column statistics, save it here. We will update it later.
// We don't want it to be part of the Table object being created, lest the create table
// event will also have the col stats which we don't want.
if (tbl.isSetColStats()) {
colStats = tbl.getColStats();
tbl.unsetColStats();
}
String validate = MetaStoreServerUtils.validateTblColumns(tbl.getSd().getCols());
if (validate != null) {
throw new InvalidObjectException("Invalid column " + validate);
}
if (tbl.getPartitionKeys() != null) {
validate = MetaStoreServerUtils.validateTblColumns(tbl.getPartitionKeys());
if (validate != null) {
throw new InvalidObjectException("Invalid partition column " + validate);
}
}
if (tbl.isSetId()) {
LOG.debug("Id shouldn't be set but table {}.{} has the Id set to {}. Id is ignored.", tbl.getDbName(),
tbl.getTableName(), tbl.getId());
tbl.unsetId();
}
SkewedInfo skew = tbl.getSd().getSkewedInfo();
if (skew != null) {
validate = MetaStoreServerUtils.validateSkewedColNames(skew.getSkewedColNames());
if (validate != null) {
throw new InvalidObjectException("Invalid skew column " + validate);
}
validate = MetaStoreServerUtils.validateSkewedColNamesSubsetCol(
skew.getSkewedColNames(), tbl.getSd().getCols());
if (validate != null) {
throw new InvalidObjectException("Invalid skew column " + validate);
}
}
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
Path tblPath = null;
boolean success = false, madeDir = false;
boolean isReplicated = false;
try {
ms.openTransaction();
db = ms.getDatabase(tbl.getCatName(), tbl.getDbName());
isReplicated = isDbReplicationTarget(db);
firePreEvent(new PreCreateTableEvent(tbl, db, this));
if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
if (tbl.getSd().getLocation() == null
|| tbl.getSd().getLocation().isEmpty()) {
tblPath = wh.getDefaultTablePath(db, tbl.getTableName() + getTableSuffix(tbl), isExternal(tbl));
} else {
if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) {
LOG.warn("Location: " + tbl.getSd().getLocation()
+ " specified for non-external table:" + tbl.getTableName());
}
tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation()));
// ignore suffix if it's already there (direct-write CTAS)
if (!tblPath.getName().matches("(.*)" + SOFT_DELETE_TABLE_PATTERN)) {
tblPath = new Path(tblPath + getTableSuffix(tbl));
}
}
tbl.getSd().setLocation(tblPath.toString());
}
if (tblPath != null) {
if (!wh.isDir(tblPath)) {
if (!wh.mkdirs(tblPath)) {
throw new MetaException(tblPath
+ " is not a directory or unable to create one");
}
madeDir = true;
}
}
MetaStoreServerUtils.updateTableStatsForCreateTable(wh, db, tbl, envContext, conf, tblPath, madeDir);
// set create time
long time = System.currentTimeMillis() / 1000;
tbl.setCreateTime((int) time);
if (tbl.getParameters() == null ||
tbl.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) {
tbl.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time));
}
if (CollectionUtils.isEmpty(constraints.getPrimaryKeys()) && CollectionUtils.isEmpty(constraints.getForeignKeys())
&& CollectionUtils.isEmpty(constraints.getUniqueConstraints())&& CollectionUtils.isEmpty(constraints.getNotNullConstraints())&& CollectionUtils.isEmpty(constraints.getDefaultConstraints())
&& CollectionUtils.isEmpty(constraints.getCheckConstraints())) {
ms.createTable(tbl);
} else {
final String catName = tbl.getCatName();
// Check that constraints have catalog name properly set first
if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys()) && !constraints.getPrimaryKeys().get(0).isSetCatName()) {
constraints.getPrimaryKeys().forEach(constraint -> constraint.setCatName(catName));
}
if (CollectionUtils.isNotEmpty(constraints.getForeignKeys()) && !constraints.getForeignKeys().get(0).isSetCatName()) {
constraints.getForeignKeys().forEach(constraint -> constraint.setCatName(catName));
}
if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints()) && !constraints.getUniqueConstraints().get(0).isSetCatName()) {
constraints.getUniqueConstraints().forEach(constraint -> constraint.setCatName(catName));
}
if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints()) && !constraints.getNotNullConstraints().get(0).isSetCatName()) {
constraints.getNotNullConstraints().forEach(constraint -> constraint.setCatName(catName));
}
if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints()) && !constraints.getDefaultConstraints().get(0).isSetCatName()) {
constraints.getDefaultConstraints().forEach(constraint -> constraint.setCatName(catName));
}
if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints()) && !constraints.getCheckConstraints().get(0).isSetCatName()) {
constraints.getCheckConstraints().forEach(constraint -> constraint.setCatName(catName));
}
// Set constraint name if null before sending to listener
constraints = ms.createTableWithConstraints(tbl, constraints);
}
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses = MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.CREATE_TABLE, new CreateTableEvent(tbl, true, this, isReplicated), envContext);
if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys())) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_PRIMARYKEY,
new AddPrimaryKeyEvent(constraints.getPrimaryKeys(), true, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getForeignKeys())) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_FOREIGNKEY,
new AddForeignKeyEvent(constraints.getForeignKeys(), true, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints())) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_UNIQUECONSTRAINT,
new AddUniqueConstraintEvent(constraints.getUniqueConstraints(), true, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints())) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_NOTNULLCONSTRAINT,
new AddNotNullConstraintEvent(constraints.getNotNullConstraints(), true, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints())) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_CHECKCONSTRAINT,
new AddCheckConstraintEvent(constraints.getCheckConstraints(), true, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints())) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_DEFAULTCONSTRAINT,
new AddDefaultConstraintEvent(constraints.getDefaultConstraints(), true, this), envContext);
}
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
if (madeDir) {
wh.deleteDir(tblPath, true, false, ReplChangeManager.shouldEnableCm(db, tbl));
}
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.CREATE_TABLE,
new CreateTableEvent(tbl, success, this, isReplicated), envContext,
transactionalListenerResponses, ms);
if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys())) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PRIMARYKEY,
new AddPrimaryKeyEvent(constraints.getPrimaryKeys(), success, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getForeignKeys())) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_FOREIGNKEY,
new AddForeignKeyEvent(constraints.getForeignKeys(), success, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints())) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_UNIQUECONSTRAINT,
new AddUniqueConstraintEvent(constraints.getUniqueConstraints(), success, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints())) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_NOTNULLCONSTRAINT,
new AddNotNullConstraintEvent(constraints.getNotNullConstraints(), success, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints())) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_DEFAULTCONSTRAINT,
new AddDefaultConstraintEvent(constraints.getDefaultConstraints(), success, this), envContext);
}
if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints())) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_CHECKCONSTRAINT,
new AddCheckConstraintEvent(constraints.getCheckConstraints(), success, this), envContext);
}
}
}
// If the table has column statistics, update it into the metastore. We need a valid
// writeId list to update column statistics for a transactional table. But during bootstrap
// replication, where we use this feature, we do not have a valid writeId list which was
// used to update the stats. But we know for sure that the writeId associated with the
// stats was valid then (otherwise stats update would have failed on the source). So, craft
// a valid transaction list with only that writeId and use it to update the stats.
if (colStats != null) {
long writeId = tbl.getWriteId();
String validWriteIds = null;
if (writeId > 0) {
ValidWriteIdList validWriteIdList =
new ValidReaderWriteIdList(TableName.getDbTable(tbl.getDbName(),
tbl.getTableName()),
new long[0], new BitSet(), writeId);
validWriteIds = validWriteIdList.toString();
}
updateTableColumnStatsInternal(colStats, validWriteIds, tbl.getWriteId());
}
}
private String getTableSuffix(Table tbl) {
return tbl.isSetTxnId() && tbl.getParameters() != null
&& Boolean.parseBoolean(tbl.getParameters().get(SOFT_DELETE_TABLE)) ?
SOFT_DELETE_PATH_SUFFIX + String.format(DELTA_DIGITS, tbl.getTxnId()) : "";
}
@Override
public void create_table(final Table tbl) throws AlreadyExistsException,
MetaException, InvalidObjectException, InvalidInputException {
create_table_with_environment_context(tbl, null);
}
@Override
public void create_table_with_environment_context(final Table tbl,
final EnvironmentContext envContext)
throws AlreadyExistsException, MetaException, InvalidObjectException,
InvalidInputException {
startFunction("create_table", ": " + tbl.toString());
boolean success = false;
Exception ex = null;
try {
create_table_core(getMS(), tbl, envContext);
success = true;
} catch (Exception e) {
LOG.warn("create_table_with_environment_context got ", e);
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
.throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
.convertIfInstance(NoSuchObjectException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
endFunction("create_table", success, ex, tbl.getTableName());
}
}
@Override
public void create_table_req(final CreateTableRequest req)
throws AlreadyExistsException, MetaException, InvalidObjectException,
InvalidInputException {
Table tbl = req.getTable();
startFunction("create_table_req", ": " + tbl.toString());
boolean success = false;
Exception ex = null;
try {
create_table_core(getMS(), req);
success = true;
} catch (Exception e) {
LOG.warn("create_table_req got ", e);
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
.throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
.convertIfInstance(NoSuchObjectException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
endFunction("create_table_req", success, ex, tbl.getTableName());
}
}
@Override
public void create_table_with_constraints(final Table tbl,
final List<SQLPrimaryKey> primaryKeys, final List<SQLForeignKey> foreignKeys,
List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints,
List<SQLDefaultConstraint> defaultConstraints,
List<SQLCheckConstraint> checkConstraints)
throws AlreadyExistsException, MetaException, InvalidObjectException,
InvalidInputException {
startFunction("create_table", ": " + tbl.toString());
boolean success = false;
Exception ex = null;
try {
CreateTableRequest req = new CreateTableRequest(tbl);
req.setPrimaryKeys(primaryKeys);
req.setForeignKeys(foreignKeys);
req.setUniqueConstraints(uniqueConstraints);
req.setNotNullConstraints(notNullConstraints);
req.setDefaultConstraints(defaultConstraints);
req.setCheckConstraints(checkConstraints);
create_table_req(req);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
.throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
.defaultMetaException();
} finally {
endFunction("create_table_with_constraints", success, ex, tbl.getTableName());
}
}
@Override
public void drop_constraint(DropConstraintRequest req)
throws MetaException, InvalidObjectException {
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
String dbName = req.getDbname();
String tableName = req.getTablename();
String constraintName = req.getConstraintname();
startFunction("drop_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
try {
ms.openTransaction();
ms.dropConstraint(catName, dbName, tableName, constraintName);
if (transactionalListeners.size() > 0) {
DropConstraintEvent dropConstraintEvent = new DropConstraintEvent(catName, dbName,
tableName, constraintName, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onDropConstraint(dropConstraintEvent);
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class)
.convertIfInstance(NoSuchObjectException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else {
for (MetaStoreEventListener listener : listeners) {
DropConstraintEvent dropConstraintEvent = new DropConstraintEvent(catName, dbName,
tableName, constraintName, true, this);
listener.onDropConstraint(dropConstraintEvent);
}
}
endFunction("drop_constraint", success, ex, constraintName);
}
}
@Override
public void add_primary_key(AddPrimaryKeyRequest req)
throws MetaException, InvalidObjectException {
List<SQLPrimaryKey> primaryKeyCols = req.getPrimaryKeyCols();
String constraintName = (CollectionUtils.isNotEmpty(primaryKeyCols)) ?
primaryKeyCols.get(0).getPk_name() : "null";
startFunction("add_primary_key", ": " + constraintName);
boolean success = false;
Exception ex = null;
if (CollectionUtils.isNotEmpty(primaryKeyCols) && !primaryKeyCols.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
primaryKeyCols.forEach(pk -> pk.setCatName(defaultCat));
}
RawStore ms = getMS();
try {
ms.openTransaction();
List<SQLPrimaryKey> primaryKeys = ms.addPrimaryKeys(primaryKeyCols);
if (transactionalListeners.size() > 0) {
if (CollectionUtils.isNotEmpty(primaryKeys)) {
AddPrimaryKeyEvent addPrimaryKeyEvent = new AddPrimaryKeyEvent(primaryKeys, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAddPrimaryKey(addPrimaryKeyEvent);
}
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (primaryKeyCols != null && primaryKeyCols.size() > 0) {
for (MetaStoreEventListener listener : listeners) {
AddPrimaryKeyEvent addPrimaryKeyEvent = new AddPrimaryKeyEvent(primaryKeyCols, true, this);
listener.onAddPrimaryKey(addPrimaryKeyEvent);
}
}
endFunction("add_primary_key", success, ex, constraintName);
}
}
@Override
public void add_foreign_key(AddForeignKeyRequest req)
throws MetaException, InvalidObjectException {
List<SQLForeignKey> foreignKeys = req.getForeignKeyCols();
String constraintName = CollectionUtils.isNotEmpty(foreignKeys) ?
foreignKeys.get(0).getFk_name() : "null";
startFunction("add_foreign_key", ": " + constraintName);
boolean success = false;
Exception ex = null;
if (CollectionUtils.isNotEmpty(foreignKeys) && !foreignKeys.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
foreignKeys.forEach(pk -> pk.setCatName(defaultCat));
}
RawStore ms = getMS();
try {
ms.openTransaction();
foreignKeys = ms.addForeignKeys(foreignKeys);
if (transactionalListeners.size() > 0) {
if (CollectionUtils.isNotEmpty(foreignKeys)) {
AddForeignKeyEvent addForeignKeyEvent = new AddForeignKeyEvent(foreignKeys, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAddForeignKey(addForeignKeyEvent);
}
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (CollectionUtils.isNotEmpty(foreignKeys)) {
for (MetaStoreEventListener listener : listeners) {
AddForeignKeyEvent addForeignKeyEvent = new AddForeignKeyEvent(foreignKeys, true, this);
listener.onAddForeignKey(addForeignKeyEvent);
}
}
endFunction("add_foreign_key", success, ex, constraintName);
}
}
@Override
public void add_unique_constraint(AddUniqueConstraintRequest req)
throws MetaException, InvalidObjectException {
List<SQLUniqueConstraint> uniqueConstraints = req.getUniqueConstraintCols();
String constraintName = (uniqueConstraints != null && uniqueConstraints.size() > 0) ?
uniqueConstraints.get(0).getUk_name() : "null";
startFunction("add_unique_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
if (!uniqueConstraints.isEmpty() && !uniqueConstraints.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
uniqueConstraints.forEach(pk -> pk.setCatName(defaultCat));
}
RawStore ms = getMS();
try {
ms.openTransaction();
uniqueConstraints = ms.addUniqueConstraints(uniqueConstraints);
if (transactionalListeners.size() > 0) {
if (CollectionUtils.isNotEmpty(uniqueConstraints)) {
AddUniqueConstraintEvent addUniqueConstraintEvent = new AddUniqueConstraintEvent(uniqueConstraints, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAddUniqueConstraint(addUniqueConstraintEvent);
}
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (CollectionUtils.isNotEmpty(uniqueConstraints)) {
for (MetaStoreEventListener listener : listeners) {
AddUniqueConstraintEvent addUniqueConstraintEvent = new AddUniqueConstraintEvent(uniqueConstraints, true, this);
listener.onAddUniqueConstraint(addUniqueConstraintEvent);
}
}
endFunction("add_unique_constraint", success, ex, constraintName);
}
}
@Override
public void add_not_null_constraint(AddNotNullConstraintRequest req)
throws MetaException, InvalidObjectException {
List<SQLNotNullConstraint> notNullConstraints = req.getNotNullConstraintCols();
String constraintName = (notNullConstraints != null && notNullConstraints.size() > 0) ?
notNullConstraints.get(0).getNn_name() : "null";
startFunction("add_not_null_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
if (!notNullConstraints.isEmpty() && !notNullConstraints.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
notNullConstraints.forEach(pk -> pk.setCatName(defaultCat));
}
RawStore ms = getMS();
try {
ms.openTransaction();
notNullConstraints = ms.addNotNullConstraints(notNullConstraints);
if (transactionalListeners.size() > 0) {
if (CollectionUtils.isNotEmpty(notNullConstraints)) {
AddNotNullConstraintEvent addNotNullConstraintEvent = new AddNotNullConstraintEvent(notNullConstraints, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAddNotNullConstraint(addNotNullConstraintEvent);
}
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class).defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (CollectionUtils.isNotEmpty(notNullConstraints)) {
for (MetaStoreEventListener listener : listeners) {
AddNotNullConstraintEvent addNotNullConstraintEvent = new AddNotNullConstraintEvent(notNullConstraints, true, this);
listener.onAddNotNullConstraint(addNotNullConstraintEvent);
}
}
endFunction("add_not_null_constraint", success, ex, constraintName);
}
}
@Override
public void add_default_constraint(AddDefaultConstraintRequest req) throws MetaException, InvalidObjectException {
List<SQLDefaultConstraint> defaultConstraints = req.getDefaultConstraintCols();
String constraintName =
CollectionUtils.isNotEmpty(defaultConstraints) ? defaultConstraints.get(0).getDc_name() : "null";
startFunction("add_default_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
if (!defaultConstraints.isEmpty() && !defaultConstraints.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
defaultConstraints.forEach(pk -> pk.setCatName(defaultCat));
}
RawStore ms = getMS();
try {
ms.openTransaction();
defaultConstraints = ms.addDefaultConstraints(defaultConstraints);
if (transactionalListeners.size() > 0) {
if (CollectionUtils.isNotEmpty(defaultConstraints)) {
AddDefaultConstraintEvent addDefaultConstraintEvent =
new AddDefaultConstraintEvent(defaultConstraints, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAddDefaultConstraint(addDefaultConstraintEvent);
}
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class).defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (CollectionUtils.isNotEmpty(defaultConstraints)) {
for (MetaStoreEventListener listener : listeners) {
AddDefaultConstraintEvent addDefaultConstraintEvent =
new AddDefaultConstraintEvent(defaultConstraints, true, this);
listener.onAddDefaultConstraint(addDefaultConstraintEvent);
}
}
endFunction("add_default_constraint", success, ex, constraintName);
}
}
@Override
public void add_check_constraint(AddCheckConstraintRequest req)
throws MetaException, InvalidObjectException {
List<SQLCheckConstraint> checkConstraints= req.getCheckConstraintCols();
String constraintName = CollectionUtils.isNotEmpty(checkConstraints) ?
checkConstraints.get(0).getDc_name() : "null";
startFunction("add_check_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
if (!checkConstraints.isEmpty() && !checkConstraints.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
checkConstraints.forEach(pk -> pk.setCatName(defaultCat));
}
RawStore ms = getMS();
try {
ms.openTransaction();
checkConstraints = ms.addCheckConstraints(checkConstraints);
if (transactionalListeners.size() > 0) {
if (CollectionUtils.isNotEmpty(checkConstraints)) {
AddCheckConstraintEvent addcheckConstraintEvent = new AddCheckConstraintEvent(checkConstraints, true, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAddCheckConstraint(addcheckConstraintEvent);
}
}
}
success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class).defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (CollectionUtils.isNotEmpty(checkConstraints)) {
for (MetaStoreEventListener listener : listeners) {
AddCheckConstraintEvent addCheckConstraintEvent = new AddCheckConstraintEvent(checkConstraints, true, this);
listener.onAddCheckConstraint(addCheckConstraintEvent);
}
}
endFunction("add_check_constraint", success, ex, constraintName);
}
}
private boolean is_table_exists(RawStore ms, String catName, String dbname, String name)
throws MetaException {
return (ms.getTable(catName, dbname, name, null) != null);
}
private boolean drop_table_core(final RawStore ms, final String catName, final String dbname,
final String name, final boolean deleteData,
final EnvironmentContext envContext, final String indexName, boolean dropPartitions)
throws TException, IOException {
boolean success = false;
boolean tableDataShouldBeDeleted = false;
Path tblPath = null;
List<Path> partPaths = null;
Table tbl = null;
boolean ifPurge = false;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
Database db = null;
boolean isReplicated = false;
try {
ms.openTransaction();
// HIVE-25282: Drop/Alter table in REMOTE db should fail
db = ms.getDatabase(catName, dbname);
if (db.getType() == DatabaseType.REMOTE) {
throw new MetaException("Drop table in REMOTE database " + db.getName() + " is not allowed");
}
isReplicated = isDbReplicationTarget(db);
// drop any partitions
GetTableRequest req = new GetTableRequest(dbname,name);
req.setCatName(catName);
tbl = get_table_core(req);
if (tbl == null) {
throw new NoSuchObjectException(name + " doesn't exist");
}
// Check if table is part of a materialized view.
// If it is, it cannot be dropped.
List<String> isPartOfMV = ms.isPartOfMaterializedView(catName, dbname, name);
if (!isPartOfMV.isEmpty()) {
throw new MetaException(String.format("Cannot drop table as it is used in the following materialized" +
" views %s%n", isPartOfMV));
}
if (tbl.getSd() == null) {
throw new MetaException("Table metadata is corrupted");
}
ifPurge = isMustPurge(envContext, tbl);
firePreEvent(new PreDropTableEvent(tbl, deleteData, this));
tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(tbl, deleteData);
if (tbl.getSd().getLocation() != null) {
tblPath = new Path(tbl.getSd().getLocation());
if (!wh.isWritable(tblPath.getParent())) {
String target = indexName == null ? "Table" : "Index table";
throw new MetaException(target + " metadata not deleted since " +
tblPath.getParent() + " is not writable by " +
SecurityUtils.getUser());
}
}
// Drop the partitions and get a list of locations which need to be deleted
// In case of drop database cascade we need not to drop the partitions, they are already dropped.
if (dropPartitions) {
partPaths = dropPartitionsAndGetLocations(ms, catName, dbname, name, tblPath, tableDataShouldBeDeleted);
}
// Drop any constraints on the table
ms.dropConstraint(catName, dbname, name, null, true);
if (!ms.dropTable(catName, dbname, name)) {
String tableName = TableName.getQualified(catName, dbname, name);
throw new MetaException(indexName == null ? "Unable to drop table " + tableName:
"Unable to drop index table " + tableName + " for index " + indexName);
} else {
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_TABLE,
new DropTableEvent(tbl, true, deleteData,
this, isReplicated),
envContext);
}
success = ms.commitTransaction();
}
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (tableDataShouldBeDeleted) {
// Data needs deletion. Check if trash may be skipped.
// Delete the data in the partitions which have other locations
deletePartitionData(partPaths, ifPurge, ReplChangeManager.shouldEnableCm(db, tbl));
// Delete the data in the table
deleteTableData(tblPath, ifPurge, ReplChangeManager.shouldEnableCm(db, tbl));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_TABLE,
new DropTableEvent(tbl, success, deleteData, this, isReplicated),
envContext,
transactionalListenerResponses, ms);
}
}
return success;
}
private boolean checkTableDataShouldBeDeleted(Table tbl, boolean deleteData) {
if (deleteData && isExternal(tbl)) {
// External table data can be deleted if EXTERNAL_TABLE_PURGE is true
return isExternalTablePurge(tbl);
}
return deleteData;
}
/**
* Deletes the data in a table's location, if it fails logs an error
*
* @param tablePath
* @param ifPurge completely purge the table (skipping trash) while removing
* data from warehouse
* @param shouldEnableCm If cm should be enabled
*/
private void deleteTableData(Path tablePath, boolean ifPurge, boolean shouldEnableCm) {
if (tablePath != null) {
deleteDataExcludeCmroot(tablePath, ifPurge, shouldEnableCm);
}
}
/**
* Deletes the data in a table's location, if it fails logs an error.
*
* @param tablePath
* @param ifPurge completely purge the table (skipping trash) while removing
* data from warehouse
* @param db Database
*/
private void deleteTableData(Path tablePath, boolean ifPurge, Database db) {
if (tablePath != null) {
try {
wh.deleteDir(tablePath, true, ifPurge, db);
} catch (Exception e) {
LOG.error("Failed to delete table directory: " + tablePath +
" " + e.getMessage());
}
}
}
/**
* Give a list of partitions' locations, tries to delete each one
* and for each that fails logs an error.
*
* @param partPaths
* @param ifPurge completely purge the partition (skipping trash) while
* removing data from warehouse
* @param shouldEnableCm If cm should be enabled
*/
private void deletePartitionData(List<Path> partPaths, boolean ifPurge, boolean shouldEnableCm) {
if (partPaths != null && !partPaths.isEmpty()) {
for (Path partPath : partPaths) {
deleteDataExcludeCmroot(partPath, ifPurge, shouldEnableCm);
}
}
}
/**
* Give a list of partitions' locations, tries to delete each one
* and for each that fails logs an error.
*
* @param partPaths
* @param ifPurge completely purge the partition (skipping trash) while
* removing data from warehouse
* @param db Database
*/
private void deletePartitionData(List<Path> partPaths, boolean ifPurge, Database db) {
if (partPaths != null && !partPaths.isEmpty()) {
for (Path partPath : partPaths) {
try {
wh.deleteDir(partPath, true, ifPurge, db);
} catch (Exception e) {
LOG.error("Failed to delete partition directory: " + partPath +
" " + e.getMessage());
}
}
}
}
/**
* Delete data from path excluding cmdir
* and for each that fails logs an error.
*
* @param path
* @param ifPurge completely purge the partition (skipping trash) while
* removing data from warehouse
* @param shouldEnableCm If cm should be enabled
*/
private void deleteDataExcludeCmroot(Path path, boolean ifPurge, boolean shouldEnableCm) {
try {
if (shouldEnableCm) {
//Don't delete cmdir if its inside the partition path
FileStatus[] statuses = path.getFileSystem(conf).listStatus(path,
ReplChangeManager.CMROOT_PATH_FILTER);
for (final FileStatus status : statuses) {
wh.deleteDir(status.getPath(), true, ifPurge, shouldEnableCm);
}
//Check if table directory is empty, delete it
FileStatus[] statusWithoutFilter = path.getFileSystem(conf).listStatus(path);
if (statusWithoutFilter.length == 0) {
wh.deleteDir(path, true, ifPurge, shouldEnableCm);
}
} else {
//If no cm delete the complete table directory
wh.deleteDir(path, true, ifPurge, shouldEnableCm);
}
} catch (Exception e) {
LOG.error("Failed to delete directory: " + path +
" " + e.getMessage());
}
}
/**
* Deletes the partitions specified by catName, dbName, tableName. If checkLocation is true, for
* locations of partitions which may not be subdirectories of tablePath checks to make sure the
* locations are writable.
*
* Drops the metadata for each partition.
*
* Provides a list of locations of partitions which may not be subdirectories of tablePath.
*
* @param ms RawStore to use for metadata retrieval and delete
* @param catName The catName
* @param dbName The dbName
* @param tableName The tableName
* @param tablePath The tablePath of which subdirectories does not have to be checked
* @param checkLocation Should we check the locations at all
* @return The list of the Path objects to delete (only in case checkLocation is true)
* @throws MetaException
* @throws IOException
* @throws NoSuchObjectException
*/
private List<Path> dropPartitionsAndGetLocations(RawStore ms, String catName, String dbName,
String tableName, Path tablePath, boolean checkLocation)
throws MetaException, IOException, NoSuchObjectException {
int batchSize = MetastoreConf.getIntVar(conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
String tableDnsPath = null;
if (tablePath != null) {
tableDnsPath = wh.getDnsPath(tablePath).toString();
}
List<Path> partPaths = new ArrayList<>();
while (true) {
List<String> partNames;
if (checkLocation) {
Map<String, String> partitionLocations = ms.getPartitionLocations(catName, dbName, tableName,
tableDnsPath, batchSize);
partNames = new ArrayList<>(partitionLocations.keySet());
for (String partName : partNames) {
String pathString = partitionLocations.get(partName);
if (pathString != null) {
Path partPath = wh.getDnsPath(new Path(pathString));
// Double check here. Maybe Warehouse.getDnsPath revealed relationship between the
// path objects
if (tableDnsPath == null ||
!FileUtils.isSubdirectory(tableDnsPath, partPath.toString())) {
if (!wh.isWritable(partPath.getParent())) {
throw new MetaException("Table metadata not deleted since the partition "
+ partName + " has parent location " + partPath.getParent()
+ " which is not writable by " + SecurityUtils.getUser());
}
partPaths.add(partPath);
}
}
}
} else {
partNames = ms.listPartitionNames(catName, dbName, tableName, (short) batchSize);
}
if (partNames == null || partNames.isEmpty()) {
// No more partitions left to drop. Return with the collected path list to delete.
return partPaths;
}
for (MetaStoreEventListener listener : listeners) {
//No drop part listener events fired for public listeners historically, for drop table case.
//Limiting to internal listeners for now, to avoid unexpected calls for public listeners.
if (listener instanceof HMSMetricsListener) {
for (@SuppressWarnings("unused") String partName : partNames) {
listener.onDropPartition(null);
}
}
}
ms.dropPartitions(catName, dbName, tableName, partNames);
}
}
@Override
public void drop_table(final String dbname, final String name, final boolean deleteData)
throws NoSuchObjectException, MetaException {
drop_table_with_environment_context(dbname, name, deleteData, null);
}
@Override
public void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData,
final EnvironmentContext envContext) throws NoSuchObjectException, MetaException {
drop_table_with_environment_context(dbname, name, deleteData, envContext, true);
}
private void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData,
final EnvironmentContext envContext, boolean dropPartitions) throws MetaException, NoSuchObjectException {
String[] parsedDbName = parseDbName(dbname, conf);
startTableFunction("drop_table", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name);
boolean success = false;
Exception ex = null;
try {
success =
drop_table_core(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, deleteData, envContext, null, dropPartitions);
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
.convertIfInstance(IOException.class, MetaException.class).defaultMetaException();
} finally {
endFunction("drop_table", success, ex, name);
}
}
private void updateStatsForTruncate(Map<String,String> props, EnvironmentContext environmentContext) {
if (null == props) {
return;
}
for (String stat : StatsSetupConst.SUPPORTED_STATS) {
String statVal = props.get(stat);
if (statVal != null) {
//In the case of truncate table, we set the stats to be 0.
props.put(stat, "0");
}
}
//first set basic stats to true
StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE);
environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK);
environmentContext.putToProperties(StatsSetupConst.DO_NOT_POPULATE_QUICK_STATS, StatsSetupConst.TRUE);
//then invalidate column stats
StatsSetupConst.clearColumnStatsState(props);
return;
}
private void alterPartitionForTruncate(RawStore ms, String catName, String dbName, String tableName,
Table table, Partition partition, String validWriteIds, long writeId) throws Exception {
EnvironmentContext environmentContext = new EnvironmentContext();
updateStatsForTruncate(partition.getParameters(), environmentContext);
if (!transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_PARTITION,
new AlterPartitionEvent(partition, partition, table, true, true,
writeId, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_PARTITION,
new AlterPartitionEvent(partition, partition, table, true, true,
writeId, this));
}
if (writeId > 0) {
partition.setWriteId(writeId);
}
alterHandler.alterPartition(ms, wh, catName, dbName, tableName, null, partition,
environmentContext, this, validWriteIds);
}
private void alterTableStatsForTruncate(RawStore ms, String catName, String dbName,
String tableName, Table table, List<String> partNames,
String validWriteIds, long writeId) throws Exception {
if (partNames == null) {
if (0 != table.getPartitionKeysSize()) {
for (Partition partition : ms.getPartitions(catName, dbName, tableName, -1)) {
alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition,
validWriteIds, writeId);
}
} else {
EnvironmentContext environmentContext = new EnvironmentContext();
updateStatsForTruncate(table.getParameters(), environmentContext);
boolean isReplicated = isDbReplicationTarget(ms.getDatabase(catName, dbName));
if (!transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_TABLE,
new AlterTableEvent(table, table, true, true,
writeId, this, isReplicated));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_TABLE,
new AlterTableEvent(table, table, true, true,
writeId, this, isReplicated));
}
// TODO: this should actually pass thru and set writeId for txn stats.
if (writeId > 0) {
table.setWriteId(writeId);
}
alterHandler.alterTable(ms, wh, catName, dbName, tableName, table,
environmentContext, this, validWriteIds);
}
} else {
for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) {
alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition,
validWriteIds, writeId);
}
}
return;
}
private List<Path> getLocationsForTruncate(final RawStore ms,
final String catName,
final String dbName,
final String tableName,
final Table table,
final List<String> partNames) throws Exception {
List<Path> locations = new ArrayList<>();
if (partNames == null) {
if (0 != table.getPartitionKeysSize()) {
for (Partition partition : ms.getPartitions(catName, dbName, tableName, -1)) {
locations.add(new Path(partition.getSd().getLocation()));
}
} else {
locations.add(new Path(table.getSd().getLocation()));
}
} else {
for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) {
locations.add(new Path(partition.getSd().getLocation()));
}
}
return locations;
}
@Override
public CmRecycleResponse cm_recycle(final CmRecycleRequest request) throws MetaException {
wh.recycleDirToCmPath(new Path(request.getDataPath()), request.isPurge());
return new CmRecycleResponse();
}
@Override
public void truncate_table(final String dbName, final String tableName, List<String> partNames)
throws NoSuchObjectException, MetaException {
// Deprecated path, won't work for txn tables.
truncateTableInternal(dbName, tableName, partNames, null, -1, null);
}
@Override
public TruncateTableResponse truncate_table_req(TruncateTableRequest req)
throws MetaException, TException {
truncateTableInternal(req.getDbName(), req.getTableName(), req.getPartNames(),
req.getValidWriteIdList(), req.getWriteId(), req.getEnvironmentContext());
return new TruncateTableResponse();
}
private void truncateTableInternal(String dbName, String tableName, List<String> partNames,
String validWriteIds, long writeId, EnvironmentContext context) throws MetaException, NoSuchObjectException {
boolean isSkipTrash = false, needCmRecycle = false;
try {
String[] parsedDbName = parseDbName(dbName, conf);
Table tbl = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
boolean skipDataDeletion = Optional.ofNullable(context)
.map(EnvironmentContext::getProperties)
.map(prop -> prop.get(TRUNCATE_SKIP_DATA_DELETION))
.map(Boolean::parseBoolean)
.orElse(false);
if (TxnUtils.isTransactionalTable(tbl) || !skipDataDeletion) {
if (!skipDataDeletion) {
isSkipTrash = MetaStoreUtils.isSkipTrash(tbl.getParameters());
Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
needCmRecycle = ReplChangeManager.shouldEnableCm(db, tbl);
}
// This is not transactional
for (Path location : getLocationsForTruncate(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
tbl, partNames)) {
if (!skipDataDeletion) {
truncateDataFiles(location, isSkipTrash, needCmRecycle);
} else {
// For Acid tables we don't need to delete the old files, only write an empty baseDir.
// Compaction and cleaner will take care of the rest
addTruncateBaseFile(location, writeId, conf, DataFormat.TRUNCATED);
}
}
}
// Alter the table/partition stats and also notify truncate table event
alterTableStatsForTruncate(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName, tbl, partNames, validWriteIds, writeId);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
.convertIfInstance(IOException.class, MetaException.class)
.defaultMetaException();
}
}
/**
* Add an empty baseDir with a truncate metadatafile
* @param location partition or table directory
* @param writeId allocated writeId
* @throws MetaException
*/
static void addTruncateBaseFile(Path location, long writeId, Configuration conf, DataFormat dataFormat)
throws MetaException {
if (location == null)
return;
Path basePath = new Path(location, AcidConstants.baseDir(writeId));
try {
FileSystem fs = location.getFileSystem(conf);
fs.mkdirs(basePath);
// We can not leave the folder empty, otherwise it will be skipped at some file listing in AcidUtils
// No need for a data file, a simple metadata is enough
AcidMetaDataFile.writeToFile(fs, basePath, dataFormat);
} catch (Exception e) {
throw newMetaException(e);
}
}
private void truncateDataFiles(Path location, boolean isSkipTrash, boolean needCmRecycle)
throws IOException, MetaException {
FileSystem fs = location.getFileSystem(getConf());
if (!HdfsUtils.isPathEncrypted(getConf(), fs.getUri(), location) &&
!FileUtils.pathHasSnapshotSubDir(location, fs)) {
HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(getConf(), fs, location);
FileStatus targetStatus = fs.getFileStatus(location);
String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
wh.deleteDir(location, true, isSkipTrash, needCmRecycle);
fs.mkdirs(location);
HdfsUtils.setFullFileStatus(getConf(), status, targetGroup, fs, location, false);
} else {
FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER);
if (statuses == null || statuses.length == 0) {
return;
}
for (final FileStatus status : statuses) {
wh.deleteDir(status.getPath(), true, isSkipTrash, needCmRecycle);
}
}
}
/**
* Is this an external table?
*
* @param table
* Check if this table is external.
* @return True if the table is external, otherwise false.
*/
private boolean isExternal(Table table) {
return MetaStoreUtils.isExternalTable(table);
}
private boolean isExternalTablePurge(Table table) {
return MetaStoreUtils.isExternalTablePurge(table);
}
@Override
@Deprecated
public Table get_table(final String dbname, final String name) throws MetaException,
NoSuchObjectException {
String[] parsedDbName = parseDbName(dbname, conf);
GetTableRequest getTableRequest = new GetTableRequest(parsedDbName[DB_NAME],name);
getTableRequest.setCatName(parsedDbName[CAT_NAME]);
return getTableInternal(getTableRequest);
}
@Override
public List<ExtendedTableInfo> get_tables_ext(final GetTablesExtRequest req) throws MetaException {
List<String> tables = new ArrayList<String>();
List<ExtendedTableInfo> ret = new ArrayList<ExtendedTableInfo>();
String catalog = req.getCatalog();
String database = req.getDatabase();
String pattern = req.getTableNamePattern();
List<String> processorCapabilities = req.getProcessorCapabilities();
int limit = req.getLimit();
String processorId = req.getProcessorIdentifier();
List<Table> tObjects = new ArrayList<>();
startTableFunction("get_tables_ext", catalog, database, pattern);
Exception ex = null;
try {
tables = getMS().getTables(catalog, database, pattern, null, limit);
LOG.debug("get_tables_ext:getTables() returned " + tables.size());
tables = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook,
catalog, database, tables);
if (tables.size() > 0) {
tObjects = getMS().getTableObjectsByName(catalog, database, tables);
LOG.debug("get_tables_ext:getTableObjectsByName() returned " + tObjects.size());
if (processorCapabilities == null || processorCapabilities.size() == 0 ||
processorCapabilities.contains("MANAGERAWMETADATA")) {
LOG.info("Skipping translation for processor with " + processorId);
} else {
if (transformer != null) {
Map<Table, List<String>> retMap = transformer.transform(tObjects, processorCapabilities, processorId);
for (Map.Entry<Table, List<String>> entry : retMap.entrySet()) {
LOG.debug("Table " + entry.getKey().getTableName() + " requires " + Arrays.toString((entry.getValue()).toArray()));
ret.add(convertTableToExtendedTable(entry.getKey(), entry.getValue(), req.getRequestedFields()));
}
} else {
for (Table table : tObjects) {
ret.add(convertTableToExtendedTable(table, processorCapabilities, req.getRequestedFields()));
}
}
}
}
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_tables_ext", ret != null, ex);
}
return ret;
}
private ExtendedTableInfo convertTableToExtendedTable (Table table,
List<String> processorCapabilities, int mask) {
ExtendedTableInfo extTable = new ExtendedTableInfo(table.getTableName());
if ((mask & GetTablesExtRequestFields.ACCESS_TYPE.getValue()) == GetTablesExtRequestFields.ACCESS_TYPE.getValue()) {
extTable.setAccessType(table.getAccessType());
}
if ((mask & GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue())
== GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) {
extTable.setRequiredReadCapabilities(table.getRequiredReadCapabilities());
extTable.setRequiredWriteCapabilities(table.getRequiredWriteCapabilities());
}
return extTable;
}
@Override
public GetTableResult get_table_req(GetTableRequest req) throws MetaException,
NoSuchObjectException {
req.setCatName(req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf));
return new GetTableResult(getTableInternal(req));
}
/**
* This function retrieves table from metastore. If getColumnStats flag is true,
* then engine should be specified so the table is retrieve with the column stats
* for that engine.
*/
private Table getTableInternal(GetTableRequest getTableRequest) throws MetaException, NoSuchObjectException {
Preconditions.checkArgument(!getTableRequest.isGetColumnStats() || getTableRequest.getEngine() != null,
"To retrieve column statistics with a table, engine parameter cannot be null");
if (isInTest) {
assertClientHasCapability(getTableRequest.getCapabilities(), ClientCapability.TEST_CAPABILITY, "Hive tests",
"get_table_req");
}
Table t = null;
startTableFunction("get_table", getTableRequest.getCatName(), getTableRequest.getDbName(),
getTableRequest.getTblName());
Exception ex = null;
try {
t = get_table_core(getTableRequest);
if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
assertClientHasCapability(getTableRequest.getCapabilities(), ClientCapability.INSERT_ONLY_TABLES,
"insert-only tables", "get_table_req");
}
if (CollectionUtils.isEmpty(getTableRequest.getProcessorCapabilities()) || getTableRequest
.getProcessorCapabilities().contains("MANAGERAWMETADATA")) {
LOG.info("Skipping translation for processor with " + getTableRequest.getProcessorIdentifier());
} else {
if (transformer != null) {
List<Table> tList = new ArrayList<>();
tList.add(t);
Map<Table, List<String>> ret = transformer
.transform(tList, getTableRequest.getProcessorCapabilities(), getTableRequest.getProcessorIdentifier());
if (ret.size() > 1) {
LOG.warn("Unexpected resultset size:" + ret.size());
throw new MetaException("Unexpected result from metadata transformer:return list size is " + ret.size());
}
t = ret.keySet().iterator().next();
}
}
firePreEvent(new PreReadTableEvent(t, this));
} catch (MetaException | NoSuchObjectException e) {
ex = e;
throw e;
} finally {
endFunction("get_table", t != null, ex, getTableRequest.getTblName());
}
return t;
}
@Override
public List<TableMeta> get_table_meta(String dbnames, String tblNames, List<String> tblTypes)
throws MetaException, NoSuchObjectException {
List<TableMeta> t = null;
String[] parsedDbName = parseDbName(dbnames, conf);
startTableFunction("get_table_metas", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblNames);
Exception ex = null;
try {
t = getMS().getTableMeta(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblNames, tblTypes);
t = FilterUtils.filterTableMetasIfEnabled(isServerFilterEnabled, filterHook, t);
t = filterReadableTables(parsedDbName[CAT_NAME], t);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_table_metas", t != null, ex);
}
return t;
}
@Override
@Deprecated
public Table get_table_core(
final String catName,
final String dbname,
final String name)
throws MetaException, NoSuchObjectException {
GetTableRequest getTableRequest = new GetTableRequest(dbname,name);
getTableRequest.setCatName(catName);
return get_table_core(getTableRequest);
}
@Override
@Deprecated
public Table get_table_core(
final String catName,
final String dbname,
final String name,
final String writeIdList)
throws MetaException, NoSuchObjectException {
GetTableRequest getTableRequest = new GetTableRequest(dbname,name);
getTableRequest.setCatName(catName);
getTableRequest.setValidWriteIdList(writeIdList);
return get_table_core(getTableRequest);
}
/**
* This function retrieves table from metastore. If getColumnStats flag is true,
* then engine should be specified so the table is retrieve with the column stats
* for that engine.
*/
@Override
public Table get_table_core(GetTableRequest getTableRequest) throws MetaException, NoSuchObjectException {
Preconditions.checkArgument(!getTableRequest.isGetColumnStats() || getTableRequest.getEngine() != null,
"To retrieve column statistics with a table, engine parameter cannot be null");
String catName = getTableRequest.getCatName();
String dbName = getTableRequest.getDbName();
String tblName = getTableRequest.getTblName();
Database db = null;
Table t = null;
try {
db = get_database_core(catName, dbName);
} catch (Exception e) { /* appears exception is not thrown currently if db doesnt exist */ }
if (db != null) {
if (db.getType().equals(DatabaseType.REMOTE)) {
t = DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName);
if (t == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + " table not found");
}
t.setDbName(dbName);
return t;
}
}
try {
t = getMS().getTable(catName, dbName, tblName, getTableRequest.getValidWriteIdList(), getTableRequest.getId());
if (t == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + " table not found");
}
// If column statistics was requested and is valid fetch it.
if (getTableRequest.isGetColumnStats()) {
ColumnStatistics colStats = getMS().getTableColumnStatistics(catName, dbName, tblName,
StatsSetupConst.getColumnsHavingStats(t.getParameters()), getTableRequest.getEngine(),
getTableRequest.getValidWriteIdList());
if (colStats != null) {
t.setColStats(colStats);
}
}
} catch (Exception e) {
throwMetaException(e);
}
return t;
}
/**
* Gets multiple tables from the hive metastore.
*
* @param dbName
* The name of the database in which the tables reside
* @param tableNames
* The names of the tables to get.
*
* @return A list of tables whose names are in the the list "names" and
* are retrievable from the database specified by "dbnames."
* There is no guarantee of the order of the returned tables.
* If there are duplicate names, only one instance of the table will be returned.
* @throws MetaException
* @throws InvalidOperationException
* @throws UnknownDBException
*/
@Override
@Deprecated
public List<Table> get_table_objects_by_name(final String dbName, final List<String> tableNames)
throws MetaException, InvalidOperationException, UnknownDBException {
String[] parsedDbName = parseDbName(dbName, conf);
return getTableObjectsInternal(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableNames, null, null, null);
}
@Override
public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req) throws TException {
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
if (isDatabaseRemote(req.getDbName())) {
return new GetTablesResult(getRemoteTableObjectsInternal(req.getDbName(), req.getTblNames(), req.getTablesPattern()));
}
return new GetTablesResult(getTableObjectsInternal(catName, req.getDbName(),
req.getTblNames(), req.getCapabilities(), req.getProjectionSpec(), req.getTablesPattern()));
}
private List<Table> filterTablesByName(List<Table> tables, List<String> tableNames) {
List<Table> filteredTables = new ArrayList<>();
for (Table table : tables) {
if (tableNames.contains(table.getTableName())) {
filteredTables.add(table);
}
}
return filteredTables;
}
private List<Table> getRemoteTableObjectsInternal(String dbname, List<String> tableNames, String pattern) throws MetaException {
String[] parsedDbName = parseDbName(dbname, conf);
try {
// retrieve tables from remote database
Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
List<Table> tables = DataConnectorProviderFactory.getDataConnectorProvider(db).getTables(null);
// filtered out undesired tables
if (tableNames != null) {
tables = filterTablesByName(tables, tableNames);
}
// set remote tables' local hive database reference
for (Table table : tables) {
table.setDbName(dbname);
}
return FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tables);
} catch (Exception e) {
LOG.warn("Unexpected exception while getting table(s) in remote database " + dbname , e);
return new ArrayList<Table>();
}
}
private List<Table> getTableObjectsInternal(String catName, String dbName,
List<String> tableNames,
ClientCapabilities capabilities,
GetProjectionsSpec projectionsSpec, String tablePattern)
throws MetaException, InvalidOperationException, UnknownDBException {
if (isInTest) {
assertClientHasCapability(capabilities, ClientCapability.TEST_CAPABILITY,
"Hive tests", "get_table_objects_by_name_req");
}
if (projectionsSpec != null) {
if (!projectionsSpec.isSetFieldList() && (projectionsSpec.isSetIncludeParamKeyPattern() ||
projectionsSpec.isSetExcludeParamKeyPattern())) {
throw new InvalidOperationException("Include and Exclude Param key are not supported.");
}
}
List<Table> tables = new ArrayList<>();
startMultiTableFunction("get_multi_table", dbName, tableNames);
Exception ex = null;
int tableBatchSize = MetastoreConf.getIntVar(conf,
ConfVars.BATCH_RETRIEVE_MAX);
try {
if (dbName == null || dbName.isEmpty()) {
throw new UnknownDBException("DB name is null or empty");
}
RawStore ms = getMS();
if(tablePattern != null){
tables = ms.getTableObjectsByName(catName, dbName, tableNames, projectionsSpec, tablePattern);
}else {
if (tableNames == null) {
throw new InvalidOperationException(dbName + " cannot find null tables");
}
// The list of table names could contain duplicates. RawStore.getTableObjectsByName()
// only guarantees returning no duplicate table objects in one batch. If we need
// to break into multiple batches, remove duplicates first.
List<String> distinctTableNames = tableNames;
if (distinctTableNames.size() > tableBatchSize) {
List<String> lowercaseTableNames = new ArrayList<>();
for (String tableName : tableNames) {
lowercaseTableNames.add(normalizeIdentifier(tableName));
}
distinctTableNames = new ArrayList<>(new HashSet<>(lowercaseTableNames));
}
int startIndex = 0;
// Retrieve the tables from the metastore in batches. Some databases like
// Oracle cannot have over 1000 expressions in a in-list
while (startIndex < distinctTableNames.size()) {
int endIndex = Math.min(startIndex + tableBatchSize, distinctTableNames.size());
tables.addAll(ms.getTableObjectsByName(catName, dbName, distinctTableNames.subList(
startIndex, endIndex), projectionsSpec, tablePattern));
startIndex = endIndex;
}
}
for (Table t : tables) {
if (t.getParameters() != null && MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES,
"insert-only tables", "get_table_req");
}
}
tables = FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tables);
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidOperationException.class, UnknownDBException.class)
.defaultMetaException();
} finally {
endFunction("get_multi_table", tables != null, ex, join(tableNames, ","));
}
return tables;
}
@Override
public Materialization get_materialization_invalidation_info(final CreationMetadata cm, final String validTxnList) throws MetaException {
return getTxnHandler().getMaterializationInvalidationInfo(cm, validTxnList);
}
@Override
public void update_creation_metadata(String catName, final String dbName, final String tableName, CreationMetadata cm) throws MetaException {
getMS().updateCreationMetadata(catName, dbName, tableName, cm);
}
private void assertClientHasCapability(ClientCapabilities client,
ClientCapability value, String what, String call) throws MetaException {
if (!doesClientHaveCapability(client, value)) {
throw new MetaException("Your client does not appear to support " + what + ". To skip"
+ " capability checks, please set " + ConfVars.CAPABILITY_CHECK.toString()
+ " to false. This setting can be set globally, or on the client for the current"
+ " metastore session. Note that this may lead to incorrect results, data loss,"
+ " undefined behavior, etc. if your client is actually incompatible. You can also"
+ " specify custom client capabilities via " + call + " API.");
}
}
private boolean doesClientHaveCapability(ClientCapabilities client, ClientCapability value) {
if (!MetastoreConf.getBoolVar(getConf(), ConfVars.CAPABILITY_CHECK)) {
return true;
}
return (client != null && client.isSetValues() && client.getValues().contains(value));
}
@Override
public List<String> get_table_names_by_filter(
final String dbName, final String filter, final short maxTables)
throws MetaException, InvalidOperationException, UnknownDBException {
List<String> tables = null;
startFunction("get_table_names_by_filter", ": db = " + dbName + ", filter = " + filter);
Exception ex = null;
String[] parsedDbName = parseDbName(dbName, conf);
try {
if (parsedDbName[CAT_NAME] == null || parsedDbName[CAT_NAME].isEmpty() ||
parsedDbName[DB_NAME] == null || parsedDbName[DB_NAME].isEmpty()) {
throw new UnknownDBException("DB name is null or empty");
}
if (filter == null) {
throw new InvalidOperationException(filter + " cannot apply null filter");
}
tables = getMS().listTableNamesByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], filter, maxTables);
tables = FilterUtils.filterTableNamesIfEnabled(
isServerFilterEnabled, filterHook, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tables);
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidOperationException.class, UnknownDBException.class)
.defaultMetaException();
} finally {
endFunction("get_table_names_by_filter", tables != null, ex, join(tables, ","));
}
return tables;
}
private Partition append_partition_common(RawStore ms, String catName, String dbName,
String tableName, List<String> part_vals,
EnvironmentContext envContext)
throws InvalidObjectException, AlreadyExistsException, MetaException, NoSuchObjectException {
Partition part = new Partition();
boolean success = false, madeDir = false;
Path partLocation = null;
Table tbl = null;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
Database db = null;
try {
ms.openTransaction();
part.setCatName(catName);
part.setDbName(dbName);
part.setTableName(tableName);
part.setValues(part_vals);
MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern);
tbl = ms.getTable(part.getCatName(), part.getDbName(), part.getTableName(), null);
if (tbl == null) {
throw new InvalidObjectException(
"Unable to add partition because table or database do not exist");
}
if (tbl.getSd().getLocation() == null) {
throw new MetaException(
"Cannot append a partition to a view");
}
db = get_database_core(catName, dbName);
firePreEvent(new PreAddPartitionEvent(tbl, part, this));
part.setSd(tbl.getSd().deepCopy());
partLocation = new Path(tbl.getSd().getLocation(), Warehouse
.makePartName(tbl.getPartitionKeys(), part_vals));
part.getSd().setLocation(partLocation.toString());
Partition old_part;
try {
old_part = ms.getPartition(part.getCatName(), part.getDbName(), part
.getTableName(), part.getValues());
} catch (NoSuchObjectException e) {
// this means there is no existing partition
old_part = null;
}
if (old_part != null) {
throw new AlreadyExistsException("Partition already exists:" + part);
}
if (!wh.isDir(partLocation)) {
if (!wh.mkdirs(partLocation)) {
throw new MetaException(partLocation
+ " is not a directory or unable to create one");
}
madeDir = true;
}
// set create time
long time = System.currentTimeMillis() / 1000;
part.setCreateTime((int) time);
part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time));
if (canUpdateStats(tbl)) {
MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, envContext, true);
}
if (ms.addPartition(part)) {
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, part, true, this),
envContext);
}
success = ms.commitTransaction();
}
} finally {
if (!success) {
ms.rollbackTransaction();
if (madeDir) {
wh.deleteDir(partLocation, true, false, ReplChangeManager.shouldEnableCm(db, tbl));
}
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, part, success, this),
envContext,
transactionalListenerResponses, ms);
}
}
return part;
}
private void firePreEvent(PreEventContext event) throws MetaException {
for (MetaStorePreEventListener listener : preListeners) {
try {
listener.onEvent(event);
} catch (NoSuchObjectException e) {
throw new MetaException(e.getMessage());
} catch (InvalidOperationException e) {
throw new MetaException(e.getMessage());
}
}
}
@Override
public Partition append_partition(final String dbName, final String tableName,
final List<String> part_vals) throws InvalidObjectException,
AlreadyExistsException, MetaException {
return append_partition_with_environment_context(dbName, tableName, part_vals, null);
}
@Override
public Partition append_partition_with_environment_context(final String dbName,
final String tableName, final List<String> part_vals, final EnvironmentContext envContext)
throws InvalidObjectException, AlreadyExistsException, MetaException {
if (part_vals == null || part_vals.isEmpty()) {
throw new MetaException("The partition values must not be null or empty.");
}
String[] parsedDbName = parseDbName(dbName, conf);
startPartitionFunction("append_partition", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, part_vals);
if (LOG.isDebugEnabled()) {
for (String part : part_vals) {
LOG.debug(part);
}
}
Partition ret = null;
Exception ex = null;
try {
ret = append_partition_common(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, part_vals, envContext);
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
.defaultMetaException();
} finally {
endFunction("append_partition", ret != null, ex, tableName);
}
return ret;
}
private static class PartValEqWrapperLite {
List<String> values;
String location;
PartValEqWrapperLite(Partition partition) {
this.values = partition.isSetValues()? partition.getValues() : null;
if (partition.getSd() != null) {
this.location = partition.getSd().getLocation();
}
}
@Override
public int hashCode() {
return values == null ? 0 : values.hashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || !(obj instanceof PartValEqWrapperLite)) {
return false;
}
List<String> lhsValues = this.values;
List<String> rhsValues = ((PartValEqWrapperLite)obj).values;
if (lhsValues == null || rhsValues == null) {
return lhsValues == rhsValues;
}
if (lhsValues.size() != rhsValues.size()) {
return false;
}
for (int i=0; i<lhsValues.size(); ++i) {
String lhsValue = lhsValues.get(i);
String rhsValue = rhsValues.get(i);
if ((lhsValue == null && rhsValue != null)
|| (lhsValue != null && !lhsValue.equals(rhsValue))) {
return false;
}
}
return true;
}
}
private List<Partition> add_partitions_core(final RawStore ms, String catName,
String dbName, String tblName, List<Partition> parts, final boolean ifNotExists)
throws TException {
logAndAudit("add_partitions");
boolean success = false;
// Ensures that the list doesn't have dups, and keeps track of directories we have created.
final Map<PartValEqWrapperLite, Boolean> addedPartitions = new ConcurrentHashMap<>();
final List<Partition> newParts = new ArrayList<>();
final List<Partition> existingParts = new ArrayList<>();
Table tbl = null;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
Database db = null;
List<ColumnStatistics> partsColStats = new ArrayList<>(parts.size());
List<Long> partsWriteIds = new ArrayList<>(parts.size());
throwUnsupportedExceptionIfRemoteDB(dbName, "add_partitions");
Lock tableLock = getTableLockFor(dbName, tblName);
tableLock.lock();
try {
ms.openTransaction();
tbl = ms.getTable(catName, dbName, tblName, null);
if (tbl == null) {
throw new InvalidObjectException("Unable to add partitions because "
+ TableName.getQualified(catName, dbName, tblName) +
" does not exist");
}
MTable mTable = getMS().ensureGetMTable(catName, dbName, tblName);
db = ms.getDatabase(catName, dbName);
if (!parts.isEmpty()) {
firePreEvent(new PreAddPartitionEvent(tbl, parts, this));
}
Set<PartValEqWrapperLite> partsToAdd = new HashSet<>(parts.size());
List<Partition> partitionsToAdd = new ArrayList<>(parts.size());
List<FieldSchema> partitionKeys = tbl.getPartitionKeys();
for (final Partition part : parts) {
// Collect partition column stats to be updated if present. Partition objects passed down
// here at the time of replication may have statistics in them, which is required to be
// updated in the metadata. But we don't want it to be part of the Partition object when
// it's being created or altered, lest it becomes part of the notification event.
if (part.isSetColStats()) {
partsColStats.add(part.getColStats());
part.unsetColStats();
partsWriteIds.add(part.getWriteId());
}
// Iterate through the partitions and validate them. If one of the partitions is
// incorrect, an exception will be thrown before the threads which create the partition
// folders are submitted. This way we can be sure that no partition and no partition
// folder will be created if the list contains an invalid partition.
if (validatePartition(part, catName, tblName, dbName, partsToAdd, ms, ifNotExists,
partitionKeys)) {
partitionsToAdd.add(part);
} else {
existingParts.add(part);
}
}
newParts.addAll(createPartitionFolders(partitionsToAdd, tbl, addedPartitions));
if (!newParts.isEmpty()) {
ms.addPartitions(catName, dbName, tblName, newParts);
}
// Notification is generated for newly created partitions only. The subset of partitions
// that already exist (existingParts), will not generate notifications.
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, newParts, true, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, newParts, true, this),
null,
transactionalListenerResponses, ms);
if (!existingParts.isEmpty()) {
// The request has succeeded but we failed to add these partitions.
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, existingParts, false, this),
null, null, ms);
}
}
// Update partition column statistics if available. We need a valid writeId list to
// update column statistics for a transactional table. But during bootstrap replication,
// where we use this feature, we do not have a valid writeId list which was used to
// update the stats. But we know for sure that the writeId associated with the stats was
// valid then (otherwise stats update would have failed on the source). So, craft a valid
// transaction list with only that writeId and use it to update the stats.
int cnt = 0;
for (ColumnStatistics partColStats: partsColStats) {
long writeId = partsWriteIds.get(cnt++);
String validWriteIds = null;
if (writeId > 0) {
ValidWriteIdList validWriteIdList =
new ValidReaderWriteIdList(TableName.getDbTable(tbl.getDbName(),
tbl.getTableName()),
new long[0], new BitSet(), writeId);
validWriteIds = validWriteIdList.toString();
}
updatePartitonColStatsInternal(tbl, mTable, partColStats, validWriteIds, writeId);
}
success = ms.commitTransaction();
} finally {
try {
if (!success) {
ms.rollbackTransaction();
cleanupPartitionFolders(addedPartitions, db);
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, parts, false, this),
null, null, ms);
}
}
} finally {
tableLock.unlock();
}
}
return newParts;
}
private Lock getTableLockFor(String dbName, String tblName) {
return tablelocks.get(dbName + "." + tblName);
}
/**
* Remove the newly created partition folders. The values in the addedPartitions map indicates
* whether or not the location of the partition was newly created. If the value is false, the
* partition folder will not be removed.
* @param addedPartitions
* @throws MetaException
* @throws IllegalArgumentException
*/
private void cleanupPartitionFolders(final Map<PartValEqWrapperLite, Boolean> addedPartitions,
Database db) throws MetaException, IllegalArgumentException {
for (Map.Entry<PartValEqWrapperLite, Boolean> e : addedPartitions.entrySet()) {
if (e.getValue()) {
// we just created this directory - it's not a case of pre-creation, so we nuke.
wh.deleteDir(new Path(e.getKey().location), true, db);
}
}
}
/**
* Validate a partition before creating it. The validation checks
* <ul>
* <li>if the database and table names set in the partition are not null and they are matching
* with the expected values set in the tblName and dbName parameters.</li>
* <li>if the partition values are set.</li>
* <li>if none of the partition values is null.</li>
* <li>if the partition values are matching with the pattern set in the
* 'metastore.partition.name.whitelist.pattern' configuration property.</li>
* <li>if the partition doesn't already exist. If the partition already exists, an exception
* will be thrown if the ifNotExists parameter is false, otherwise it will be just ignored.</li>
* <li>if the partsToAdd set doesn't contain the partition. The partsToAdd set contains the
* partitions which are already validated. If the set contains the current partition, it means
* that the partition is tried to be added multiple times in the same batch. Please note that
* the set will be updated with the current partition if the validation was successful.</li>
* </ul>
* @param part
* @param catName
* @param tblName
* @param dbName
* @param partsToAdd
* @param ms
* @param ifNotExists
* @return
* @throws MetaException
* @throws TException
*/
private boolean validatePartition(final Partition part, final String catName,
final String tblName, final String dbName, final Set<PartValEqWrapperLite> partsToAdd,
final RawStore ms, final boolean ifNotExists, List<FieldSchema> partitionKeys) throws MetaException, TException {
if (part.getDbName() == null || part.getTableName() == null) {
throw new MetaException("The database and table name must be set in the partition.");
}
if (!part.getTableName().equalsIgnoreCase(tblName)
|| !part.getDbName().equalsIgnoreCase(dbName)) {
String errorMsg = String.format(
"Partition does not belong to target table %s. It belongs to the table %s.%s : %s",
TableName.getQualified(catName, dbName, tblName), part.getDbName(),
part.getTableName(), part.toString());
throw new MetaException(errorMsg);
}
if (part.getValues() == null || part.getValues().isEmpty()) {
throw new MetaException("The partition values cannot be null or empty.");
}
if (part.getValues().contains(null)) {
throw new MetaException("Partition value cannot be null.");
}
boolean shouldAdd = startAddPartition(ms, part, partitionKeys, ifNotExists);
if (!shouldAdd) {
LOG.info("Not adding partition {} as it already exists", part);
return false;
}
if (!partsToAdd.add(new PartValEqWrapperLite(part))) {
// Technically, for ifNotExists case, we could insert one and discard the other
// because the first one now "exists", but it seems better to report the problem
// upstream as such a command doesn't make sense.
throw new MetaException("Duplicate partitions in the list: " + part);
}
return true;
}
/**
* Create the location folders for the partitions. For each partition a separate thread will be
* started to create the folder. The method will wait until all threads are finished and returns
* the partitions whose folders were created successfully. If an error occurs during the
* execution of a thread, a MetaException will be thrown.
* @param partitionsToAdd
* @param table
* @param addedPartitions
* @return
* @throws MetaException
*/
private List<Partition> createPartitionFolders(final List<Partition> partitionsToAdd,
final Table table, final Map<PartValEqWrapperLite, Boolean> addedPartitions)
throws MetaException {
final AtomicBoolean failureOccurred = new AtomicBoolean(false);
final List<Future<Partition>> partFutures = new ArrayList<>(partitionsToAdd.size());
final Map<PartValEqWrapperLite, Boolean> addedParts = new ConcurrentHashMap<>();
final UserGroupInformation ugi;
try {
ugi = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new RuntimeException(e);
}
for (final Partition partition : partitionsToAdd) {
initializePartitionParameters(table, partition);
partFutures.add(threadPool.submit(() -> {
if (failureOccurred.get()) {
return null;
}
ugi.doAs((PrivilegedExceptionAction<Partition>) () -> {
try {
boolean madeDir = createLocationForAddedPartition(table, partition);
addedParts.put(new PartValEqWrapperLite(partition), madeDir);
initializeAddedPartition(table, partition, madeDir, null);
} catch (MetaException e) {
throw new IOException(e.getMessage(), e);
}
return null;
});
return partition;
}));
}
List<Partition> newParts = new ArrayList<>(partitionsToAdd.size());
String errorMessage = null;
for (Future<Partition> partFuture : partFutures) {
try {
Partition part = partFuture.get();
if (part != null && !failureOccurred.get()) {
newParts.add(part);
}
} catch (ExecutionException e) {
// If an exception is thrown in the execution of a task, set the failureOccurred flag to
// true. This flag is visible in the tasks and if its value is true, the partition
// folders won't be created.
// Then iterate through the remaining tasks and wait for them to finish. The tasks which
// are started before the flag got set will then finish creating the partition folders.
// The tasks which are started after the flag got set, won't create the partition
// folders, to avoid unnecessary work.
// This way it is sure that all tasks are finished, when entering the finally part where
// the partition folders are cleaned up. It won't happen that a task is still running
// when cleaning up the folders, so it is sure we won't have leftover folders.
// Canceling the other tasks would be also an option but during testing it turned out
// that it is not a trustworthy solution to avoid leftover folders.
failureOccurred.compareAndSet(false, true);
errorMessage = e.getMessage();
} catch (InterruptedException e) {
failureOccurred.compareAndSet(false, true);
errorMessage = e.getMessage();
// Restore interruption status of the corresponding thread
Thread.currentThread().interrupt();
}
}
addedPartitions.putAll(addedParts);
if (failureOccurred.get()) {
throw new MetaException(errorMessage);
}
return newParts;
}
@Override
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
throws TException {
AddPartitionsResult result = new AddPartitionsResult();
if (request.getParts().isEmpty()) {
return result;
}
try {
if (!request.isSetCatName()) {
request.setCatName(getDefaultCatalog(conf));
}
// Make sure all of the partitions have the catalog set as well
request.getParts().forEach(p -> {
if (!p.isSetCatName()) {
p.setCatName(getDefaultCatalog(conf));
}
});
List<Partition> parts = add_partitions_core(getMS(), request.getCatName(), request.getDbName(),
request.getTblName(), request.getParts(), request.isIfNotExists());
if (request.isNeedResult()) {
result.setPartitions(parts);
}
} catch (Exception e) {
throw handleException(e).throwIfInstance(TException.class).defaultMetaException();
}
return result;
}
@Override
public int add_partitions(final List<Partition> parts) throws MetaException,
InvalidObjectException, AlreadyExistsException {
startFunction("add_partition");
if (parts == null) {
throw new MetaException("Partition list cannot be null.");
}
if (parts.isEmpty()) {
return 0;
}
Integer ret = null;
Exception ex = null;
try {
// Old API assumed all partitions belong to the same table; keep the same assumption
if (!parts.get(0).isSetCatName()) {
String defaultCat = getDefaultCatalog(conf);
for (Partition p : parts) {
p.setCatName(defaultCat);
}
}
ret = add_partitions_core(getMS(), parts.get(0).getCatName(), parts.get(0).getDbName(),
parts.get(0).getTableName(), parts, false).size();
assert ret == parts.size();
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
.defaultMetaException();
} finally {
String tableName = parts.get(0).getTableName();
endFunction("add_partition", ret != null, ex, tableName);
}
return ret;
}
@Override
public int add_partitions_pspec(final List<PartitionSpec> partSpecs)
throws TException {
logAndAudit("add_partitions_pspec");
if (partSpecs.isEmpty()) {
return 0;
}
String dbName = partSpecs.get(0).getDbName();
String tableName = partSpecs.get(0).getTableName();
// If the catalog name isn't set, we need to go through and set it.
String catName;
if (!partSpecs.get(0).isSetCatName()) {
catName = getDefaultCatalog(conf);
partSpecs.forEach(ps -> ps.setCatName(catName));
} else {
catName = partSpecs.get(0).getCatName();
}
return add_partitions_pspec_core(getMS(), catName, dbName, tableName, partSpecs, false);
}
private int add_partitions_pspec_core(RawStore ms, String catName, String dbName,
String tblName, List<PartitionSpec> partSpecs,
boolean ifNotExists)
throws TException {
boolean success = false;
if (dbName == null || tblName == null) {
throw new MetaException("The database and table name cannot be null.");
}
// Ensures that the list doesn't have dups, and keeps track of directories we have created.
final Map<PartValEqWrapperLite, Boolean> addedPartitions = new ConcurrentHashMap<>();
PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partSpecs);
final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy
.getPartitionIterator();
Table tbl = null;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
Database db = null;
Lock tableLock = getTableLockFor(dbName, tblName);
tableLock.lock();
try {
ms.openTransaction();
try {
db = ms.getDatabase(catName, dbName);
} catch (NoSuchObjectException notExists) {
throw new InvalidObjectException("Unable to add partitions because "
+ "database or table " + dbName + "." + tblName + " does not exist");
}
if (db.getType() == DatabaseType.REMOTE) {
throw new MetaException("Operation add_partitions_pspec not supported on tables in REMOTE database");
}
tbl = ms.getTable(catName, dbName, tblName, null);
if (tbl == null) {
throw new InvalidObjectException("Unable to add partitions because "
+ "database or table " + dbName + "." + tblName + " does not exist");
}
firePreEvent(new PreAddPartitionEvent(tbl, partitionSpecProxy, this));
Set<PartValEqWrapperLite> partsToAdd = new HashSet<>(partitionSpecProxy.size());
List<Partition> partitionsToAdd = new ArrayList<>(partitionSpecProxy.size());
List<FieldSchema> partitionKeys = tbl.getPartitionKeys();
while (partitionIterator.hasNext()) {
// Iterate through the partitions and validate them. If one of the partitions is
// incorrect, an exception will be thrown before the threads which create the partition
// folders are submitted. This way we can be sure that no partition or partition folder
// will be created if the list contains an invalid partition.
final Partition part = partitionIterator.getCurrent();
if (validatePartition(part, catName, tblName, dbName, partsToAdd, ms, ifNotExists,
partitionKeys)) {
partitionsToAdd.add(part);
}
partitionIterator.next();
}
createPartitionFolders(partitionsToAdd, tbl, addedPartitions);
ms.addPartitions(catName, dbName, tblName, partitionSpecProxy, ifNotExists);
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, partitionSpecProxy, true, this));
}
success = ms.commitTransaction();
return addedPartitions.size();
} finally {
try {
if (!success) {
ms.rollbackTransaction();
cleanupPartitionFolders(addedPartitions, db);
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, partitionSpecProxy, true, this),
null,
transactionalListenerResponses, ms);
}
} finally {
tableLock.unlock();
}
}
}
private boolean startAddPartition(
RawStore ms, Partition part, List<FieldSchema> partitionKeys, boolean ifNotExists)
throws TException {
MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(),
partitionValidationPattern);
boolean doesExist = ms.doesPartitionExist(part.getCatName(),
part.getDbName(), part.getTableName(), partitionKeys, part.getValues());
if (doesExist && !ifNotExists) {
throw new AlreadyExistsException("Partition already exists: " + part);
}
return !doesExist;
}
/**
* Handles the location for a partition being created.
* @param tbl Table.
* @param part Partition.
* @return Whether the partition SD location is set to a newly created directory.
*/
private boolean createLocationForAddedPartition(
final Table tbl, final Partition part) throws MetaException {
Path partLocation = null;
String partLocationStr = null;
if (part.getSd() != null) {
partLocationStr = part.getSd().getLocation();
}
if (partLocationStr == null || partLocationStr.isEmpty()) {
// set default location if not specified and this is
// a physical table partition (not a view)
if (tbl.getSd().getLocation() != null) {
partLocation = new Path(tbl.getSd().getLocation(), Warehouse
.makePartName(tbl.getPartitionKeys(), part.getValues()));
}
} else {
if (tbl.getSd().getLocation() == null) {
throw new MetaException("Cannot specify location for a view partition");
}
partLocation = wh.getDnsPath(new Path(partLocationStr));
}
boolean result = false;
if (partLocation != null) {
part.getSd().setLocation(partLocation.toString());
// Check to see if the directory already exists before calling
// mkdirs() because if the file system is read-only, mkdirs will
// throw an exception even if the directory already exists.
if (!wh.isDir(partLocation)) {
if (!wh.mkdirs(partLocation)) {
throw new MetaException(partLocation
+ " is not a directory or unable to create one");
}
result = true;
}
}
return result;
}
/**
* Verify if update stats while altering partition(s)
* For the following three cases HMS will not update partition stats
* 1) Table property 'DO_NOT_UPDATE_STATS' = True
* 2) HMS configuration property 'STATS_AUTO_GATHER' = False
* 3) Is View
*/
private boolean canUpdateStats(Table tbl) {
Map<String,String> tblParams = tbl.getParameters();
boolean updateStatsTbl = true;
if ((tblParams != null) && tblParams.containsKey(StatsSetupConst.DO_NOT_UPDATE_STATS)) {
updateStatsTbl = !Boolean.valueOf(tblParams.get(StatsSetupConst.DO_NOT_UPDATE_STATS));
}
if (!MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) ||
MetaStoreUtils.isView(tbl) ||
!updateStatsTbl) {
return false;
}
return true;
}
private void initializeAddedPartition(final Table tbl, final Partition part, boolean madeDir,
EnvironmentContext environmentContext) throws MetaException {
initializeAddedPartition(tbl,
new PartitionSpecProxy.SimplePartitionWrapperIterator(part), madeDir, environmentContext);
}
private void initializeAddedPartition(
final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir,
EnvironmentContext environmentContext) throws MetaException {
if (canUpdateStats(tbl)) {
MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir,
false, environmentContext, true);
}
// set create time
long time = System.currentTimeMillis() / 1000;
part.setCreateTime((int) time);
if (part.getParameters() == null ||
part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) {
part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time));
}
}
private void initializePartitionParameters(final Table tbl, final Partition part)
throws MetaException {
initializePartitionParameters(tbl,
new PartitionSpecProxy.SimplePartitionWrapperIterator(part));
}
private void initializePartitionParameters(final Table tbl,
final PartitionSpecProxy.PartitionIterator part) throws MetaException {
// Inherit table properties into partition properties.
Map<String, String> tblParams = tbl.getParameters();
String inheritProps = MetastoreConf.getVar(conf, ConfVars.PART_INHERIT_TBL_PROPS).trim();
// Default value is empty string in which case no properties will be inherited.
// * implies all properties needs to be inherited
Set<String> inheritKeys = new HashSet<>(Arrays.asList(inheritProps.split(",")));
if (inheritKeys.contains("*")) {
inheritKeys = tblParams.keySet();
}
for (String key : inheritKeys) {
String paramVal = tblParams.get(key);
if (null != paramVal) { // add the property only if it exists in table properties
part.putToParameters(key, paramVal);
}
}
}
private Partition add_partition_core(final RawStore ms,
final Partition part, final EnvironmentContext envContext)
throws TException {
boolean success = false;
Table tbl = null;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
if (!part.isSetCatName()) {
part.setCatName(getDefaultCatalog(conf));
}
try {
ms.openTransaction();
tbl = ms.getTable(part.getCatName(), part.getDbName(), part.getTableName(), null);
if (tbl == null) {
throw new InvalidObjectException(
"Unable to add partition because table or database do not exist");
}
firePreEvent(new PreAddPartitionEvent(tbl, part, this));
if (part.getValues() == null || part.getValues().isEmpty()) {
throw new MetaException("The partition values cannot be null or empty.");
}
boolean shouldAdd = startAddPartition(ms, part, tbl.getPartitionKeys(), false);
assert shouldAdd; // start would throw if it already existed here
boolean madeDir = createLocationForAddedPartition(tbl, part);
try {
initializeAddedPartition(tbl, part, madeDir, envContext);
initializePartitionParameters(tbl, part);
success = ms.addPartition(part);
} finally {
if (!success && madeDir) {
wh.deleteDir(new Path(part.getSd().getLocation()), true, false,
ReplChangeManager.shouldEnableCm(ms.getDatabase(part.getCatName(), part.getDbName()), tbl));
}
}
// Setting success to false to make sure that if the listener fails, rollback happens.
success = false;
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, Arrays.asList(part), true, this),
envContext);
}
// we proceed only if we'd actually succeeded anyway, otherwise,
// we'd have thrown an exception
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(tbl, Arrays.asList(part), success, this),
envContext,
transactionalListenerResponses, ms);
}
}
return part;
}
@Override
public Partition add_partition(final Partition part)
throws InvalidObjectException, AlreadyExistsException, MetaException {
return add_partition_with_environment_context(part, null);
}
@Override
public Partition add_partition_with_environment_context(
final Partition part, EnvironmentContext envContext)
throws InvalidObjectException, AlreadyExistsException,
MetaException {
if (part == null) {
throw new MetaException("Partition cannot be null.");
}
startTableFunction("add_partition",
part.getCatName(), part.getDbName(), part.getTableName());
Partition ret = null;
Exception ex = null;
try {
ret = add_partition_core(getMS(), part, envContext);
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
.defaultMetaException();
} finally {
endFunction("add_partition", ret != null, ex, part != null ? part.getTableName(): null);
}
return ret;
}
@Override
public Partition exchange_partition(Map<String, String> partitionSpecs,
String sourceDbName, String sourceTableName, String destDbName,
String destTableName) throws TException {
exchange_partitions(partitionSpecs, sourceDbName, sourceTableName, destDbName, destTableName);
// Wouldn't it make more sense to return the first element of the list returned by the
// previous call?
return new Partition();
}
@Override
public List<Partition> exchange_partitions(Map<String, String> partitionSpecs,
String sourceDbName, String sourceTableName, String destDbName,
String destTableName) throws TException {
String[] parsedDestDbName = parseDbName(destDbName, conf);
String[] parsedSourceDbName = parseDbName(sourceDbName, conf);
// No need to check catalog for null as parseDbName() will never return null for the catalog.
if (partitionSpecs == null || parsedSourceDbName[DB_NAME] == null || sourceTableName == null
|| parsedDestDbName[DB_NAME] == null || destTableName == null) {
throw new MetaException("The DB and table name for the source and destination tables,"
+ " and the partition specs must not be null.");
}
if (!parsedDestDbName[CAT_NAME].equals(parsedSourceDbName[CAT_NAME])) {
throw new MetaException("You cannot move a partition across catalogs");
}
boolean success = false;
boolean pathCreated = false;
RawStore ms = getMS();
ms.openTransaction();
Table destinationTable =
ms.getTable(
parsedDestDbName[CAT_NAME], parsedDestDbName[DB_NAME], destTableName, null);
if (destinationTable == null) {
throw new MetaException( "The destination table " +
TableName.getQualified(parsedDestDbName[CAT_NAME],
parsedDestDbName[DB_NAME], destTableName) + " not found");
}
Table sourceTable =
ms.getTable(
parsedSourceDbName[CAT_NAME], parsedSourceDbName[DB_NAME], sourceTableName, null);
if (sourceTable == null) {
throw new MetaException("The source table " +
TableName.getQualified(parsedSourceDbName[CAT_NAME],
parsedSourceDbName[DB_NAME], sourceTableName) + " not found");
}
List<String> partVals = MetaStoreUtils.getPvals(sourceTable.getPartitionKeys(),
partitionSpecs);
List<String> partValsPresent = new ArrayList<> ();
List<FieldSchema> partitionKeysPresent = new ArrayList<> ();
int i = 0;
for (FieldSchema fs: sourceTable.getPartitionKeys()) {
String partVal = partVals.get(i);
if (partVal != null && !partVal.equals("")) {
partValsPresent.add(partVal);
partitionKeysPresent.add(fs);
}
i++;
}
// Passed the unparsed DB name here, as get_partitions_ps expects to parse it
List<Partition> partitionsToExchange = get_partitions_ps(sourceDbName, sourceTableName,
partVals, (short)-1);
if (partitionsToExchange == null || partitionsToExchange.isEmpty()) {
throw new MetaException("No partition is found with the values " + partitionSpecs
+ " for the table " + sourceTableName);
}
boolean sameColumns = MetaStoreUtils.compareFieldColumns(
sourceTable.getSd().getCols(), destinationTable.getSd().getCols());
boolean samePartitions = MetaStoreUtils.compareFieldColumns(
sourceTable.getPartitionKeys(), destinationTable.getPartitionKeys());
if (!sameColumns || !samePartitions) {
throw new MetaException("The tables have different schemas." +
" Their partitions cannot be exchanged.");
}
Path sourcePath = new Path(sourceTable.getSd().getLocation(),
Warehouse.makePartName(partitionKeysPresent, partValsPresent));
Path destPath = new Path(destinationTable.getSd().getLocation(),
Warehouse.makePartName(partitionKeysPresent, partValsPresent));
List<Partition> destPartitions = new ArrayList<>();
Map<String, String> transactionalListenerResponsesForAddPartition = Collections.emptyMap();
List<Map<String, String>> transactionalListenerResponsesForDropPartition =
Lists.newArrayListWithCapacity(partitionsToExchange.size());
// Check if any of the partitions already exists in destTable.
List<String> destPartitionNames = ms.listPartitionNames(parsedDestDbName[CAT_NAME],
parsedDestDbName[DB_NAME], destTableName, (short) -1);
if (destPartitionNames != null && !destPartitionNames.isEmpty()) {
for (Partition partition : partitionsToExchange) {
String partToExchangeName =
Warehouse.makePartName(destinationTable.getPartitionKeys(), partition.getValues());
if (destPartitionNames.contains(partToExchangeName)) {
throw new MetaException("The partition " + partToExchangeName
+ " already exists in the table " + destTableName);
}
}
}
Database srcDb = ms.getDatabase(parsedSourceDbName[CAT_NAME], parsedSourceDbName[DB_NAME]);
Database destDb = ms.getDatabase(parsedDestDbName[CAT_NAME], parsedDestDbName[DB_NAME]);
if (!HiveMetaStore.isRenameAllowed(srcDb, destDb)) {
throw new MetaException("Exchange partition not allowed for " +
TableName.getQualified(parsedSourceDbName[CAT_NAME],
parsedSourceDbName[DB_NAME], sourceTableName) + " Dest db : " + destDbName);
}
try {
for (Partition partition: partitionsToExchange) {
Partition destPartition = new Partition(partition);
destPartition.setDbName(parsedDestDbName[DB_NAME]);
destPartition.setTableName(destinationTable.getTableName());
Path destPartitionPath = new Path(destinationTable.getSd().getLocation(),
Warehouse.makePartName(destinationTable.getPartitionKeys(), partition.getValues()));
destPartition.getSd().setLocation(destPartitionPath.toString());
ms.addPartition(destPartition);
destPartitions.add(destPartition);
ms.dropPartition(parsedSourceDbName[CAT_NAME], partition.getDbName(), sourceTable.getTableName(),
Warehouse.makePartName(sourceTable.getPartitionKeys(), partition.getValues()));
}
Path destParentPath = destPath.getParent();
if (!wh.isDir(destParentPath)) {
if (!wh.mkdirs(destParentPath)) {
throw new MetaException("Unable to create path " + destParentPath);
}
}
/*
* TODO: Use the hard link feature of hdfs
* once https://issues.apache.org/jira/browse/HDFS-3370 is done
*/
pathCreated = wh.renameDir(sourcePath, destPath, false);
// Setting success to false to make sure that if the listener fails, rollback happens.
success = false;
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponsesForAddPartition =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ADD_PARTITION,
new AddPartitionEvent(destinationTable, destPartitions, true, this));
for (Partition partition : partitionsToExchange) {
DropPartitionEvent dropPartitionEvent =
new DropPartitionEvent(sourceTable, partition, true, true, this);
transactionalListenerResponsesForDropPartition.add(
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_PARTITION,
dropPartitionEvent));
}
}
success = ms.commitTransaction();
return destPartitions;
} finally {
if (!success || !pathCreated) {
ms.rollbackTransaction();
if (pathCreated) {
wh.renameDir(destPath, sourcePath, false);
}
}
if (!listeners.isEmpty()) {
AddPartitionEvent addPartitionEvent = new AddPartitionEvent(destinationTable, destPartitions, success, this);
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ADD_PARTITION,
addPartitionEvent,
null,
transactionalListenerResponsesForAddPartition, ms);
i = 0;
for (Partition partition : partitionsToExchange) {
DropPartitionEvent dropPartitionEvent =
new DropPartitionEvent(sourceTable, partition, success, true, this);
Map<String, String> parameters =
(transactionalListenerResponsesForDropPartition.size() > i)
? transactionalListenerResponsesForDropPartition.get(i)
: null;
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_PARTITION,
dropPartitionEvent,
null,
parameters, ms);
i++;
}
}
}
}
private boolean drop_partition_common(RawStore ms, String catName, String db_name, String tbl_name,
List<String> part_vals, boolean deleteData, final EnvironmentContext envContext)
throws MetaException, NoSuchObjectException, IOException, InvalidObjectException, InvalidInputException {
Path partPath = null;
boolean isArchived = false;
Path archiveParentDir = null;
boolean success = false;
Table tbl = null;
Partition part = null;
boolean mustPurge = false;
long writeId = 0;
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
boolean needsCm = false;
if (db_name == null) {
throw new MetaException("The DB name cannot be null.");
}
if (tbl_name == null) {
throw new MetaException("The table name cannot be null.");
}
if (part_vals == null) {
throw new MetaException("The partition values cannot be null.");
}
try {
ms.openTransaction();
part = ms.getPartition(catName, db_name, tbl_name, part_vals);
GetTableRequest request = new GetTableRequest(db_name,tbl_name);
request.setCatName(catName);
tbl = get_table_core(request);
firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this));
mustPurge = isMustPurge(envContext, tbl);
writeId = getWriteId(envContext);
if (part == null) {
throw new NoSuchObjectException("Partition doesn't exist. " + part_vals);
}
isArchived = MetaStoreUtils.isArchived(part);
if (isArchived) {
archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
verifyIsWritablePath(archiveParentDir);
}
if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
partPath = new Path(part.getSd().getLocation());
verifyIsWritablePath(partPath);
}
String partName = Warehouse.makePartName(tbl.getPartitionKeys(), part_vals);
ms.dropPartition(catName, db_name, tbl_name, partName);
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_PARTITION,
new DropPartitionEvent(tbl, part, true, deleteData, this),
envContext);
}
needsCm = ReplChangeManager.shouldEnableCm(ms.getDatabase(catName, db_name), tbl);
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (checkTableDataShouldBeDeleted(tbl, deleteData) &&
(partPath != null || archiveParentDir != null)) {
LOG.info(mustPurge ?
"dropPartition() will purge " + partPath + " directly, skipping trash." :
"dropPartition() will move " + partPath + " to trash-directory.");
// Archived partitions have har:/to_har_file as their location.
// The original directory was saved in params
if (isArchived) {
wh.deleteDir(archiveParentDir, true, mustPurge, needsCm);
} else {
wh.deleteDir(partPath, true, mustPurge, needsCm);
deleteParentRecursive(partPath.getParent(), part_vals.size() - 1, mustPurge, needsCm);
}
// ok even if the data is not deleted
} else if (TxnUtils.isTransactionalTable(tbl) && writeId > 0) {
addTruncateBaseFile(partPath, writeId, conf, DataFormat.DROPPED);
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_PARTITION,
new DropPartitionEvent(tbl, part, success, deleteData, this),
envContext,
transactionalListenerResponses, ms);
}
}
return true;
}
static boolean isMustPurge(EnvironmentContext envContext, Table tbl) {
// Data needs deletion. Check if trash may be skipped.
// Trash may be skipped iff:
// 1. deleteData == true, obviously.
// 2. tbl is external.
// 3. Either
// 3.1. User has specified PURGE from the commandline, and if not,
// 3.2. User has set the table to auto-purge.
return ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
|| MetaStoreUtils.isSkipTrash(tbl.getParameters());
}
static long getWriteId(EnvironmentContext context){
return Optional.ofNullable(context)
.map(EnvironmentContext::getProperties)
.map(prop -> prop.get(hive_metastoreConstants.WRITE_ID))
.map(Long::parseLong)
.orElse(0L);
}
private void throwUnsupportedExceptionIfRemoteDB(String dbName, String operationName) throws MetaException {
if (isDatabaseRemote(dbName)) {
throw new MetaException("Operation " + operationName + " not supported for REMOTE database " + dbName);
}
}
private boolean isDatabaseRemote(String name) {
try {
String[] parsedDbName = parseDbName(name, conf);
Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
if (db != null && db.getType() != null && db.getType() == DatabaseType.REMOTE) {
return true;
}
} catch (Exception e) {
return false;
}
return false;
}
private void deleteParentRecursive(Path parent, int depth, boolean mustPurge, boolean needRecycle)
throws IOException, MetaException {
if (depth > 0 && parent != null && wh.isWritable(parent) && wh.isEmptyDir(parent)) {
wh.deleteDir(parent, true, mustPurge, needRecycle);
deleteParentRecursive(parent.getParent(), depth - 1, mustPurge, needRecycle);
}
}
@Override
public boolean drop_partition(final String db_name, final String tbl_name,
final List<String> part_vals, final boolean deleteData)
throws TException {
return drop_partition_with_environment_context(db_name, tbl_name, part_vals, deleteData,
null);
}
/** Stores a path and its size. */
private static class PathAndDepth implements Comparable<PathAndDepth> {
final Path path;
final int depth;
public PathAndDepth(Path path, int depth) {
this.path = path;
this.depth = depth;
}
@Override
public int hashCode() {
return Objects.hash(path.hashCode(), depth);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
PathAndDepth that = (PathAndDepth) o;
return depth == that.depth && Objects.equals(path, that.path);
}
/** The largest {@code depth} is processed first in a {@link PriorityQueue}. */
@Override
public int compareTo(PathAndDepth o) {
return o.depth - depth;
}
}
@Override
public DropPartitionsResult drop_partitions_req(
DropPartitionsRequest request) throws TException {
RawStore ms = getMS();
String dbName = request.getDbName(), tblName = request.getTblName();
String catName = request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf);
boolean ifExists = request.isSetIfExists() && request.isIfExists();
boolean deleteData = request.isSetDeleteData() && request.isDeleteData();
boolean ignoreProtection = request.isSetIgnoreProtection() && request.isIgnoreProtection();
boolean needResult = !request.isSetNeedResult() || request.isNeedResult();
List<PathAndDepth> dirsToDelete = new ArrayList<>();
List<Path> archToDelete = new ArrayList<>();
EnvironmentContext envContext =
request.isSetEnvironmentContext() ? request.getEnvironmentContext() : null;
boolean success = false;
Table tbl = null;
List<Partition> parts = null;
boolean mustPurge = false;
long writeId = 0;
Map<String, String> transactionalListenerResponses = null;
boolean needsCm = false;
try {
ms.openTransaction();
// We need Partition-s for firing events and for result; DN needs MPartition-s to drop.
// Great... Maybe we could bypass fetching MPartitions by issuing direct SQL deletes.
tbl = get_table_core(catName, dbName, tblName);
mustPurge = isMustPurge(envContext, tbl);
writeId = getWriteId(envContext);
int minCount = 0;
RequestPartsSpec spec = request.getParts();
List<String> partNames = null;
if (spec.isSetExprs()) {
// Dropping by expressions.
parts = new ArrayList<>(spec.getExprs().size());
for (DropPartitionsExpr expr : spec.getExprs()) {
++minCount; // At least one partition per expression, if not ifExists
List<Partition> result = new ArrayList<>();
boolean hasUnknown = ms.getPartitionsByExpr(
catName, dbName, tblName, expr.getExpr(), null, (short)-1, result);
if (hasUnknown) {
// Expr is built by DDLSA, it should only contain part cols and simple ops
throw new MetaException("Unexpected unknown partitions to drop");
}
// this is to prevent dropping archived partition which is archived in a
// different level the drop command specified.
if (!ignoreProtection && expr.isSetPartArchiveLevel()) {
for (Partition part : parts) {
if (MetaStoreUtils.isArchived(part)
&& MetaStoreUtils.getArchivingLevel(part) < expr.getPartArchiveLevel()) {
throw new MetaException("Cannot drop a subset of partitions "
+ " in an archive, partition " + part);
}
}
}
parts.addAll(result);
}
} else if (spec.isSetNames()) {
partNames = spec.getNames();
minCount = partNames.size();
parts = ms.getPartitionsByNames(catName, dbName, tblName, partNames);
} else {
throw new MetaException("Partition spec is not set");
}
if ((parts.size() < minCount) && !ifExists) {
throw new NoSuchObjectException("Some partitions to drop are missing");
}
List<String> colNames = null;
if (partNames == null) {
partNames = new ArrayList<>(parts.size());
colNames = new ArrayList<>(tbl.getPartitionKeys().size());
for (FieldSchema col : tbl.getPartitionKeys()) {
colNames.add(col.getName());
}
}
for (Partition part : parts) {
// TODO - we need to speed this up for the normal path where all partitions are under
// the table and we don't have to stat every partition
firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this));
if (colNames != null) {
partNames.add(FileUtils.makePartName(colNames, part.getValues()));
}
// Preserve the old behavior of failing when we cannot write, even w/o deleteData,
// and even if the table is external. That might not make any sense.
if (MetaStoreUtils.isArchived(part)) {
Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
verifyIsWritablePath(archiveParentDir);
archToDelete.add(archiveParentDir);
}
if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
Path partPath = new Path(part.getSd().getLocation());
verifyIsWritablePath(partPath);
dirsToDelete.add(new PathAndDepth(partPath, part.getValues().size()));
}
}
ms.dropPartitions(catName, dbName, tblName, partNames);
if (!parts.isEmpty() && !transactionalListeners.isEmpty()) {
transactionalListenerResponses = MetaStoreListenerNotifier
.notifyEvent(transactionalListeners, EventType.DROP_PARTITION,
new DropPartitionEvent(tbl, parts, true, deleteData, this), envContext);
}
success = ms.commitTransaction();
needsCm = ReplChangeManager.shouldEnableCm(ms.getDatabase(catName, dbName), tbl);
DropPartitionsResult result = new DropPartitionsResult();
if (needResult) {
result.setPartitions(parts);
}
return result;
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (checkTableDataShouldBeDeleted(tbl, deleteData)) {
LOG.info(mustPurge ?
"dropPartition() will purge partition-directories directly, skipping trash."
: "dropPartition() will move partition-directories to trash-directory.");
// Archived partitions have har:/to_har_file as their location.
// The original directory was saved in params
for (Path path : archToDelete) {
wh.deleteDir(path, true, mustPurge, needsCm);
}
// Uses a priority queue to delete the parents of deleted directories if empty.
// Parents with the deepest path are always processed first. It guarantees that the emptiness
// of a parent won't be changed once it has been processed. So duplicated processing can be
// avoided.
PriorityQueue<PathAndDepth> parentsToDelete = new PriorityQueue<>();
for (PathAndDepth p : dirsToDelete) {
wh.deleteDir(p.path, true, mustPurge, needsCm);
addParentForDel(parentsToDelete, p);
}
HashSet<PathAndDepth> processed = new HashSet<>();
while (!parentsToDelete.isEmpty()) {
try {
PathAndDepth p = parentsToDelete.poll();
if (processed.contains(p)) {
continue;
}
processed.add(p);
Path path = p.path;
if (wh.isWritable(path) && wh.isEmptyDir(path)) {
wh.deleteDir(path, true, mustPurge, needsCm);
addParentForDel(parentsToDelete, p);
}
} catch (IOException ex) {
LOG.warn("Error from recursive parent deletion", ex);
throw new MetaException("Failed to delete parent: " + ex.getMessage());
}
}
} else if (TxnUtils.isTransactionalTable(tbl) && writeId > 0) {
for (Partition part : parts) {
if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
Path partPath = new Path(part.getSd().getLocation());
verifyIsWritablePath(partPath);
addTruncateBaseFile(partPath, writeId, conf, DataFormat.DROPPED);
}
}
}
if (parts != null) {
if (!parts.isEmpty() && !listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_PARTITION,
new DropPartitionEvent(tbl, parts, success, deleteData, this),
envContext,
transactionalListenerResponses, ms);
}
}
}
}
private static void addParentForDel(PriorityQueue<PathAndDepth> parentsToDelete, PathAndDepth p) {
Path parent = p.path.getParent();
if (parent != null && p.depth - 1 > 0) {
parentsToDelete.add(new PathAndDepth(parent, p.depth - 1));
}
}
private void verifyIsWritablePath(Path dir) throws MetaException {
try {
if (!wh.isWritable(dir.getParent())) {
throw new MetaException("Table partition not deleted since " + dir.getParent()
+ " is not writable by " + SecurityUtils.getUser());
}
} catch (IOException ex) {
LOG.warn("Error from isWritable", ex);
throw new MetaException("Table partition not deleted since " + dir.getParent()
+ " access cannot be checked: " + ex.getMessage());
}
}
@Override
public boolean drop_partition_with_environment_context(final String db_name,
final String tbl_name, final List<String> part_vals, final boolean deleteData,
final EnvironmentContext envContext)
throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("drop_partition", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals);
LOG.info("Partition values:" + part_vals);
boolean ret = false;
Exception ex = null;
try {
ret = drop_partition_common(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals, deleteData, envContext);
} catch (Exception e) {
ex = e;
handleException(e).convertIfInstance(IOException.class, MetaException.class)
.rethrowException(e);
} finally {
endFunction("drop_partition", ret, ex, tbl_name);
}
return ret;
}
/**
* Use {@link #get_partition_req(GetPartitionRequest)} ()} instead.
*
*/
@Override
@Deprecated
public Partition get_partition(final String db_name, final String tbl_name,
final List<String> part_vals) throws MetaException, NoSuchObjectException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("get_partition", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals);
Partition ret = null;
Exception ex = null;
try {
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
ret = getMS().getPartition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals);
ret = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class).defaultMetaException();
} finally {
endFunction("get_partition", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public GetPartitionResponse get_partition_req(GetPartitionRequest req)
throws MetaException, NoSuchObjectException, TException {
// TODO Move the logic from get_partition to here, as that method is getting deprecated
String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
Partition p = get_partition(dbName, req.getTblName(), req.getPartVals());
GetPartitionResponse res = new GetPartitionResponse();
res.setPartition(p);
return res;
}
/**
* Fire a pre-event for read table operation, if there are any
* pre-event listeners registered
*/
private void fireReadTablePreEvent(String catName, String dbName, String tblName)
throws MetaException, NoSuchObjectException {
if(preListeners.size() > 0) {
Supplier<Table> tableSupplier = Suppliers.memoize(new Supplier<Table>() {
@Override public Table get() {
try {
Table t = getMS().getTable(catName, dbName, tblName, null);
if (t == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName)
+ " table not found");
}
return t;
} catch(MetaException | NoSuchObjectException e) {
throw new RuntimeException(e);
}
}
});
firePreEvent(new PreReadTableEvent(tableSupplier, this));
}
}
/**
* filters out the table meta for which read database access is not granted
* @param catName catalog name
* @param tableMetas list of table metas
* @return filtered list of table metas
* @throws RuntimeException
* @throws NoSuchObjectException
*/
private List<TableMeta> filterReadableTables(String catName, List<TableMeta> tableMetas)
throws RuntimeException, NoSuchObjectException {
List<TableMeta> finalT = new ArrayList<>();
Map<String, Boolean> databaseNames = new HashMap();
for (TableMeta tableMeta : tableMetas) {
String fullDbName = prependCatalogToDbName(catName, tableMeta.getDbName(), conf);
if (databaseNames.get(fullDbName) == null) {
boolean isExecptionThrown = false;
try {
fireReadDatabasePreEvent(fullDbName);
} catch (MetaException e) {
isExecptionThrown = true;
}
databaseNames.put(fullDbName, isExecptionThrown);
}
if (!databaseNames.get(fullDbName)) {
finalT.add(tableMeta);
}
}
return finalT;
}
/**
* Fire a pre-event for read database operation, if there are any
* pre-event listeners registered
*/
private void fireReadDatabasePreEvent(final String name)
throws MetaException, RuntimeException, NoSuchObjectException {
if(preListeners.size() > 0) {
String[] parsedDbName = parseDbName(name, conf);
Database db = null;
try {
db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
if (db == null) {
throw new NoSuchObjectException("Database: " + name + " not found");
}
} catch(MetaException | NoSuchObjectException e) {
throw new RuntimeException(e);
}
firePreEvent(new PreReadDatabaseEvent(db, this));
}
}
@Override
@Deprecated
public Partition get_partition_with_auth(final String db_name,
final String tbl_name, final List<String> part_vals,
final String user_name, final List<String> group_names)
throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("get_partition_with_auth", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tbl_name, part_vals);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
Partition ret = null;
Exception ex = null;
try {
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
ret = getMS().getPartitionWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals, user_name, group_names);
ret = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
handleException(e).convertIfInstance(InvalidObjectException.class, NoSuchObjectException.class)
.rethrowException(e);
} finally {
endFunction("get_partition_with_auth", ret != null, ex, tbl_name);
}
return ret;
}
/**
* Use {@link #get_partitions_req(PartitionsRequest)} ()} instead.
*
*/
@Override
@Deprecated
public List<Partition> get_partitions(final String db_name, final String tbl_name,
final short max_parts) throws NoSuchObjectException, MetaException {
String[] parsedDbName = parseDbName(db_name, conf);
startTableFunction("get_partitions", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
List<Partition> ret = null;
Exception ex = null;
try {
checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, NO_FILTER_STRING, max_parts);
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
ret = getMS().getPartitions(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
max_parts);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
throwMetaException(e);
} finally {
endFunction("get_partitions", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public PartitionsResponse get_partitions_req(PartitionsRequest req)
throws NoSuchObjectException, MetaException, TException {
String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
List<Partition> partitions = get_partitions(dbName, req.getTblName(), req.getMaxParts());
PartitionsResponse res = new PartitionsResponse();
res.setPartitions(partitions);
return res;
}
@Override
@Deprecated
public List<Partition> get_partitions_with_auth(final String dbName,
final String tblName, final short maxParts, final String userName,
final List<String> groupNames) throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
startTableFunction("get_partitions_with_auth", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
List<Partition> ret = null;
Exception ex = null;
try {
checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tblName, NO_FILTER_STRING, maxParts);
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
ret = getMS().getPartitionsWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName,
maxParts, userName, groupNames);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
handleException(e).convertIfInstance(InvalidObjectException.class, NoSuchObjectException.class)
.rethrowException(e);
} finally {
endFunction("get_partitions_with_auth", ret != null, ex, tblName);
}
return ret;
}
private void checkLimitNumberOfPartitionsByFilter(String catName, String dbName,
String tblName, String filterString,
int maxParts) throws TException {
if (isPartitionLimitEnabled()) {
checkLimitNumberOfPartitions(tblName, get_num_partitions_by_filter(prependCatalogToDbName(
catName, dbName, conf), tblName, filterString), maxParts);
}
}
private void checkLimitNumberOfPartitionsByExpr(String catName, String dbName, String tblName,
byte[] filterExpr, int maxParts)
throws TException {
if (isPartitionLimitEnabled()) {
checkLimitNumberOfPartitions(tblName, get_num_partitions_by_expr(catName, dbName, tblName,
filterExpr), maxParts);
}
}
private void checkLimitNumberOfPartitionsByPs(String catName, String dbName, String tblName,
List<String> partVals, int maxParts)
throws TException {
if (isPartitionLimitEnabled()) {
checkLimitNumberOfPartitions(tblName, getNumPartitionsByPs(catName, dbName, tblName,
partVals), maxParts);
}
}
private boolean isPartitionLimitEnabled() {
int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST);
return partitionLimit > -1;
}
private void checkLimitNumberOfPartitions(String tblName, int numPartitions, int maxToFetch) throws MetaException {
if (isPartitionLimitEnabled()) {
int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST);
int partitionRequest = (maxToFetch < 0) ? numPartitions : maxToFetch;
if (partitionRequest > partitionLimit) {
String configName = ConfVars.LIMIT_PARTITION_REQUEST.toString();
throw new MetaException(String.format(PARTITION_NUMBER_EXCEED_LIMIT_MSG, partitionRequest,
tblName, partitionLimit, configName));
}
}
}
@Override
@Deprecated
public List<PartitionSpec> get_partitions_pspec(final String db_name, final String tbl_name, final int max_parts)
throws NoSuchObjectException, MetaException {
String[] parsedDbName = parseDbName(db_name, conf);
String tableName = tbl_name.toLowerCase();
startTableFunction("get_partitions_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
List<PartitionSpec> partitionSpecs = null;
try {
Table table = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
// get_partitions will parse out the catalog and db names itself
List<Partition> partitions = get_partitions(db_name, tableName, (short) max_parts);
if (is_partition_spec_grouping_enabled(table)) {
partitionSpecs = MetaStoreServerUtils
.getPartitionspecsGroupedByStorageDescriptor(table, partitions);
}
else {
PartitionSpec pSpec = new PartitionSpec();
pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
pSpec.setCatName(parsedDbName[CAT_NAME]);
pSpec.setDbName(parsedDbName[DB_NAME]);
pSpec.setTableName(tableName);
pSpec.setRootPath(table.getSd().getLocation());
partitionSpecs = Arrays.asList(pSpec);
}
return partitionSpecs;
}
finally {
endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tbl_name);
}
}
@Override
public GetPartitionsResponse get_partitions_with_specs(GetPartitionsRequest request)
throws MetaException, TException {
String catName = null;
if (request.isSetCatName()) {
catName = request.getCatName();
}
String[] parsedDbName = parseDbName(request.getDbName(), conf);
String tableName = request.getTblName();
if (catName == null) {
// if catName is not provided in the request use the catName parsed from the dbName
catName = parsedDbName[CAT_NAME];
}
startTableFunction("get_partitions_with_specs", catName, parsedDbName[DB_NAME],
tableName);
GetPartitionsResponse response = null;
Exception ex = null;
try {
Table table = get_table_core(catName, parsedDbName[DB_NAME], tableName);
List<Partition> partitions = getMS()
.getPartitionSpecsByFilterAndProjection(table, request.getProjectionSpec(),
request.getFilterSpec());
List<String> processorCapabilities = request.getProcessorCapabilities();
String processorId = request.getProcessorIdentifier();
if (processorCapabilities == null || processorCapabilities.size() == 0 ||
processorCapabilities.contains("MANAGERAWMETADATA")) {
LOG.info("Skipping translation for processor with " + processorId);
} else {
if (transformer != null) {
partitions = transformer.transformPartitions(partitions, table, processorCapabilities, processorId);
}
}
List<PartitionSpec> partitionSpecs =
MetaStoreServerUtils.getPartitionspecsGroupedByStorageDescriptor(table, partitions);
response = new GetPartitionsResponse();
response.setPartitionSpec(partitionSpecs);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partitions_with_specs", response != null, ex, tableName);
}
return response;
}
private static boolean is_partition_spec_grouping_enabled(Table table) {
Map<String, String> parameters = table.getParameters();
return parameters.containsKey("hive.hcatalog.partition.spec.grouping.enabled")
&& parameters.get("hive.hcatalog.partition.spec.grouping.enabled").equalsIgnoreCase("true");
}
@Override
@Deprecated
public List<String> get_partition_names(final String db_name, final String tbl_name,
final short max_parts) throws NoSuchObjectException, MetaException {
String[] parsedDbName = parseDbName(db_name, conf);
startTableFunction("get_partition_names", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
List<String> ret = null;
Exception ex = null;
try {
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
ret = getMS().listPartitionNames(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
max_parts);
ret = FilterUtils.filterPartitionNamesIfEnabled(isServerFilterEnabled,
filterHook, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, ret);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_partition_names", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public PartitionValuesResponse get_partition_values(PartitionValuesRequest request)
throws MetaException {
String catName = request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf);
String dbName = request.getDbName();
String tblName = request.getTblName();
try {
authorizeTableForPartitionMetadata(catName, dbName, tblName);
// This is serious black magic, as the following 2 lines do nothing AFAICT but without them
// the subsequent call to listPartitionValues fails.
List<FieldSchema> partCols = new ArrayList<FieldSchema>();
partCols.add(request.getPartitionKeys().get(0));
return getMS().listPartitionValues(catName, dbName, tblName, request.getPartitionKeys(),
request.isApplyDistinct(), request.getFilter(), request.isAscending(),
request.getPartitionOrder(), request.getMaxParts());
} catch (NoSuchObjectException e) {
LOG.error(String.format("Unable to get partition for %s.%s.%s", catName, dbName, tblName), e);
throw new MetaException(e.getMessage());
}
}
@Deprecated
@Override
public void alter_partition(final String db_name, final String tbl_name,
final Partition new_part)
throws TException {
rename_partition(db_name, tbl_name, null, new_part);
}
@Deprecated
@Override
public void alter_partition_with_environment_context(final String dbName,
final String tableName, final Partition newPartition,
final EnvironmentContext envContext)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
// TODO: this method name is confusing, it actually does full alter (sortof)
rename_partition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, null, newPartition,
envContext, null);
}
@Deprecated
@Override
public void rename_partition(final String db_name, final String tbl_name,
final List<String> part_vals, final Partition new_part)
throws TException {
// Call rename_partition without an environment context.
String[] parsedDbName = parseDbName(db_name, conf);
rename_partition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, new_part,
null, null);
}
@Override
public RenamePartitionResponse rename_partition_req(RenamePartitionRequest req) throws TException {
EnvironmentContext context = new EnvironmentContext();
context.putToProperties(RENAME_PARTITION_MAKE_COPY, String.valueOf(req.isClonePart()));
context.putToProperties(hive_metastoreConstants.TXN_ID, String.valueOf(req.getTxnId()));
rename_partition(req.getCatName(), req.getDbName(), req.getTableName(), req.getPartVals(),
req.getNewPart(), context, req.getValidWriteIdList());
return new RenamePartitionResponse();
};
private void rename_partition(String catName, String db_name, String tbl_name,
List<String> part_vals, Partition new_part, EnvironmentContext envContext,
String validWriteIds) throws TException {
startTableFunction("alter_partition", catName, db_name, tbl_name);
if (LOG.isInfoEnabled()) {
LOG.info("New partition values:" + new_part.getValues());
if (part_vals != null && part_vals.size() > 0) {
LOG.info("Old Partition values:" + part_vals);
}
}
// Adds the missing scheme/authority for the new partition location
if (new_part.getSd() != null) {
String newLocation = new_part.getSd().getLocation();
if (org.apache.commons.lang3.StringUtils.isNotEmpty(newLocation)) {
Path tblPath = wh.getDnsPath(new Path(newLocation));
new_part.getSd().setLocation(tblPath.toString());
}
}
// Make sure the new partition has the catalog value set
if (!new_part.isSetCatName()) {
new_part.setCatName(catName);
}
Partition oldPart = null;
Exception ex = null;
try {
Table table = null;
table = getMS().getTable(catName, db_name, tbl_name, null);
firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table, part_vals, new_part, this));
if (part_vals != null && !part_vals.isEmpty()) {
MetaStoreServerUtils.validatePartitionNameCharacters(new_part.getValues(),
partitionValidationPattern);
}
oldPart = alterHandler.alterPartition(getMS(), wh, catName, db_name, tbl_name,
part_vals, new_part, envContext, this, validWriteIds);
// Only fetch the table if we actually have a listener
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_PARTITION,
new AlterPartitionEvent(oldPart, new_part, table, false,
true, new_part.getWriteId(), this),
envContext);
}
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidOperationException.class)
.convertIfInstance(InvalidObjectException.class, InvalidOperationException.class)
.convertIfInstance(AlreadyExistsException.class, InvalidOperationException.class)
.defaultMetaException();
} finally {
endFunction("alter_partition", oldPart != null, ex, tbl_name);
}
}
@Override
public void alter_partitions(final String db_name, final String tbl_name,
final List<Partition> new_parts)
throws TException {
String[] o = parseDbName(db_name, conf);
alter_partitions_with_environment_context(o[0], o[1],
tbl_name, new_parts, null, null, -1);
}
@Override
public AlterPartitionsResponse alter_partitions_req(AlterPartitionsRequest req) throws TException {
alter_partitions_with_environment_context(req.getCatName(),
req.getDbName(), req.getTableName(), req.getPartitions(), req.getEnvironmentContext(),
req.isSetValidWriteIdList() ? req.getValidWriteIdList() : null,
req.isSetWriteId() ? req.getWriteId() : -1);
return new AlterPartitionsResponse();
}
// The old API we are keeping for backward compat. Not used within Hive.
@Deprecated
@Override
public void alter_partitions_with_environment_context(final String db_name, final String tbl_name,
final List<Partition> new_parts, EnvironmentContext environmentContext)
throws TException {
String[] o = parseDbName(db_name, conf);
alter_partitions_with_environment_context(o[0], o[1], tbl_name, new_parts, environmentContext,
null, -1);
}
private void alter_partitions_with_environment_context(String catName, String db_name, final String tbl_name,
final List<Partition> new_parts, EnvironmentContext environmentContext,
String writeIdList, long writeId)
throws TException {
if (environmentContext == null) {
environmentContext = new EnvironmentContext();
}
if (catName == null) {
catName = getDefaultCatalog(conf);
}
startTableFunction("alter_partitions", catName, db_name, tbl_name);
if (LOG.isInfoEnabled()) {
for (Partition tmpPart : new_parts) {
LOG.info("New partition values:" + tmpPart.getValues());
}
}
// all partitions are altered atomically
// all prehooks are fired together followed by all post hooks
List<Partition> oldParts = null;
Exception ex = null;
Lock tableLock = getTableLockFor(db_name, tbl_name);
tableLock.lock();
try {
Table table = null;
table = getMS().getTable(catName, db_name, tbl_name, null);
for (Partition tmpPart : new_parts) {
// Make sure the catalog name is set in the new partition
if (!tmpPart.isSetCatName()) {
tmpPart.setCatName(getDefaultCatalog(conf));
}
firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table, null, tmpPart, this));
}
oldParts = alterHandler.alterPartitions(getMS(), wh,
catName, db_name, tbl_name, new_parts, environmentContext, writeIdList, writeId, this);
Iterator<Partition> olditr = oldParts.iterator();
for (Partition tmpPart : new_parts) {
Partition oldTmpPart;
if (olditr.hasNext()) {
oldTmpPart = olditr.next();
}
else {
throw new InvalidOperationException("failed to alterpartitions");
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.ALTER_PARTITION,
new AlterPartitionEvent(oldTmpPart, tmpPart, table, false,
true, writeId, this));
}
}
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidOperationException.class)
.convertIfInstance(InvalidObjectException.class, InvalidOperationException.class)
.convertIfInstance(AlreadyExistsException.class, InvalidOperationException.class)
.defaultMetaException();
} finally {
tableLock.unlock();
endFunction("alter_partitions", oldParts != null, ex, tbl_name);
}
}
@Override
public String getVersion() throws TException {
String version = MetastoreVersionInfo.getVersion();
endFunction(startFunction("getVersion"), version != null, null);
return version;
}
@Override
public void alter_table(final String dbname, final String name,
final Table newTable)
throws InvalidOperationException, MetaException {
// Do not set an environment context.
String[] parsedDbName = parseDbName(dbname, conf);
alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable,
null, null, null, null, null, null);
}
@Override
public void alter_table_with_cascade(final String dbname, final String name,
final Table newTable, final boolean cascade)
throws InvalidOperationException, MetaException {
EnvironmentContext envContext = null;
if (cascade) {
envContext = new EnvironmentContext();
envContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);
}
String[] parsedDbName = parseDbName(dbname, conf);
alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, newTable,
envContext, null, null, null, null, null);
}
@Override
public AlterTableResponse alter_table_req(AlterTableRequest req)
throws InvalidOperationException, MetaException, TException {
alter_table_core(req.getCatName(), req.getDbName(), req.getTableName(),
req.getTable(), req.getEnvironmentContext(), req.getValidWriteIdList(),
req.getProcessorCapabilities(), req.getProcessorIdentifier(),
req.getExpectedParameterKey(), req.getExpectedParameterValue());
return new AlterTableResponse();
}
@Override
public void alter_table_with_environment_context(final String dbname,
final String name, final Table newTable,
final EnvironmentContext envContext)
throws InvalidOperationException, MetaException {
String[] parsedDbName = parseDbName(dbname, conf);
alter_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
name, newTable, envContext, null, null, null, null, null);
}
private void alter_table_core(String catName, String dbname, String name, Table newTable,
EnvironmentContext envContext, String validWriteIdList, List<String> processorCapabilities,
String processorId, String expectedPropertyKey, String expectedPropertyValue)
throws InvalidOperationException, MetaException {
startFunction("alter_table", ": " + TableName.getQualified(catName, dbname, name)
+ " newtbl=" + newTable.getTableName());
if (envContext == null) {
envContext = new EnvironmentContext();
}
// Set the values to the envContext, so we do not have to change the HiveAlterHandler API
if (expectedPropertyKey != null) {
envContext.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, expectedPropertyKey);
}
if (expectedPropertyValue != null) {
envContext.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE, expectedPropertyValue);
}
if (catName == null) {
catName = getDefaultCatalog(conf);
}
// HIVE-25282: Drop/Alter table in REMOTE db should fail
try {
Database db = get_database_core(catName, dbname);
if (db != null && db.getType().equals(DatabaseType.REMOTE)) {
throw new MetaException("Alter table in REMOTE database " + db.getName() + " is not allowed");
}
} catch (NoSuchObjectException e) {
throw new InvalidOperationException("Alter table in REMOTE database is not allowed");
}
// Update the time if it hasn't been specified.
if (newTable.getParameters() == null ||
newTable.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) {
newTable.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
// Adds the missing scheme/authority for the new table location
if (newTable.getSd() != null) {
String newLocation = newTable.getSd().getLocation();
if (org.apache.commons.lang3.StringUtils.isNotEmpty(newLocation)) {
Path tblPath = wh.getDnsPath(new Path(newLocation));
newTable.getSd().setLocation(tblPath.toString());
}
}
// Set the catalog name if it hasn't been set in the new table
if (!newTable.isSetCatName()) {
newTable.setCatName(catName);
}
boolean success = false;
Exception ex = null;
try {
GetTableRequest request = new GetTableRequest(dbname, name);
request.setCatName(catName);
Table oldt = get_table_core(request);
if (transformer != null) {
newTable = transformer.transformAlterTable(oldt, newTable, processorCapabilities, processorId);
}
firePreEvent(new PreAlterTableEvent(oldt, newTable, this));
alterHandler.alterTable(getMS(), wh, catName, dbname, name, newTable,
envContext, this, validWriteIdList);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, InvalidOperationException.class)
.convertIfInstance(NoSuchObjectException.class, InvalidOperationException.class)
.defaultMetaException();
} finally {
endFunction("alter_table", success, ex, name);
}
}
@Override
public List<String> get_tables(final String dbname, final String pattern)
throws MetaException {
startFunction("get_tables", ": db=" + dbname + " pat=" + pattern);
List<String> ret = null;
Exception ex = null;
String[] parsedDbName = parseDbName(dbname, conf);
try {
if (isDatabaseRemote(dbname)) {
Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
}
} catch (Exception e) { /* appears we return empty set instead of throwing an exception */ }
try {
ret = getMS().getTables(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], pattern);
if(ret != null && !ret.isEmpty()) {
List<Table> tableInfo = new ArrayList<>();
tableInfo = getMS().getTableObjectsByName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], ret);
tableInfo = FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tableInfo);// tableInfo object has the owner information of the table which is being passed to FilterUtils.
ret = new ArrayList<>();
for (Table tbl : tableInfo) {
ret.add(tbl.getTableName());
}
}
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_tables", ret != null, ex);
}
return ret;
}
@Override
public List<String> get_tables_by_type(final String dbname, final String pattern, final String tableType)
throws MetaException {
startFunction("get_tables_by_type", ": db=" + dbname + " pat=" + pattern + ",type=" + tableType);
List<String> ret = null;
Exception ex = null;
String[] parsedDbName = parseDbName(dbname, conf);
try {
ret = getTablesByTypeCore(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], pattern, tableType);
ret = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook,
parsedDbName[CAT_NAME], parsedDbName[DB_NAME], ret);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_tables_by_type", ret != null, ex);
}
return ret;
}
private List<String> getTablesByTypeCore(final String catName, final String dbname,
final String pattern, final String tableType) throws MetaException {
startFunction("getTablesByTypeCore", ": catName=" + catName +
": db=" + dbname + " pat=" + pattern + ",type=" + tableType);
List<String> ret = null;
Exception ex = null;
Database db = null;
try {
db = get_database_core(catName, dbname);
if (db != null) {
if (db.getType().equals(DatabaseType.REMOTE)) {
return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
}
}
} catch (Exception e) { /* ignore */ }
try {
ret = getMS().getTables(catName, dbname, pattern, TableType.valueOf(tableType), -1);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("getTablesByTypeCore", ret != null, ex);
}
return ret;
}
@Override
public List<Table> get_all_materialized_view_objects_for_rewriting()
throws MetaException {
startFunction("get_all_materialized_view_objects_for_rewriting");
List<Table> ret = null;
Exception ex = null;
try {
ret = getMS().getAllMaterializedViewObjectsForRewriting(DEFAULT_CATALOG_NAME);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_all_materialized_view_objects_for_rewriting", ret != null, ex);
}
return ret;
}
@Override
public List<String> get_materialized_views_for_rewriting(final String dbname)
throws MetaException {
startFunction("get_materialized_views_for_rewriting", ": db=" + dbname);
List<String> ret = null;
Exception ex = null;
String[] parsedDbName = parseDbName(dbname, conf);
try {
ret = getMS().getMaterializedViewsForRewriting(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_materialized_views_for_rewriting", ret != null, ex);
}
return ret;
}
@Override
public List<String> get_all_tables(final String dbname) throws MetaException {
startFunction("get_all_tables", ": db=" + dbname);
List<String> ret = null;
Exception ex = null;
String[] parsedDbName = parseDbName(dbname, conf);
try {
if (isDatabaseRemote(dbname)) {
Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
}
} catch (Exception e) { /* ignore */ }
try {
ret = getMS().getAllTables(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
ret = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook,
parsedDbName[CAT_NAME], parsedDbName[DB_NAME], ret);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_all_tables", ret != null, ex);
}
return ret;
}
/**
* Use {@link #get_fields_req(GetFieldsRequest)} ()} instead.
*
*/
@Override
@Deprecated
public List<FieldSchema> get_fields(String db, String tableName)
throws MetaException, UnknownTableException, UnknownDBException {
return get_fields_with_environment_context(db, tableName, null);
}
@Override
@Deprecated
public List<FieldSchema> get_fields_with_environment_context(String db, String tableName,
final EnvironmentContext envContext)
throws MetaException, UnknownTableException, UnknownDBException {
startFunction("get_fields_with_environment_context", ": db=" + db + "tbl=" + tableName);
String[] names = tableName.split("\\.");
String base_table_name = names[0];
String[] parsedDbName = parseDbName(db, conf);
Table tbl;
List<FieldSchema> ret = null;
Exception ex = null;
try {
try {
tbl = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], base_table_name);
firePreEvent(new PreReadTableEvent(tbl, this));
} catch (NoSuchObjectException e) {
throw new UnknownTableException(e.getMessage());
}
if (null == tbl.getSd().getSerdeInfo().getSerializationLib() ||
MetastoreConf.getStringCollection(conf,
ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(
tbl.getSd().getSerdeInfo().getSerializationLib())) {
ret = tbl.getSd().getCols();
} else {
StorageSchemaReader schemaReader = getStorageSchemaReader();
ret = schemaReader.readSchema(tbl, envContext, getConf());
}
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(UnknownTableException.class, MetaException.class).defaultMetaException();
} finally {
endFunction("get_fields_with_environment_context", ret != null, ex, tableName);
}
return ret;
}
@Override
public GetFieldsResponse get_fields_req(GetFieldsRequest req)
throws MetaException, UnknownTableException, UnknownDBException, TException {
String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
List<FieldSchema> fields = get_fields_with_environment_context(
dbName, req.getTblName(), req.getEnvContext());
GetFieldsResponse res = new GetFieldsResponse();
res.setFields(fields);
return res;
}
private StorageSchemaReader getStorageSchemaReader() throws MetaException {
if (storageSchemaReader == null) {
String className =
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL);
Class<? extends StorageSchemaReader> readerClass =
JavaUtils.getClass(className, StorageSchemaReader.class);
try {
storageSchemaReader = readerClass.newInstance();
} catch (InstantiationException|IllegalAccessException e) {
LOG.error("Unable to instantiate class " + className, e);
throw new MetaException(e.getMessage());
}
}
return storageSchemaReader;
}
/**
* Use {@link #get_schema_req(GetSchemaRequest)} ()} instead.
*
*/
@Override
@Deprecated
public List<FieldSchema> get_schema(String db, String tableName)
throws MetaException, UnknownTableException, UnknownDBException {
return get_schema_with_environment_context(db,tableName, null);
}
/**
* Return the schema of the table. This function includes partition columns
* in addition to the regular columns.
*
* @param db
* Name of the database
* @param tableName
* Name of the table
* @param envContext
* Store session based properties
* @return List of columns, each column is a FieldSchema structure
* @throws MetaException
* @throws UnknownTableException
* @throws UnknownDBException
*/
@Override
@Deprecated
public List<FieldSchema> get_schema_with_environment_context(String db, String tableName,
final EnvironmentContext envContext)
throws MetaException, UnknownTableException, UnknownDBException {
startFunction("get_schema_with_environment_context", ": db=" + db + "tbl=" + tableName);
boolean success = false;
Exception ex = null;
try {
String[] names = tableName.split("\\.");
String base_table_name = names[0];
String[] parsedDbName = parseDbName(db, conf);
Table tbl;
try {
tbl = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], base_table_name);
} catch (NoSuchObjectException e) {
throw new UnknownTableException(e.getMessage());
}
// Pass unparsed db name here
List<FieldSchema> fieldSchemas = get_fields_with_environment_context(db, base_table_name,
envContext);
if (tbl == null || fieldSchemas == null) {
throw new UnknownTableException(tableName + " doesn't exist");
}
if (tbl.getPartitionKeys() != null) {
// Combine the column field schemas and the partition keys to create the
// whole schema
fieldSchemas.addAll(tbl.getPartitionKeys());
}
success = true;
return fieldSchemas;
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(UnknownDBException.class, UnknownTableException.class, MetaException.class)
.defaultMetaException();
} finally {
endFunction("get_schema_with_environment_context", success, ex, tableName);
}
}
@Override
public GetSchemaResponse get_schema_req(GetSchemaRequest req)
throws MetaException, UnknownTableException, UnknownDBException, TException {
String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
List<FieldSchema> fields = get_schema_with_environment_context(
dbName, req.getTblName(), req.getEnvContext());
GetSchemaResponse res = new GetSchemaResponse();
res.setFields(fields);
return res;
}
@Override
public String getCpuProfile(int profileDurationInSec) throws TException {
return "";
}
/**
* Returns the value of the given configuration variable name. If the
* configuration variable with the given name doesn't exist, or if there
* were an exception thrown while retrieving the variable, or if name is
* null, defaultValue is returned.
*/
@Override
public String get_config_value(String name, String defaultValue)
throws TException {
startFunction("get_config_value", ": name=" + name + " defaultValue="
+ defaultValue);
boolean success = false;
Exception ex = null;
try {
if (name == null) {
success = true;
return defaultValue;
}
// Allow only keys that start with hive.*, hdfs.*, mapred.* for security
// i.e. don't allow access to db password
if (!Pattern.matches("(hive|hdfs|mapred|metastore).*", name)) {
throw new ConfigValSecurityException("For security reasons, the "
+ "config key " + name + " cannot be accessed");
}
String toReturn = defaultValue;
try {
toReturn = MetastoreConf.get(conf, name);
if (toReturn == null) {
toReturn = defaultValue;
}
} catch (RuntimeException e) {
LOG.error("RuntimeException thrown in get_config_value - msg: "
+ e.getMessage() + " cause: " + e.getCause());
}
success = true;
return toReturn;
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(TException.class).defaultMetaException();
} finally {
endFunction("get_config_value", success, ex);
}
}
public static List<String> getPartValsFromName(Table t, String partName)
throws MetaException, InvalidObjectException {
Preconditions.checkArgument(t != null, "Table can not be null");
// Unescape the partition name
LinkedHashMap<String, String> hm = Warehouse.makeSpecFromName(partName);
List<String> partVals = new ArrayList<>();
for (FieldSchema field : t.getPartitionKeys()) {
String key = field.getName();
String val = hm.get(key);
if (val == null) {
throw new InvalidObjectException("incomplete partition name - missing " + key);
}
partVals.add(val);
}
return partVals;
}
private List<String> getPartValsFromName(RawStore ms, String catName, String dbName,
String tblName, String partName)
throws MetaException, InvalidObjectException {
Table t = ms.getTable(catName, dbName, tblName, null);
if (t == null) {
throw new InvalidObjectException(dbName + "." + tblName
+ " table not found");
}
return getPartValsFromName(t, partName);
}
private Partition get_partition_by_name_core(final RawStore ms, final String catName,
final String db_name, final String tbl_name,
final String part_name) throws TException {
fireReadTablePreEvent(catName, db_name, tbl_name);
List<String> partVals;
try {
partVals = getPartValsFromName(ms, catName, db_name, tbl_name, part_name);
} catch (InvalidObjectException e) {
throw new NoSuchObjectException(e.getMessage());
}
Partition p = ms.getPartition(catName, db_name, tbl_name, partVals);
p = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, p);
if (p == null) {
throw new NoSuchObjectException(TableName.getQualified(catName, db_name, tbl_name)
+ " partition (" + part_name + ") not found");
}
return p;
}
@Override
@Deprecated
public Partition get_partition_by_name(final String db_name, final String tbl_name,
final String part_name) throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startFunction("get_partition_by_name", ": tbl=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name)
+ " part=" + part_name);
Partition ret = null;
Exception ex = null;
try {
ret = get_partition_by_name_core(getMS(), parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tbl_name, part_name);
ret = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partition_by_name", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public Partition append_partition_by_name(final String db_name, final String tbl_name,
final String part_name) throws TException {
return append_partition_by_name_with_environment_context(db_name, tbl_name, part_name, null);
}
@Override
public Partition append_partition_by_name_with_environment_context(final String db_name,
final String tbl_name, final String part_name, final EnvironmentContext env_context)
throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startFunction("append_partition_by_name", ": tbl="
+ TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name) + " part=" + part_name);
Partition ret = null;
Exception ex = null;
try {
RawStore ms = getMS();
List<String> partVals = getPartValsFromName(ms, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_name);
ret = append_partition_common(ms, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, partVals, env_context);
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(InvalidObjectException.class, AlreadyExistsException.class, MetaException.class)
.defaultMetaException();
} finally {
endFunction("append_partition_by_name", ret != null, ex, tbl_name);
}
return ret;
}
private boolean drop_partition_by_name_core(final RawStore ms, final String catName,
final String db_name, final String tbl_name,
final String part_name, final boolean deleteData,
final EnvironmentContext envContext)
throws TException, IOException {
List<String> partVals;
try {
partVals = getPartValsFromName(ms, catName, db_name, tbl_name, part_name);
} catch (InvalidObjectException e) {
throw new NoSuchObjectException(e.getMessage());
}
return drop_partition_common(ms, catName, db_name, tbl_name, partVals, deleteData, envContext);
}
@Override
public boolean drop_partition_by_name(final String db_name, final String tbl_name,
final String part_name, final boolean deleteData) throws TException {
return drop_partition_by_name_with_environment_context(db_name, tbl_name, part_name,
deleteData, null);
}
@Override
public boolean drop_partition_by_name_with_environment_context(final String db_name,
final String tbl_name, final String part_name, final boolean deleteData,
final EnvironmentContext envContext) throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startFunction("drop_partition_by_name", ": tbl=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name)
+ " part=" + part_name);
boolean ret = false;
Exception ex = null;
try {
ret = drop_partition_by_name_core(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_name, deleteData, envContext);
} catch (Exception e) {
ex = e;
handleException(e).convertIfInstance(IOException.class, MetaException.class).rethrowException(e);
} finally {
endFunction("drop_partition_by_name", ret, ex, tbl_name);
}
return ret;
}
@Override
@Deprecated
public List<Partition> get_partitions_ps(final String db_name,
final String tbl_name, final List<String> part_vals,
final short max_parts) throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("get_partitions_ps", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals);
List<Partition> ret = null;
Exception ex = null;
try {
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
// Don't send the parsedDbName, as this method will parse itself.
ret = get_partitions_ps_with_auth(db_name, tbl_name, part_vals,
max_parts, null, null);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partitions_ps", ret != null, ex, tbl_name);
}
return ret;
}
/**
* Use {@link #get_partitions_ps_with_auth_req(GetPartitionsPsWithAuthRequest)} ()} instead.
*
*/
@Override
@Deprecated
public List<Partition> get_partitions_ps_with_auth(final String db_name,
final String tbl_name, final List<String> part_vals,
final short max_parts, final String userName,
final List<String> groupNames) throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("get_partitions_ps_with_auth", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tbl_name, part_vals);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
List<Partition> ret = null;
Exception ex = null;
try {
checkLimitNumberOfPartitionsByPs(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals, max_parts);
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
ret = getMS().listPartitionsPsWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, part_vals, max_parts, userName, groupNames);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
handleException(e).convertIfInstance(InvalidObjectException.class, MetaException.class).rethrowException(e);
} finally {
endFunction("get_partitions_ps_with_auth", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public GetPartitionsPsWithAuthResponse get_partitions_ps_with_auth_req(GetPartitionsPsWithAuthRequest req)
throws MetaException, NoSuchObjectException, TException {
String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
List<Partition> partitions = null;
if (req.getPartVals() == null) {
partitions = get_partitions_with_auth(dbName, req.getTblName(), req.getMaxParts(), req.getUserName(),
req.getGroupNames());
} else {
partitions =
get_partitions_ps_with_auth(dbName, req.getTblName(), req.getPartVals(), req.getMaxParts(),
req.getUserName(), req.getGroupNames());
}
GetPartitionsPsWithAuthResponse res = new GetPartitionsPsWithAuthResponse();
res.setPartitions(partitions);
return res;
}
/**
* Use {@link #get_partition_names_ps_req(GetPartitionNamesPsRequest)} ()} instead.
*
*/
@Override
@Deprecated
public List<String> get_partition_names_ps(final String db_name,
final String tbl_name, final List<String> part_vals, final short max_parts)
throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("get_partitions_names_ps", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tbl_name, part_vals);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
List<String> ret = null;
Exception ex = null;
try {
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
ret = getMS().listPartitionNamesPs(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
part_vals, max_parts);
ret = FilterUtils.filterPartitionNamesIfEnabled(isServerFilterEnabled,
filterHook, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, ret);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partitions_names_ps", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public GetPartitionNamesPsResponse get_partition_names_ps_req(GetPartitionNamesPsRequest req)
throws MetaException, NoSuchObjectException, TException {
String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
List<String> names = get_partition_names_ps(dbName, req.getTblName(), req.getPartValues(),
req.getMaxParts());
GetPartitionNamesPsResponse res = new GetPartitionNamesPsResponse();
res.setNames(names);
return res;
}
@Override
public List<String> get_partition_names_req(PartitionsByExprRequest req)
throws MetaException, NoSuchObjectException, TException {
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
String dbName = req.getDbName(), tblName = req.getTblName();
startTableFunction("get_partition_names_req", catName,
dbName, tblName);
fireReadTablePreEvent(catName, dbName, tblName);
List<String> ret = null;
Exception ex = null;
try {
authorizeTableForPartitionMetadata(catName, dbName, tblName);
ret = getMS().listPartitionNames(catName, dbName, tblName,
req.getDefaultPartitionName(), req.getExpr(), req.getOrder(), req.getMaxParts());
ret = FilterUtils.filterPartitionNamesIfEnabled(isServerFilterEnabled,
filterHook, catName, dbName, tblName, ret);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partition_names_req", ret != null, ex, tblName);
}
return ret;
}
@Override
public List<String> partition_name_to_vals(String part_name) throws TException {
if (part_name.length() == 0) {
return Collections.emptyList();
}
LinkedHashMap<String, String> map = Warehouse.makeSpecFromName(part_name);
return new ArrayList<>(map.values());
}
@Override
public Map<String, String> partition_name_to_spec(String part_name) throws TException {
if (part_name.length() == 0) {
return new HashMap<>();
}
return Warehouse.makeSpecFromName(part_name);
}
public static String lowerCaseConvertPartName(String partName) throws MetaException {
if (partName == null) {
return partName;
}
boolean isFirst = true;
Map<String, String> partSpec = Warehouse.makeEscSpecFromName(partName);
String convertedPartName = new String();
for (Map.Entry<String, String> entry : partSpec.entrySet()) {
String partColName = entry.getKey();
String partColVal = entry.getValue();
if (!isFirst) {
convertedPartName += "/";
} else {
isFirst = false;
}
convertedPartName += partColName.toLowerCase() + "=" + partColVal;
}
return convertedPartName;
}
@Override
@Deprecated
public ColumnStatistics get_table_column_statistics(String dbName, String tableName,
String colName) throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
parsedDbName[CAT_NAME] = parsedDbName[CAT_NAME].toLowerCase();
parsedDbName[DB_NAME] = parsedDbName[DB_NAME].toLowerCase();
tableName = tableName.toLowerCase();
colName = colName.toLowerCase();
startFunction("get_column_statistics_by_table", ": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName) + " column=" + colName);
ColumnStatistics statsObj = null;
try {
statsObj = getMS().getTableColumnStatistics(
parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(colName),
"hive", null);
if (statsObj != null) {
assert statsObj.getStatsObjSize() <= 1;
}
return statsObj;
} finally {
endFunction("get_column_statistics_by_table", statsObj != null, null, tableName);
}
}
@Override
public TableStatsResult get_table_statistics_req(TableStatsRequest request) throws TException {
String catName = request.isSetCatName() ? request.getCatName().toLowerCase() :
getDefaultCatalog(conf);
String dbName = request.getDbName().toLowerCase();
String tblName = request.getTblName().toLowerCase();
startFunction("get_table_statistics_req", ": table=" +
TableName.getQualified(catName, dbName, tblName));
TableStatsResult result = null;
List<String> lowerCaseColNames = new ArrayList<>(request.getColNames().size());
for (String colName : request.getColNames()) {
lowerCaseColNames.add(colName.toLowerCase());
}
try {
ColumnStatistics cs = getMS().getTableColumnStatistics(
catName, dbName, tblName, lowerCaseColNames,
request.getEngine(), request.getValidWriteIdList());
// Note: stats compliance is not propagated to the client; instead, we just return nothing
// if stats are not compliant for now. This won't work for stats merging, but that
// is currently only done on metastore size (see set_aggr...).
// For some optimizations we might make use of incorrect stats that are "better than
// nothing", so this may change in future.
result = new TableStatsResult((cs == null || cs.getStatsObj() == null
|| (cs.isSetIsStatsCompliant() && !cs.isIsStatsCompliant()))
? Lists.newArrayList() : cs.getStatsObj());
} finally {
endFunction("get_table_statistics_req", result == null, null, tblName);
}
return result;
}
@Override
@Deprecated
public ColumnStatistics get_partition_column_statistics(String dbName, String tableName,
String partName, String colName) throws TException {
// Note: this method appears to be unused within Hive.
// It doesn't take txn stats into account.
dbName = dbName.toLowerCase();
String[] parsedDbName = parseDbName(dbName, conf);
tableName = tableName.toLowerCase();
colName = colName.toLowerCase();
String convertedPartName = lowerCaseConvertPartName(partName);
startFunction("get_column_statistics_by_partition", ": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName) + " partition=" + convertedPartName + " column=" + colName);
ColumnStatistics statsObj = null;
try {
List<ColumnStatistics> list = getMS().getPartitionColumnStatistics(
parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
Lists.newArrayList(convertedPartName), Lists.newArrayList(colName),
"hive");
if (list.isEmpty()) {
return null;
}
if (list.size() != 1) {
throw new MetaException(list.size() + " statistics for single column and partition");
}
statsObj = list.get(0);
} finally {
endFunction("get_column_statistics_by_partition", statsObj != null, null, tableName);
}
return statsObj;
}
@Override
public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request)
throws TException {
String catName = request.isSetCatName() ? request.getCatName().toLowerCase() : getDefaultCatalog(conf);
String dbName = request.getDbName().toLowerCase();
String tblName = request.getTblName().toLowerCase();
startFunction("get_partitions_statistics_req", ": table=" +
TableName.getQualified(catName, dbName, tblName));
PartitionsStatsResult result = null;
List<String> lowerCaseColNames = new ArrayList<>(request.getColNames().size());
for (String colName : request.getColNames()) {
lowerCaseColNames.add(colName.toLowerCase());
}
List<String> lowerCasePartNames = new ArrayList<>(request.getPartNames().size());
for (String partName : request.getPartNames()) {
lowerCasePartNames.add(lowerCaseConvertPartName(partName));
}
try {
List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics(
catName, dbName, tblName, lowerCasePartNames, lowerCaseColNames,
request.getEngine(), request.isSetValidWriteIdList() ? request.getValidWriteIdList() : null);
Map<String, List<ColumnStatisticsObj>> map = new HashMap<>();
if (stats != null) {
for (ColumnStatistics stat : stats) {
// Note: stats compliance is not propagated to the client; instead, we just return nothing
// if stats are not compliant for now. This won't work for stats merging, but that
// is currently only done on metastore size (see set_aggr...).
// For some optimizations we might make use of incorrect stats that are "better than
// nothing", so this may change in future.
if (stat.isSetIsStatsCompliant() && !stat.isIsStatsCompliant()) {
continue;
}
map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj());
}
}
result = new PartitionsStatsResult(map);
} finally {
endFunction("get_partitions_statistics_req", result == null, null, tblName);
}
return result;
}
@Override
public boolean update_table_column_statistics(ColumnStatistics colStats) throws TException {
// Deprecated API, won't work for transactional tables
return updateTableColumnStatsInternal(colStats, null, -1);
}
@Override
public SetPartitionsStatsResponse update_table_column_statistics_req(
SetPartitionsStatsRequest req) throws NoSuchObjectException,
InvalidObjectException, MetaException, InvalidInputException,
TException {
if (req.getColStatsSize() != 1) {
throw new InvalidInputException("Only one stats object expected");
}
if (req.isNeedMerge()) {
throw new InvalidInputException("Merge is not supported for non-aggregate stats");
}
ColumnStatistics colStats = req.getColStatsIterator().next();
boolean ret = updateTableColumnStatsInternal(colStats,
req.getValidWriteIdList(), req.getWriteId());
return new SetPartitionsStatsResponse(ret);
}
private boolean updateTableColumnStatsInternal(ColumnStatistics colStats,
String validWriteIds, long writeId)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
normalizeColStatsInput(colStats);
startFunction("write_column_statistics", ": table=" + TableName.getQualified(
colStats.getStatsDesc().getCatName(), colStats.getStatsDesc().getDbName(),
colStats.getStatsDesc().getTableName()));
Map<String, String> parameters = null;
getMS().openTransaction();
boolean committed = false;
try {
parameters = getMS().updateTableColumnStatistics(colStats, validWriteIds, writeId);
if (parameters != null) {
Table tableObj = getMS().getTable(colStats.getStatsDesc().getCatName(),
colStats.getStatsDesc().getDbName(),
colStats.getStatsDesc().getTableName(), validWriteIds);
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.UPDATE_TABLE_COLUMN_STAT,
new UpdateTableColumnStatEvent(colStats, tableObj, parameters,
writeId, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.UPDATE_TABLE_COLUMN_STAT,
new UpdateTableColumnStatEvent(colStats, tableObj, parameters,
writeId,this));
}
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
getMS().rollbackTransaction();
}
endFunction("write_column_statistics", parameters != null, null,
colStats.getStatsDesc().getTableName());
}
return parameters != null;
}
private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException {
// TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless.
ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf));
statsDesc.setDbName(statsDesc.getDbName().toLowerCase());
statsDesc.setTableName(statsDesc.getTableName().toLowerCase());
statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName()));
long time = System.currentTimeMillis() / 1000;
statsDesc.setLastAnalyzed(time);
for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) {
statsObj.setColName(statsObj.getColName().toLowerCase());
statsObj.setColType(statsObj.getColType().toLowerCase());
}
colStats.setStatsDesc(statsDesc);
colStats.setStatsObj(colStats.getStatsObj());
}
private boolean updatePartitonColStatsInternal(Table tbl, MTable mTable, ColumnStatistics colStats,
String validWriteIds, long writeId)
throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
normalizeColStatsInput(colStats);
ColumnStatisticsDesc csd = colStats.getStatsDesc();
String catName = csd.getCatName(), dbName = csd.getDbName(), tableName = csd.getTableName();
startFunction("write_partition_column_statistics", ": db=" + dbName + " table=" + tableName
+ " part=" + csd.getPartName());
boolean ret = false;
Map<String, String> parameters;
List<String> partVals;
boolean committed = false;
getMS().openTransaction();
try {
tbl = Optional.ofNullable(tbl).orElse(getTable(catName, dbName, tableName));
mTable = Optional.ofNullable(mTable).orElse(getMS().ensureGetMTable(catName, dbName, tableName));
partVals = getPartValsFromName(tbl, csd.getPartName());
parameters = getMS().updatePartitionColumnStatistics(tbl, mTable, colStats, partVals, validWriteIds, writeId);
if (parameters != null) {
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.UPDATE_PARTITION_COLUMN_STAT,
new UpdatePartitionColumnStatEvent(colStats, partVals, parameters, tbl,
writeId, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.UPDATE_PARTITION_COLUMN_STAT,
new UpdatePartitionColumnStatEvent(colStats, partVals, parameters, tbl,
writeId, this));
}
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
getMS().rollbackTransaction();
}
endFunction("write_partition_column_statistics", ret != false, null, tableName);
}
return parameters != null;
}
private void updatePartitionColStatsForOneBatch(Table tbl, Map<String, ColumnStatistics> statsMap,
String validWriteIds, long writeId)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
Map<String, Map<String, String>> result =
getMS().updatePartitionColumnStatisticsInBatch(statsMap, tbl, transactionalListeners, validWriteIds, writeId);
if (result != null && result.size() != 0 && listeners != null) {
// The normal listeners, unlike transaction listeners are not using the same transactions used by the update
// operations. So there is no need of keeping them within the same transactions. If notification to one of
// the listeners failed, then even if we abort the transaction, we can not revert the notifications sent to the
// other listeners.
for (Map.Entry entry : result.entrySet()) {
Map<String, String> parameters = (Map<String, String>) entry.getValue();
ColumnStatistics colStats = statsMap.get(entry.getKey());
List<String> partVals = getPartValsFromName(tbl, colStats.getStatsDesc().getPartName());
MetaStoreListenerNotifier.notifyEvent(listeners,
EventMessage.EventType.UPDATE_PARTITION_COLUMN_STAT,
new UpdatePartitionColumnStatEvent(colStats, partVals, parameters,
tbl, writeId, this));
}
}
}
private boolean updatePartitionColStatsInBatch(Table tbl, Map<String, ColumnStatistics> statsMap,
String validWriteIds, long writeId)
throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
if (statsMap.size() == 0) {
return false;
}
String catalogName = tbl.getCatName();
String dbName = tbl.getDbName();
String tableName = tbl.getTableName();
startFunction("updatePartitionColStatsInBatch", ": db=" + dbName + " table=" + tableName);
long start = System.currentTimeMillis();
Map<String, ColumnStatistics> newStatsMap = new HashMap<>();
long numStats = 0;
long numStatsMax = MetastoreConf.getIntVar(conf, ConfVars.JDBC_MAX_BATCH_SIZE);
try {
for (Map.Entry entry : statsMap.entrySet()) {
ColumnStatistics colStats = (ColumnStatistics) entry.getValue();
normalizeColStatsInput(colStats);
assert catalogName.equalsIgnoreCase(colStats.getStatsDesc().getCatName());
assert dbName.equalsIgnoreCase(colStats.getStatsDesc().getDbName());
assert tableName.equalsIgnoreCase(colStats.getStatsDesc().getTableName());
newStatsMap.put((String) entry.getKey(), colStats);
numStats += colStats.getStatsObjSize();
if (newStatsMap.size() >= numStatsMax) {
updatePartitionColStatsForOneBatch(tbl, newStatsMap, validWriteIds, writeId);
newStatsMap.clear();
numStats = 0;
}
}
if (numStats != 0) {
updatePartitionColStatsForOneBatch(tbl, newStatsMap, validWriteIds, writeId);
}
} finally {
endFunction("updatePartitionColStatsInBatch", true, null, tableName);
long end = System.currentTimeMillis();
float sec = (end - start) / 1000F;
LOG.info("updatePartitionColStatsInBatch took " + sec + " seconds for " + statsMap.size() + " stats");
}
return true;
}
@Override
public boolean update_partition_column_statistics(ColumnStatistics colStats) throws TException {
// Deprecated API.
return updatePartitonColStatsInternal(null, null, colStats, null, -1);
}
@Override
public SetPartitionsStatsResponse update_partition_column_statistics_req(
SetPartitionsStatsRequest req) throws NoSuchObjectException,
InvalidObjectException, MetaException, InvalidInputException,
TException {
if (req.getColStatsSize() != 1) {
throw new InvalidInputException("Only one stats object expected");
}
if (req.isNeedMerge()) {
throw new InvalidInputException("Merge is not supported for non-aggregate stats");
}
ColumnStatistics colStats = req.getColStatsIterator().next();
boolean ret = updatePartitonColStatsInternal(null, null, colStats,
req.getValidWriteIdList(), req.getWriteId());
return new SetPartitionsStatsResponse(ret);
}
@Override
public boolean delete_partition_column_statistics(String dbName, String tableName,
String partName, String colName, String engine) throws TException {
dbName = dbName.toLowerCase();
String[] parsedDbName = parseDbName(dbName, conf);
tableName = tableName.toLowerCase();
if (colName != null) {
colName = colName.toLowerCase();
}
String convertedPartName = lowerCaseConvertPartName(partName);
startFunction("delete_column_statistics_by_partition",": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) +
" partition=" + convertedPartName + " column=" + colName);
boolean ret = false, committed = false;
getMS().openTransaction();
try {
List<String> partVals = getPartValsFromName(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, convertedPartName);
Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
// This API looks unused; if it were used we'd need to update stats state and write ID.
// We cannot just randomly nuke some txn stats.
if (TxnUtils.isTransactionalTable(table)) {
throw new MetaException("Cannot delete stats via this API for a transactional table");
}
ret = getMS().deletePartitionColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
convertedPartName, partVals, colName, engine);
if (ret) {
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DELETE_PARTITION_COLUMN_STAT,
new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
convertedPartName, partVals, colName, engine, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DELETE_PARTITION_COLUMN_STAT,
new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
convertedPartName, partVals, colName, engine, this));
}
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
getMS().rollbackTransaction();
}
endFunction("delete_column_statistics_by_partition", ret != false, null, tableName);
}
return ret;
}
@Override
public boolean delete_table_column_statistics(String dbName, String tableName, String colName, String engine)
throws TException {
dbName = dbName.toLowerCase();
tableName = tableName.toLowerCase();
String[] parsedDbName = parseDbName(dbName, conf);
if (colName != null) {
colName = colName.toLowerCase();
}
startFunction("delete_column_statistics_by_table", ": table=" +
TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) + " column=" +
colName);
boolean ret = false, committed = false;
getMS().openTransaction();
try {
Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName);
// This API looks unused; if it were used we'd need to update stats state and write ID.
// We cannot just randomly nuke some txn stats.
if (TxnUtils.isTransactionalTable(table)) {
throw new MetaException("Cannot delete stats via this API for a transactional table");
}
ret = getMS().deleteTableColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colName, engine);
if (ret) {
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DELETE_TABLE_COLUMN_STAT,
new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName, colName, engine, this));
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DELETE_TABLE_COLUMN_STAT,
new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tableName, colName, engine, this));
}
}
committed = getMS().commitTransaction();
} finally {
if (!committed) {
getMS().rollbackTransaction();
}
endFunction("delete_column_statistics_by_table", ret != false, null, tableName);
}
return ret;
}
@Override
public void update_transaction_statistics(UpdateTransactionalStatsRequest req) throws TException {
getTxnHandler().updateTransactionStatistics(req);
}
@Override
@Deprecated
public List<Partition> get_partitions_by_filter(final String dbName, final String tblName,
final String filter, final short maxParts)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
startTableFunction("get_partitions_by_filter", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tblName);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
List<Partition> ret = null;
Exception ex = null;
try {
checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tblName, filter, maxParts);
authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
ret = getMS().getPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName,
filter, maxParts);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partitions_by_filter", ret != null, ex, tblName);
}
return ret;
}
@Override
@Deprecated
public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName,
final String filter, final int maxParts)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
startTableFunction("get_partitions_by_filter_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
List<PartitionSpec> partitionSpecs = null;
try {
Table table = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
// Don't pass the parsed db name, as get_partitions_by_filter will parse it itself
List<Partition> partitions = get_partitions_by_filter(dbName, tblName, filter, (short) maxParts);
if (is_partition_spec_grouping_enabled(table)) {
partitionSpecs = MetaStoreServerUtils
.getPartitionspecsGroupedByStorageDescriptor(table, partitions);
}
else {
PartitionSpec pSpec = new PartitionSpec();
pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
pSpec.setRootPath(table.getSd().getLocation());
pSpec.setCatName(parsedDbName[CAT_NAME]);
pSpec.setDbName(parsedDbName[DB_NAME]);
pSpec.setTableName(tblName);
partitionSpecs = Arrays.asList(pSpec);
}
return partitionSpecs;
}
finally {
endFunction("get_partitions_by_filter_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tblName);
}
}
@Override
public PartitionsSpecByExprResult get_partitions_spec_by_expr(
PartitionsByExprRequest req) throws TException {
String dbName = req.getDbName(), tblName = req.getTblName();
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
startTableFunction("get_partitions_spec_by_expr", catName, dbName, tblName);
fireReadTablePreEvent(catName, dbName, tblName);
PartitionsSpecByExprResult ret = null;
Exception ex = null;
try {
checkLimitNumberOfPartitionsByExpr(catName, dbName, tblName, req.getExpr(), UNLIMITED_MAX_PARTITIONS);
List<Partition> partitions = new LinkedList<>();
boolean hasUnknownPartitions = getMS().getPartitionsByExpr(catName, dbName, tblName,
req.getExpr(), req.getDefaultPartitionName(), req.getMaxParts(), partitions);
Table table = get_table_core(catName, dbName, tblName);
List<PartitionSpec> partitionSpecs =
MetaStoreServerUtils.getPartitionspecsGroupedByStorageDescriptor(table, partitions);
ret = new PartitionsSpecByExprResult(partitionSpecs, hasUnknownPartitions);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partitions_spec_by_expr", ret != null, ex, tblName);
}
return ret;
}
@Override
public PartitionsByExprResult get_partitions_by_expr(
PartitionsByExprRequest req) throws TException {
String dbName = req.getDbName(), tblName = req.getTblName();
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
startTableFunction("get_partitions_by_expr", catName, dbName, tblName);
fireReadTablePreEvent(catName, dbName, tblName);
PartitionsByExprResult ret = null;
Exception ex = null;
try {
checkLimitNumberOfPartitionsByExpr(catName, dbName, tblName, req.getExpr(), UNLIMITED_MAX_PARTITIONS);
List<Partition> partitions = new LinkedList<>();
boolean hasUnknownPartitions = getMS().getPartitionsByExpr(catName, dbName, tblName,
req.getExpr(), req.getDefaultPartitionName(), req.getMaxParts(), partitions);
ret = new PartitionsByExprResult(partitions, hasUnknownPartitions);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_partitions_by_expr", ret != null, ex, tblName);
}
return ret;
}
@Override
@Deprecated
public int get_num_partitions_by_filter(final String dbName,
final String tblName, final String filter)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
if (parsedDbName[DB_NAME] == null || tblName == null) {
throw new MetaException("The DB and table name cannot be null.");
}
startTableFunction("get_num_partitions_by_filter", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tblName);
int ret = -1;
Exception ex = null;
try {
ret = getMS().getNumPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tblName, filter);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_num_partitions_by_filter", ret != -1, ex, tblName);
}
return ret;
}
private int get_num_partitions_by_expr(final String catName, final String dbName,
final String tblName, final byte[] expr)
throws TException {
int ret = -1;
Exception ex = null;
try {
ret = getMS().getNumPartitionsByExpr(catName, dbName, tblName, expr);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_num_partitions_by_expr", ret != -1, ex, tblName);
}
return ret;
}
private int getNumPartitionsByPs(final String catName, final String dbName,
final String tblName, List<String> partVals)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
startTableFunction("getNumPartitionsByPs", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tblName);
int ret = -1;
Exception ex = null;
try {
ret = getMS().getNumPartitionsByPs(catName, dbName, tblName, partVals);
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("getNumPartitionsByPs", ret != -1, ex, tblName);
}
return ret;
}
@Override
@Deprecated
public List<Partition> get_partitions_by_names(final String dbName, final String tblName,
final List<String> partNames)
throws TException {
return get_partitions_by_names(dbName, tblName, partNames, false, null, null);
}
@Override
public GetPartitionsByNamesResult get_partitions_by_names_req(GetPartitionsByNamesRequest gpbnr)
throws TException {
List<Partition> partitions = get_partitions_by_names(gpbnr.getDb_name(),
gpbnr.getTbl_name(), gpbnr.getNames(),
gpbnr.isSetGet_col_stats() && gpbnr.isGet_col_stats(), gpbnr.getEngine(),
gpbnr.getProcessorCapabilities(), gpbnr.getProcessorIdentifier());
return new GetPartitionsByNamesResult(partitions);
}
public List<Partition> get_partitions_by_names(final String dbName, final String tblName,
final List<String> partNames, boolean getColStats, String engine, String validWriteIdList)
throws TException {
return get_partitions_by_names(
dbName, tblName, partNames, getColStats, engine, null, null);
}
public List<Partition> get_partitions_by_names(final String dbName, final String tblName,
final List<String> partNames, boolean getColStats, String engine,
List<String> processorCapabilities, String processorId) throws TException {
String[] dbNameParts = parseDbName(dbName, conf);
String parsedCatName = dbNameParts[CAT_NAME];
String parsedDbName = dbNameParts[DB_NAME];
List<Partition> ret = null;
Table table = null;
Exception ex = null;
boolean success = false;
startTableFunction("get_partitions_by_names", parsedCatName, parsedDbName,
tblName);
try {
getMS().openTransaction();
authorizeTableForPartitionMetadata(parsedCatName, parsedDbName, tblName);
fireReadTablePreEvent(parsedCatName, parsedDbName, tblName);
ret = getMS().getPartitionsByNames(parsedCatName, parsedDbName, tblName, partNames);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
table = getTable(parsedCatName, parsedDbName, tblName);
// If requested add column statistics in each of the partition objects
if (getColStats) {
// Since each partition may have stats collected for different set of columns, we
// request them separately.
for (Partition part: ret) {
String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
List<ColumnStatistics> partColStatsList =
getMS().getPartitionColumnStatistics(parsedCatName, parsedDbName, tblName,
Collections.singletonList(partName),
StatsSetupConst.getColumnsHavingStats(part.getParameters()),
engine);
if (partColStatsList != null && !partColStatsList.isEmpty()) {
ColumnStatistics partColStats = partColStatsList.get(0);
if (partColStats != null) {
part.setColStats(partColStats);
}
}
}
}
if (processorCapabilities == null || processorCapabilities.size() == 0 ||
processorCapabilities.contains("MANAGERAWMETADATA")) {
LOG.info("Skipping translation for processor with " + processorId);
} else {
if (transformer != null) {
ret = transformer.transformPartitions(ret, table, processorCapabilities, processorId);
}
}
success = getMS().commitTransaction();
} catch (Exception e) {
ex = e;
throw handleException(e)
.throwIfInstance(MetaException.class, NoSuchObjectException.class, InvalidObjectException.class)
.defaultMetaException();
} finally {
if (!success) {
getMS().rollbackTransaction();
}
endFunction("get_partitions_by_names", ret != null, ex, tblName);
}
return ret;
}
/**
* Creates an instance of property manager based on the (declared) namespace.
* @param ns the namespace
* @return the manager instance
* @throws TException
*/
private PropertyManager getPropertyManager(String ns) throws MetaException, NoSuchObjectException {
PropertyStore propertyStore = getMS().getPropertyStore();
PropertyManager mgr = PropertyManager.create(ns, propertyStore);
return mgr;
}
@Override
public PropertyGetResponse get_properties(PropertyGetRequest req) throws TException {
try {
PropertyManager mgr = getPropertyManager(req.getNameSpace());
Map<String, PropertyMap> selected = mgr.selectProperties(req.getMapPrefix(), req.getMapPredicate(), req.getMapSelection());
PropertyGetResponse response = new PropertyGetResponse();
Map<String, Map<String, String>> returned = new TreeMap<>();
selected.forEach((k, v) -> {
returned.put(k, v.export());
});
response.setProperties(returned);
return response;
} catch(PropertyException exception) {
throw ExceptionHandler.newMetaException(exception);
}
}
@Override
public boolean set_properties(PropertySetRequest req) throws TException {
try {
PropertyManager mgr = getPropertyManager(req.getNameSpace());
mgr.setProperties((Map<String, Object>) (Map<?, ?>) req.getPropertyMap());
mgr.commit();
return true;
} catch(PropertyException exception) {
throw ExceptionHandler.newMetaException(exception);
}
}
@Override
public PrincipalPrivilegeSet get_privilege_set(HiveObjectRef hiveObject, String userName,
List<String> groupNames) throws TException {
firePreEvent(new PreAuthorizationCallEvent(this));
String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() : getDefaultCatalog(conf);
HiveObjectType debug = hiveObject.getObjectType();
if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
String partName = getPartName(hiveObject);
return this.get_column_privilege_set(catName, hiveObject.getDbName(), hiveObject
.getObjectName(), partName, hiveObject.getColumnName(), userName,
groupNames);
} else if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
String partName = getPartName(hiveObject);
return this.get_partition_privilege_set(catName, hiveObject.getDbName(),
hiveObject.getObjectName(), partName, userName, groupNames);
} else if (hiveObject.getObjectType() == HiveObjectType.DATABASE) {
return this.get_db_privilege_set(catName, hiveObject.getDbName(), userName,
groupNames);
} else if (hiveObject.getObjectType() == HiveObjectType.DATACONNECTOR) {
return this.get_connector_privilege_set(catName, hiveObject.getObjectName(), userName, groupNames);
} else if (hiveObject.getObjectType() == HiveObjectType.TABLE) {
return this.get_table_privilege_set(catName, hiveObject.getDbName(), hiveObject
.getObjectName(), userName, groupNames);
} else if (hiveObject.getObjectType() == HiveObjectType.GLOBAL) {
return this.get_user_privilege_set(userName, groupNames);
}
return null;
}
private String getPartName(HiveObjectRef hiveObject) throws MetaException {
String partName = null;
List<String> partValue = hiveObject.getPartValues();
if (partValue != null && partValue.size() > 0) {
try {
String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() :
getDefaultCatalog(conf);
Table table = get_table_core(catName, hiveObject.getDbName(), hiveObject
.getObjectName());
partName = Warehouse
.makePartName(table.getPartitionKeys(), partValue);
} catch (NoSuchObjectException e) {
throw new MetaException(e.getMessage());
}
}
return partName;
}
private PrincipalPrivilegeSet get_column_privilege_set(String catName, final String dbName,
final String tableName, final String partName, final String columnName,
final String userName, final List<String> groupNames) throws TException {
incrementCounter("get_column_privilege_set");
PrincipalPrivilegeSet ret;
try {
ret = getMS().getColumnPrivilegeSet(
catName, dbName, tableName, partName, columnName, userName, groupNames);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
return ret;
}
private PrincipalPrivilegeSet get_db_privilege_set(String catName, final String dbName,
final String userName, final List<String> groupNames) throws TException {
incrementCounter("get_db_privilege_set");
PrincipalPrivilegeSet ret;
try {
ret = getMS().getDBPrivilegeSet(catName, dbName, userName, groupNames);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
return ret;
}
private PrincipalPrivilegeSet get_connector_privilege_set(String catName, final String connectorName,
final String userName, final List<String> groupNames) throws TException {
incrementCounter("get_connector_privilege_set");
PrincipalPrivilegeSet ret;
try {
ret = getMS().getConnectorPrivilegeSet(catName, connectorName, userName, groupNames);
} catch (MetaException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
return ret;
}
private PrincipalPrivilegeSet get_partition_privilege_set(
String catName, final String dbName, final String tableName, final String partName,
final String userName, final List<String> groupNames)
throws TException {
incrementCounter("get_partition_privilege_set");
PrincipalPrivilegeSet ret;
try {
ret = getMS().getPartitionPrivilegeSet(catName, dbName, tableName, partName,
userName, groupNames);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
return ret;
}
private PrincipalPrivilegeSet get_table_privilege_set(String catName, final String dbName,
final String tableName, final String userName,
final List<String> groupNames) throws TException {
incrementCounter("get_table_privilege_set");
PrincipalPrivilegeSet ret;
try {
ret = getMS().getTablePrivilegeSet(catName, dbName, tableName, userName,
groupNames);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
return ret;
}
@Override
public boolean grant_role(final String roleName,
final String principalName, final PrincipalType principalType,
final String grantor, final PrincipalType grantorType, final boolean grantOption)
throws TException {
incrementCounter("add_role_member");
firePreEvent(new PreAuthorizationCallEvent(this));
if (PUBLIC.equals(roleName)) {
throw new MetaException("No user can be added to " + PUBLIC +". Since all users implicitly"
+ " belong to " + PUBLIC + " role.");
}
Boolean ret;
try {
RawStore ms = getMS();
Role role = ms.getRole(roleName);
if(principalType == PrincipalType.ROLE){
//check if this grant statement will end up creating a cycle
if(isNewRoleAParent(principalName, roleName)){
throw new MetaException("Cannot grant role " + principalName + " to " + roleName +
" as " + roleName + " already belongs to the role " + principalName +
". (no cycles allowed)");
}
}
ret = ms.grantRole(role, principalName, principalType, grantor, grantorType, grantOption);
} catch (Exception e) {
String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
LOG.error(exInfo, e);
throw handleException(e).throwIfInstance(MetaException.class)
.toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
.defaultTException();
}
return ret;
}
/**
* Check if newRole is in parent hierarchy of curRole
* @param newRole
* @param curRole
* @return true if newRole is curRole or present in its hierarchy
* @throws MetaException
*/
private boolean isNewRoleAParent(String newRole, String curRole) throws MetaException {
if(newRole.equals(curRole)){
return true;
}
//do this check recursively on all the parent roles of curRole
List<Role> parentRoleMaps = getMS().listRoles(curRole, PrincipalType.ROLE);
for(Role parentRole : parentRoleMaps){
if(isNewRoleAParent(newRole, parentRole.getRoleName())){
return true;
}
}
return false;
}
@Override
public List<Role> list_roles(final String principalName,
final PrincipalType principalType) throws TException {
incrementCounter("list_roles");
firePreEvent(new PreAuthorizationCallEvent(this));
return getMS().listRoles(principalName, principalType);
}
@Override
public boolean create_role(final Role role) throws TException {
incrementCounter("create_role");
firePreEvent(new PreAuthorizationCallEvent(this));
if (PUBLIC.equals(role.getRoleName())) {
throw new MetaException(PUBLIC + " role implicitly exists. It can't be created.");
}
Boolean ret;
try {
ret = getMS().addRole(role.getRoleName(), role.getOwnerName());
} catch (Exception e) {
String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
LOG.error(exInfo, e);
throw handleException(e).throwIfInstance(MetaException.class)
.toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
.defaultTException();
}
return ret;
}
@Override
public boolean drop_role(final String roleName) throws TException {
incrementCounter("drop_role");
firePreEvent(new PreAuthorizationCallEvent(this));
if (ADMIN.equals(roleName) || PUBLIC.equals(roleName)) {
throw new MetaException(PUBLIC + "," + ADMIN + " roles can't be dropped.");
}
Boolean ret;
try {
ret = getMS().removeRole(roleName);
} catch (Exception e) {
String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
LOG.error(exInfo, e);
throw handleException(e).throwIfInstance(MetaException.class)
.toMetaExceptionIfInstance(exInfo, NoSuchObjectException.class)
.defaultTException();
}
return ret;
}
@Override
public List<String> get_role_names() throws TException {
incrementCounter("get_role_names");
firePreEvent(new PreAuthorizationCallEvent(this));
List<String> ret;
try {
ret = getMS().listRoleNames();
return ret;
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
@Override
public boolean grant_privileges(final PrivilegeBag privileges) throws TException {
incrementCounter("grant_privileges");
firePreEvent(new PreAuthorizationCallEvent(this));
Boolean ret;
try {
ret = getMS().grantPrivileges(privileges);
} catch (Exception e) {
String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
LOG.error(exInfo, e);
throw handleException(e).throwIfInstance(MetaException.class)
.toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
.defaultTException();
}
return ret;
}
@Override
public boolean revoke_role(final String roleName, final String userName,
final PrincipalType principalType) throws TException {
return revoke_role(roleName, userName, principalType, false);
}
private boolean revoke_role(final String roleName, final String userName,
final PrincipalType principalType, boolean grantOption) throws TException {
incrementCounter("remove_role_member");
firePreEvent(new PreAuthorizationCallEvent(this));
if (PUBLIC.equals(roleName)) {
throw new MetaException(PUBLIC + " role can't be revoked.");
}
Boolean ret;
try {
RawStore ms = getMS();
Role mRole = ms.getRole(roleName);
ret = ms.revokeRole(mRole, userName, principalType, grantOption);
} catch (Exception e) {
String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
LOG.error(exInfo, e);
throw handleException(e).throwIfInstance(MetaException.class)
.toMetaExceptionIfInstance(exInfo, NoSuchObjectException.class)
.defaultTException();
}
return ret;
}
@Override
public GrantRevokeRoleResponse grant_revoke_role(GrantRevokeRoleRequest request)
throws TException {
GrantRevokeRoleResponse response = new GrantRevokeRoleResponse();
boolean grantOption = false;
if (request.isSetGrantOption()) {
grantOption = request.isGrantOption();
}
switch (request.getRequestType()) {
case GRANT: {
boolean result = grant_role(request.getRoleName(),
request.getPrincipalName(), request.getPrincipalType(),
request.getGrantor(), request.getGrantorType(), grantOption);
response.setSuccess(result);
break;
}
case REVOKE: {
boolean result = revoke_role(request.getRoleName(), request.getPrincipalName(),
request.getPrincipalType(), grantOption);
response.setSuccess(result);
break;
}
default:
throw new MetaException("Unknown request type " + request.getRequestType());
}
return response;
}
@Override
public GrantRevokePrivilegeResponse grant_revoke_privileges(GrantRevokePrivilegeRequest request)
throws TException {
GrantRevokePrivilegeResponse response = new GrantRevokePrivilegeResponse();
switch (request.getRequestType()) {
case GRANT: {
boolean result = grant_privileges(request.getPrivileges());
response.setSuccess(result);
break;
}
case REVOKE: {
boolean revokeGrantOption = false;
if (request.isSetRevokeGrantOption()) {
revokeGrantOption = request.isRevokeGrantOption();
}
boolean result = revoke_privileges(request.getPrivileges(), revokeGrantOption);
response.setSuccess(result);
break;
}
default:
throw new MetaException("Unknown request type " + request.getRequestType());
}
return response;
}
@Override
public GrantRevokePrivilegeResponse refresh_privileges(HiveObjectRef objToRefresh, String authorizer,
GrantRevokePrivilegeRequest grantRequest)
throws TException {
incrementCounter("refresh_privileges");
firePreEvent(new PreAuthorizationCallEvent(this));
GrantRevokePrivilegeResponse response = new GrantRevokePrivilegeResponse();
try {
boolean result = getMS().refreshPrivileges(objToRefresh, authorizer, grantRequest.getPrivileges());
response.setSuccess(result);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
return response;
}
@Override
public boolean revoke_privileges(final PrivilegeBag privileges) throws TException {
return revoke_privileges(privileges, false);
}
public boolean revoke_privileges(final PrivilegeBag privileges, boolean grantOption)
throws TException {
incrementCounter("revoke_privileges");
firePreEvent(new PreAuthorizationCallEvent(this));
Boolean ret;
try {
ret = getMS().revokePrivileges(privileges, grantOption);
} catch (Exception e) {
String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
LOG.error(exInfo, e);
throw handleException(e).throwIfInstance(MetaException.class)
.toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
.defaultTException();
}
return ret;
}
private PrincipalPrivilegeSet get_user_privilege_set(final String userName,
final List<String> groupNames) throws TException {
incrementCounter("get_user_privilege_set");
PrincipalPrivilegeSet ret;
try {
ret = getMS().getUserPrivilegeSet(userName, groupNames);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
return ret;
}
@Override
public List<HiveObjectPrivilege> list_privileges(String principalName,
PrincipalType principalType, HiveObjectRef hiveObject)
throws TException {
firePreEvent(new PreAuthorizationCallEvent(this));
String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() : getDefaultCatalog(conf);
if (hiveObject.getObjectType() == null) {
return getAllPrivileges(principalName, principalType, catName);
}
if (hiveObject.getObjectType() == HiveObjectType.GLOBAL) {
return list_global_privileges(principalName, principalType);
}
if (hiveObject.getObjectType() == HiveObjectType.DATABASE) {
return list_db_privileges(principalName, principalType, catName, hiveObject
.getDbName());
}
if (hiveObject.getObjectType() == HiveObjectType.DATACONNECTOR) {
return list_dc_privileges(principalName, principalType, hiveObject
.getObjectName());
}
if (hiveObject.getObjectType() == HiveObjectType.TABLE) {
return list_table_privileges(principalName, principalType,
catName, hiveObject.getDbName(), hiveObject.getObjectName());
}
if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
return list_partition_privileges(principalName, principalType,
catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject
.getPartValues());
}
if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
if (hiveObject.getPartValues() == null || hiveObject.getPartValues().isEmpty()) {
return list_table_column_privileges(principalName, principalType,
catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject.getColumnName());
}
return list_partition_column_privileges(principalName, principalType,
catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject
.getPartValues(), hiveObject.getColumnName());
}
return null;
}
private List<HiveObjectPrivilege> getAllPrivileges(String principalName,
PrincipalType principalType, String catName) throws TException {
List<HiveObjectPrivilege> privs = new ArrayList<>();
privs.addAll(list_global_privileges(principalName, principalType));
privs.addAll(list_db_privileges(principalName, principalType, catName, null));
privs.addAll(list_dc_privileges(principalName, principalType, null));
privs.addAll(list_table_privileges(principalName, principalType, catName, null, null));
privs.addAll(list_partition_privileges(principalName, principalType, catName, null, null, null));
privs.addAll(list_table_column_privileges(principalName, principalType, catName, null, null, null));
privs.addAll(list_partition_column_privileges(principalName, principalType,
catName, null, null, null, null));
return privs;
}
private List<HiveObjectPrivilege> list_table_column_privileges(
final String principalName, final PrincipalType principalType, String catName,
final String dbName, final String tableName, final String columnName) throws TException {
incrementCounter("list_table_column_privileges");
try {
if (dbName == null) {
return getMS().listPrincipalTableColumnGrantsAll(principalName, principalType);
}
if (principalName == null) {
return getMS().listTableColumnGrantsAll(catName, dbName, tableName, columnName);
}
return getMS().listPrincipalTableColumnGrants(principalName, principalType,
catName, dbName, tableName, columnName);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
private List<HiveObjectPrivilege> list_partition_column_privileges(
final String principalName, final PrincipalType principalType,
String catName, final String dbName, final String tableName, final List<String> partValues,
final String columnName) throws TException {
incrementCounter("list_partition_column_privileges");
try {
if (dbName == null) {
return getMS().listPrincipalPartitionColumnGrantsAll(principalName, principalType);
}
Table tbl = get_table_core(catName, dbName, tableName);
String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues);
if (principalName == null) {
return getMS().listPartitionColumnGrantsAll(catName, dbName, tableName, partName, columnName);
}
return getMS().listPrincipalPartitionColumnGrants(principalName, principalType, catName, dbName,
tableName, partValues, partName, columnName);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
private List<HiveObjectPrivilege> list_db_privileges(final String principalName,
final PrincipalType principalType, String catName, final String dbName) throws TException {
incrementCounter("list_security_db_grant");
try {
if (dbName == null) {
return getMS().listPrincipalDBGrantsAll(principalName, principalType);
}
if (principalName == null) {
return getMS().listDBGrantsAll(catName, dbName);
} else {
return getMS().listPrincipalDBGrants(principalName, principalType, catName, dbName);
}
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
private List<HiveObjectPrivilege> list_dc_privileges(final String principalName,
final PrincipalType principalType, final String dcName) throws TException {
incrementCounter("list_security_dc_grant");
try {
if (dcName == null) {
return getMS().listPrincipalDCGrantsAll(principalName, principalType);
}
if (principalName == null) {
return getMS().listDCGrantsAll(dcName);
} else {
return getMS().listPrincipalDCGrants(principalName, principalType, dcName);
}
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
private List<HiveObjectPrivilege> list_partition_privileges(
final String principalName, final PrincipalType principalType,
String catName, final String dbName, final String tableName, final List<String> partValues)
throws TException {
incrementCounter("list_security_partition_grant");
try {
if (dbName == null) {
return getMS().listPrincipalPartitionGrantsAll(principalName, principalType);
}
Table tbl = get_table_core(catName, dbName, tableName);
String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues);
if (principalName == null) {
return getMS().listPartitionGrantsAll(catName, dbName, tableName, partName);
}
return getMS().listPrincipalPartitionGrants(
principalName, principalType, catName, dbName, tableName, partValues, partName);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
private List<HiveObjectPrivilege> list_table_privileges(
final String principalName, final PrincipalType principalType,
String catName, final String dbName, final String tableName) throws TException {
incrementCounter("list_security_table_grant");
try {
if (dbName == null) {
return getMS().listPrincipalTableGrantsAll(principalName, principalType);
}
if (principalName == null) {
return getMS().listTableGrantsAll(catName, dbName, tableName);
}
return getMS().listAllTableGrants(principalName, principalType, catName, dbName, tableName);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
private List<HiveObjectPrivilege> list_global_privileges(
final String principalName, final PrincipalType principalType) throws TException {
incrementCounter("list_security_user_grant");
try {
if (principalName == null) {
return getMS().listGlobalGrantsAll();
}
return getMS().listPrincipalGlobalGrants(principalName, principalType);
} catch (Exception e) {
throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
}
}
@Override
public void cancel_delegation_token(String token_str_form) throws TException {
startFunction("cancel_delegation_token");
boolean success = false;
Exception ex = null;
try {
HiveMetaStore.cancelDelegationToken(token_str_form);
success = true;
} catch (Exception e) {
ex = e;
throw handleException(e).convertIfInstance(IOException.class, MetaException.class).defaultMetaException();
} finally {
endFunction("cancel_delegation_token", success, ex);
}
}
@Override
public long renew_delegation_token(String token_str_form) throws TException {
startFunction("renew_delegation_token");
Long ret = null;
Exception ex = null;
try {
ret = HiveMetaStore.renewDelegationToken(token_str_form);
} catch (Exception e) {
ex = e;
throw handleException(e).convertIfInstance(IOException.class, MetaException.class).defaultMetaException();
} finally {
endFunction("renew_delegation_token", ret != null, ex);
}
return ret;
}
@Override
public String get_delegation_token(String token_owner, String renewer_kerberos_principal_name)
throws TException {
startFunction("get_delegation_token");
String ret = null;
Exception ex = null;
try {
ret =
HiveMetaStore.getDelegationToken(token_owner,
renewer_kerberos_principal_name, getIPAddress());
} catch (Exception e) {
ex = e;
throw handleException(e).convertIfInstance(IOException.class, MetaException.class)
.convertIfInstance(InterruptedException.class, MetaException.class)
.defaultMetaException();
} finally {
endFunction("get_delegation_token", ret != null, ex);
}
return ret;
}
@Override
public boolean add_token(String token_identifier, String delegation_token) throws TException {
startFunction("add_token", ": " + token_identifier);
boolean ret = false;
Exception ex = null;
try {
ret = getMS().addToken(token_identifier, delegation_token);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("add_token", ret == true, ex);
}
return ret;
}
@Override
public boolean remove_token(String token_identifier) throws TException {
startFunction("remove_token", ": " + token_identifier);
boolean ret = false;
Exception ex = null;
try {
ret = getMS().removeToken(token_identifier);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("remove_token", ret == true, ex);
}
return ret;
}
@Override
public String get_token(String token_identifier) throws TException {
startFunction("get_token for", ": " + token_identifier);
String ret = null;
Exception ex = null;
try {
ret = getMS().getToken(token_identifier);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_token", ret != null, ex);
}
//Thrift cannot return null result
return ret == null ? "" : ret;
}
@Override
public List<String> get_all_token_identifiers() throws TException {
startFunction("get_all_token_identifiers.");
List<String> ret;
Exception ex = null;
try {
ret = getMS().getAllTokenIdentifiers();
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_all_token_identifiers.", ex == null, ex);
}
return ret;
}
@Override
public int add_master_key(String key) throws TException {
startFunction("add_master_key.");
int ret;
Exception ex = null;
try {
ret = getMS().addMasterKey(key);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("add_master_key.", ex == null, ex);
}
return ret;
}
@Override
public void update_master_key(int seq_number, String key) throws TException {
startFunction("update_master_key.");
Exception ex = null;
try {
getMS().updateMasterKey(seq_number, key);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("update_master_key.", ex == null, ex);
}
}
@Override
public boolean remove_master_key(int key_seq) throws TException {
startFunction("remove_master_key.");
Exception ex = null;
boolean ret;
try {
ret = getMS().removeMasterKey(key_seq);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("remove_master_key.", ex == null, ex);
}
return ret;
}
@Override
public List<String> get_master_keys() throws TException {
startFunction("get_master_keys.");
Exception ex = null;
String [] ret = null;
try {
ret = getMS().getMasterKeys();
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_master_keys.", ret != null, ex);
}
return Arrays.asList(ret);
}
@Override
public void markPartitionForEvent(final String db_name, final String tbl_name,
final Map<String, String> partName, final PartitionEventType evtType) throws TException {
Table tbl = null;
Exception ex = null;
RawStore ms = getMS();
boolean success = false;
try {
String[] parsedDbName = parseDbName(db_name, conf);
ms.openTransaction();
startPartitionFunction("markPartitionForEvent", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, partName);
firePreEvent(new PreLoadPartitionDoneEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, partName, this));
tbl = ms.markPartitionForEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
partName, evtType);
if (null == tbl) {
throw new UnknownTableException("Table: " + tbl_name + " not found.");
}
if (transactionalListeners.size() > 0) {
LoadPartitionDoneEvent lpde = new LoadPartitionDoneEvent(true, tbl, partName, this);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onLoadPartitionDone(lpde);
}
}
success = ms.commitTransaction();
for (MetaStoreEventListener listener : listeners) {
listener.onLoadPartitionDone(new LoadPartitionDoneEvent(true, tbl, partName, this));
}
} catch (Exception original) {
ex = original;
LOG.error("Exception caught in mark partition event ", original);
throw handleException(original)
.throwIfInstance(UnknownTableException.class, InvalidPartitionException.class, MetaException.class)
.defaultMetaException();
} finally {
if (!success) {
ms.rollbackTransaction();
}
endFunction("markPartitionForEvent", tbl != null, ex, tbl_name);
}
}
@Override
public boolean isPartitionMarkedForEvent(final String db_name, final String tbl_name,
final Map<String, String> partName, final PartitionEventType evtType) throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
startPartitionFunction("isPartitionMarkedForEvent", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, partName);
Boolean ret = null;
Exception ex = null;
try {
ret = getMS().isPartitionMarkedForEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tbl_name, partName, evtType);
} catch (Exception original) {
LOG.error("Exception caught for isPartitionMarkedForEvent ", original);
ex = original;
throw handleException(original).throwIfInstance(UnknownTableException.class, InvalidPartitionException.class)
.throwIfInstance(UnknownPartitionException.class, MetaException.class)
.defaultMetaException();
} finally {
endFunction("isPartitionMarkedForEvent", ret != null, ex, tbl_name);
}
return ret;
}
@Override
public List<String> set_ugi(String username, List<String> groupNames) throws TException {
Collections.addAll(groupNames, username);
return groupNames;
}
@Override
public boolean partition_name_has_valid_characters(List<String> part_vals,
boolean throw_exception) throws TException {
startFunction("partition_name_has_valid_characters");
boolean ret;
Exception ex = null;
try {
if (throw_exception) {
MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern);
ret = true;
} else {
ret = MetaStoreServerUtils.partitionNameHasValidCharacters(part_vals,
partitionValidationPattern);
}
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("partition_name_has_valid_characters", true, ex);
}
return ret;
}
private void validateFunctionInfo(Function func) throws InvalidObjectException, MetaException {
if (func == null) {
throw new MetaException("Function cannot be null.");
}
if (func.getFunctionName() == null) {
throw new MetaException("Function name cannot be null.");
}
if (func.getDbName() == null) {
throw new MetaException("Database name in Function cannot be null.");
}
if (!MetaStoreUtils.validateName(func.getFunctionName(), null)) {
throw new InvalidObjectException(func.getFunctionName() + " is not a valid object name");
}
String className = func.getClassName();
if (className == null) {
throw new InvalidObjectException("Function class name cannot be null");
}
if (func.getOwnerType() == null) {
throw new MetaException("Function owner type cannot be null.");
}
if (func.getFunctionType() == null) {
throw new MetaException("Function type cannot be null.");
}
}
@Override
public void create_function(Function func) throws TException {
validateFunctionInfo(func);
boolean success = false;
RawStore ms = getMS();
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
try {
String catName = func.isSetCatName() ? func.getCatName() : getDefaultCatalog(conf);
if (!func.isSetOwnerName()) {
try {
func.setOwnerName(SecurityUtils.getUGI().getShortUserName());
} catch (Exception ex) {
LOG.error("Cannot obtain username from the session to create a function", ex);
throw new TException(ex);
}
}
ms.openTransaction();
Database db = ms.getDatabase(catName, func.getDbName());
if (db == null) {
throw new NoSuchObjectException("The database " + func.getDbName() + " does not exist");
}
if (db.getType() == DatabaseType.REMOTE) {
throw new MetaException("Operation create_function not support for REMOTE database");
}
Function existingFunc = ms.getFunction(catName, func.getDbName(), func.getFunctionName());
if (existingFunc != null) {
throw new AlreadyExistsException(
"Function " + func.getFunctionName() + " already exists");
}
firePreEvent(new PreCreateFunctionEvent(func, this));
long time = System.currentTimeMillis() / 1000;
func.setCreateTime((int) time);
ms.createFunction(func);
if (!transactionalListeners.isEmpty()) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.CREATE_FUNCTION,
new CreateFunctionEvent(func, true, this));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.CREATE_FUNCTION,
new CreateFunctionEvent(func, success, this),
null,
transactionalListenerResponses, ms);
}
}
}
@Override
public void drop_function(String dbName, String funcName)
throws NoSuchObjectException, MetaException,
InvalidObjectException, InvalidInputException {
if (funcName == null) {
throw new MetaException("Function name cannot be null.");
}
boolean success = false;
Function func = null;
RawStore ms = getMS();
Map<String, String> transactionalListenerResponses = Collections.emptyMap();
String[] parsedDbName = parseDbName(dbName, conf);
if (parsedDbName[DB_NAME] == null) {
throw new MetaException("Database name cannot be null.");
}
try {
ms.openTransaction();
func = ms.getFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName);
if (func == null) {
throw new NoSuchObjectException("Function " + funcName + " does not exist");
}
Boolean needsCm =
ReplChangeManager.isSourceOfReplication(get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]));
// if copy of jar to change management fails we fail the metastore transaction, since the
// user might delete the jars on HDFS externally after dropping the function, hence having
// a copy is required to allow incremental replication to work correctly.
if (func.getResourceUris() != null && !func.getResourceUris().isEmpty()) {
for (ResourceUri uri : func.getResourceUris()) {
if (uri.getUri().toLowerCase().startsWith("hdfs:") && needsCm) {
wh.addToChangeManagement(new Path(uri.getUri()));
}
}
}
firePreEvent(new PreDropFunctionEvent(func, this));
// if the operation on metastore fails, we don't do anything in change management, but fail
// the metastore transaction, as having a copy of the jar in change management is not going
// to cause any problem, the cleaner thread will remove this when this jar expires.
ms.dropFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName);
if (transactionalListeners.size() > 0) {
transactionalListenerResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_FUNCTION,
new DropFunctionEvent(func, true, this));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (listeners.size() > 0) {
MetaStoreListenerNotifier.notifyEvent(listeners,
EventType.DROP_FUNCTION,
new DropFunctionEvent(func, success, this),
null,
transactionalListenerResponses, ms);
}
}
}
@Override
public void alter_function(String dbName, String funcName, Function newFunc) throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
validateForAlterFunction(parsedDbName[DB_NAME], funcName, newFunc);
boolean success = false;
RawStore ms = getMS();
try {
firePreEvent(new PreCreateFunctionEvent(newFunc, this));
ms.openTransaction();
ms.alterFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName, newFunc);
success = ms.commitTransaction();
} catch (InvalidObjectException e) {
// Throwing MetaException instead of InvalidObjectException as the InvalidObjectException
// is not defined for the alter_function method in the Thrift interface.
throwMetaException(e);
} finally {
if (!success) {
ms.rollbackTransaction();
}
}
}
private void validateForAlterFunction(String dbName, String funcName, Function newFunc)
throws MetaException {
if (dbName == null || funcName == null) {
throw new MetaException("Database and function name cannot be null.");
}
try {
validateFunctionInfo(newFunc);
} catch (InvalidObjectException e) {
// The validateFunctionInfo method is used by the create and alter function methods as well
// and it can throw InvalidObjectException. But the InvalidObjectException is not defined
// for the alter_function method in the Thrift interface, therefore a TApplicationException
// will occur at the caller side. Re-throwing the InvalidObjectException as MetaException
// would eliminate the TApplicationException at caller side.
throw newMetaException(e);
}
}
@Override
public List<String> get_functions(String dbName, String pattern)
throws MetaException {
startFunction("get_functions", ": db=" + dbName + " pat=" + pattern);
RawStore ms = getMS();
Exception ex = null;
List<String> funcNames = null;
String[] parsedDbName = parseDbName(dbName, conf);
try {
funcNames = ms.getFunctions(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], pattern);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_functions", funcNames != null, ex);
}
return funcNames;
}
@Override
public GetAllFunctionsResponse get_all_functions()
throws MetaException {
GetAllFunctionsResponse response = new GetAllFunctionsResponse();
startFunction("get_all_functions");
RawStore ms = getMS();
List<Function> allFunctions = null;
Exception ex = null;
try {
// Leaving this as the 'hive' catalog (rather than choosing the default from the
// configuration) because all the default UDFs are in that catalog, and I think that's
// would people really want here.
allFunctions = ms.getAllFunctions(DEFAULT_CATALOG_NAME);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_all_functions", allFunctions != null, ex);
}
response.setFunctions(allFunctions);
return response;
}
@Override
public Function get_function(String dbName, String funcName) throws TException {
if (dbName == null || funcName == null) {
throw new MetaException("Database and function name cannot be null.");
}
startFunction("get_function", ": " + dbName + "." + funcName);
RawStore ms = getMS();
Function func = null;
Exception ex = null;
String[] parsedDbName = parseDbName(dbName, conf);
try {
func = ms.getFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName);
if (func == null) {
throw new NoSuchObjectException(
"Function " + dbName + "." + funcName + " does not exist");
}
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(NoSuchObjectException.class).defaultMetaException();
} finally {
endFunction("get_function", func != null, ex);
}
return func;
}
// Transaction and locking methods
@Override
public GetOpenTxnsResponse get_open_txns() throws TException {
return getTxnHandler().getOpenTxns();
}
@Override
public GetOpenTxnsResponse get_open_txns_req(GetOpenTxnsRequest getOpenTxnsRequest) throws TException {
return getTxnHandler().getOpenTxns(getOpenTxnsRequest.getExcludeTxnTypes());
}
// Transaction and locking methods
@Override
public GetOpenTxnsInfoResponse get_open_txns_info() throws TException {
return getTxnHandler().getOpenTxnsInfo();
}
@Override
public OpenTxnsResponse open_txns(OpenTxnRequest rqst) throws TException {
OpenTxnsResponse response = getTxnHandler().openTxns(rqst);
List<Long> txnIds = response.getTxn_ids();
boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
if (txnIds != null && listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.OPEN_TXN,
new OpenTxnEvent(txnIds, this));
}
return response;
}
@Override
public void abort_txn(AbortTxnRequest rqst) throws TException {
getTxnHandler().abortTxn(rqst);
boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
if (listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
// Not adding dbsUpdated to AbortTxnEvent because
// only DbNotificationListener cares about it, and this is already
// handled with transactional listeners in TxnHandler.
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ABORT_TXN,
new AbortTxnEvent(rqst.getTxnid(), this));
}
}
@Override
public void abort_txns(AbortTxnsRequest rqst) throws TException {
getTxnHandler().abortTxns(rqst);
if (listeners != null && !listeners.isEmpty()) {
for (Long txnId : rqst.getTxn_ids()) {
// See above abort_txn() note about not adding dbsUpdated.
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ABORT_TXN,
new AbortTxnEvent(txnId, this));
}
}
}
@Override
public AbortCompactResponse abort_Compactions(AbortCompactionRequest rqst) throws TException {
return getTxnHandler().abortCompactions(rqst);
}
@Override
public long get_latest_txnid_in_conflict(long txnId) throws MetaException {
return getTxnHandler().getLatestTxnIdInConflict(txnId);
}
@Override
public void commit_txn(CommitTxnRequest rqst) throws TException {
boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
// in replication flow, the write notification log table will be updated here.
if (rqst.isSetWriteEventInfos() && isReplayedReplTxn) {
assert (rqst.isSetReplPolicy());
long targetTxnId = getTxnHandler().getTargetTxnId(rqst.getReplPolicy(), rqst.getTxnid());
if (targetTxnId < 0) {
//looks like a retry
return;
}
for (WriteEventInfo writeEventInfo : rqst.getWriteEventInfos()) {
String[] filesAdded = ReplChangeManager.getListFromSeparatedString(writeEventInfo.getFiles());
List<String> partitionValue = null;
Partition ptnObj = null;
String root;
Table tbl = getTblObject(writeEventInfo.getDatabase(), writeEventInfo.getTable(), null);
if (writeEventInfo.getPartition() != null && !writeEventInfo.getPartition().isEmpty()) {
partitionValue = Warehouse.getPartValuesFromPartName(writeEventInfo.getPartition());
ptnObj = getPartitionObj(writeEventInfo.getDatabase(), writeEventInfo.getTable(), partitionValue, tbl);
root = ptnObj.getSd().getLocation();
} else {
root = tbl.getSd().getLocation();
}
InsertEventRequestData insertData = new InsertEventRequestData();
insertData.setReplace(true);
// The files in the commit txn message during load will have files with path corresponding to source
// warehouse. Need to transform them to target warehouse using table or partition object location.
for (String file : filesAdded) {
String[] decodedPath = ReplChangeManager.decodeFileUri(file);
String name = (new Path(decodedPath[0])).getName();
Path newPath = FileUtils.getTransformedPath(name, decodedPath[3], root);
insertData.addToFilesAdded(newPath.toUri().toString());
insertData.addToSubDirectoryList(decodedPath[3]);
try {
insertData.addToFilesAddedChecksum(ReplChangeManager.checksumFor(newPath, newPath.getFileSystem(conf)));
} catch (IOException e) {
LOG.error("failed to get checksum for the file " + newPath + " with error: " + e.getMessage());
throw new TException(e.getMessage());
}
}
WriteNotificationLogRequest wnRqst = new WriteNotificationLogRequest(targetTxnId,
writeEventInfo.getWriteId(), writeEventInfo.getDatabase(), writeEventInfo.getTable(), insertData);
if (partitionValue != null) {
wnRqst.setPartitionVals(partitionValue);
}
addTxnWriteNotificationLog(tbl, ptnObj, wnRqst);
}
}
getTxnHandler().commitTxn(rqst);
if (listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.COMMIT_TXN,
new CommitTxnEvent(rqst.getTxnid(), this));
Optional<CompactionInfo> compactionInfo = getTxnHandler().getCompactionByTxnId(rqst.getTxnid());
if (compactionInfo.isPresent()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.COMMIT_COMPACTION,
new CommitCompactionEvent(rqst.getTxnid(), compactionInfo.get(), this));
}
}
}
@Override
public void repl_tbl_writeid_state(ReplTblWriteIdStateRequest rqst) throws TException {
getTxnHandler().replTableWriteIdState(rqst);
}
@Override
public GetValidWriteIdsResponse get_valid_write_ids(GetValidWriteIdsRequest rqst) throws TException {
return getTxnHandler().getValidWriteIds(rqst);
}
@Override
public void add_write_ids_to_min_history(long txnId, Map<String, Long> validWriteIds) throws TException {
getTxnHandler().addWriteIdsToMinHistory(txnId, validWriteIds);
}
@Override
public void set_hadoop_jobid(String jobId, long cqId) {
getTxnHandler().setHadoopJobId(jobId, cqId);
}
@Deprecated
@Override
public OptionalCompactionInfoStruct find_next_compact(String workerId) throws MetaException{
return CompactionInfo.compactionInfoToOptionalStruct(
getTxnHandler().findNextToCompact(workerId));
}
@Override
public OptionalCompactionInfoStruct find_next_compact2(FindNextCompactRequest rqst) throws MetaException{
return CompactionInfo.compactionInfoToOptionalStruct(
getTxnHandler().findNextToCompact(rqst));
}
@Override
public void mark_cleaned(CompactionInfoStruct cr) throws MetaException {
getTxnHandler().markCleaned(CompactionInfo.compactionStructToInfo(cr), false);
}
@Override
public void mark_compacted(CompactionInfoStruct cr) throws MetaException {
getTxnHandler().markCompacted(CompactionInfo.compactionStructToInfo(cr));
}
@Override
public void mark_failed(CompactionInfoStruct cr) throws MetaException {
getTxnHandler().markFailed(CompactionInfo.compactionStructToInfo(cr));
}
@Override
public void mark_refused(CompactionInfoStruct cr) throws MetaException {
getTxnHandler().markRefused(CompactionInfo.compactionStructToInfo(cr));
}
@Override
public boolean update_compaction_metrics_data(CompactionMetricsDataStruct struct) throws MetaException, TException {
return getTxnHandler().updateCompactionMetricsData(CompactionMetricsDataConverter.structToData(struct));
}
@Override
public void remove_compaction_metrics_data(CompactionMetricsDataRequest request)
throws MetaException, TException {
getTxnHandler().removeCompactionMetricsData(request.getDbName(), request.getTblName(), request.getPartitionName(),
CompactionMetricsDataConverter.thriftCompactionMetricType2DbType(request.getType()));
}
@Override
public List<String> find_columns_with_stats(CompactionInfoStruct cr) throws MetaException {
return getTxnHandler().findColumnsWithStats(CompactionInfo.compactionStructToInfo(cr));
}
@Override
public void update_compactor_state(CompactionInfoStruct cr, long highWaterMark) throws MetaException {
getTxnHandler().updateCompactorState(
CompactionInfo.compactionStructToInfo(cr), highWaterMark);
}
@Override
public GetLatestCommittedCompactionInfoResponse get_latest_committed_compaction_info(
GetLatestCommittedCompactionInfoRequest rqst) throws MetaException {
if (rqst.getDbname() == null || rqst.getTablename() == null) {
throw new MetaException("Database name and table name cannot be null.");
}
GetLatestCommittedCompactionInfoResponse response = getTxnHandler().getLatestCommittedCompactionInfo(rqst);
return FilterUtils.filterCommittedCompactionInfoStructIfEnabled(isServerFilterEnabled, filterHook,
getDefaultCatalog(conf), rqst.getDbname(), rqst.getTablename(), response);
}
@Override
public AllocateTableWriteIdsResponse allocate_table_write_ids(
AllocateTableWriteIdsRequest rqst) throws TException {
AllocateTableWriteIdsResponse response = getTxnHandler().allocateTableWriteIds(rqst);
if (listeners != null && !listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ALLOC_WRITE_ID,
new AllocWriteIdEvent(response.getTxnToWriteIds(), rqst.getDbName(),
rqst.getTableName(), this));
}
return response;
}
@Override
public MaxAllocatedTableWriteIdResponse get_max_allocated_table_write_id(MaxAllocatedTableWriteIdRequest rqst)
throws MetaException {
return getTxnHandler().getMaxAllocatedTableWrited(rqst);
}
@Override
public void seed_write_id(SeedTableWriteIdsRequest rqst) throws MetaException {
getTxnHandler().seedWriteId(rqst);
}
@Override
public void seed_txn_id(SeedTxnIdRequest rqst) throws MetaException {
getTxnHandler().seedTxnId(rqst);
}
private void addTxnWriteNotificationLog(Table tableObj, Partition ptnObj, WriteNotificationLogRequest rqst)
throws MetaException {
String partition = ""; //Empty string is an invalid partition name. Can be used for non partitioned table.
if (ptnObj != null) {
partition = Warehouse.makePartName(tableObj.getPartitionKeys(), rqst.getPartitionVals());
}
AcidWriteEvent event = new AcidWriteEvent(partition, tableObj, ptnObj, rqst);
getTxnHandler().addWriteNotificationLog(event);
if (listeners != null && !listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ACID_WRITE, event);
}
}
private Table getTblObject(String db, String table, String catalog) throws MetaException, NoSuchObjectException {
GetTableRequest req = new GetTableRequest(db, table);
if (catalog != null) {
req.setCatName(catalog);
}
req.setCapabilities(new ClientCapabilities(Lists.newArrayList(ClientCapability.TEST_CAPABILITY, ClientCapability.INSERT_ONLY_TABLES)));
return get_table_req(req).getTable();
}
private Partition getPartitionObj(String db, String table, List<String> partitionVals, Table tableObj)
throws MetaException, NoSuchObjectException {
if (tableObj.isSetPartitionKeys() && !tableObj.getPartitionKeys().isEmpty()) {
return get_partition(db, table, partitionVals);
}
return null;
}
@Override
public WriteNotificationLogResponse add_write_notification_log(WriteNotificationLogRequest rqst)
throws TException {
Table tableObj = getTblObject(rqst.getDb(), rqst.getTable(), null);
Partition ptnObj = getPartitionObj(rqst.getDb(), rqst.getTable(), rqst.getPartitionVals(), tableObj);
addTxnWriteNotificationLog(tableObj, ptnObj, rqst);
return new WriteNotificationLogResponse();
}
@Override
public WriteNotificationLogBatchResponse add_write_notification_log_in_batch(
WriteNotificationLogBatchRequest batchRequest) throws TException {
if (batchRequest.getRequestList().size() == 0) {
return new WriteNotificationLogBatchResponse();
}
Table tableObj = getTblObject(batchRequest.getDb(), batchRequest.getTable(), batchRequest.getCatalog());
BatchAcidWriteEvent event = new BatchAcidWriteEvent();
List<String> partNameList = new ArrayList<>();
List<Partition> ptnObjList;
Map<String, WriteNotificationLogRequest> rqstMap = new HashMap<>();
if (tableObj.getPartitionKeys().size() != 0) {
// partitioned table
for (WriteNotificationLogRequest rqst : batchRequest.getRequestList()) {
String partition = Warehouse.makePartName(tableObj.getPartitionKeys(), rqst.getPartitionVals());
partNameList.add(partition);
// This is used to ignore those request for which the partition does not exists.
rqstMap.put(partition, rqst);
}
ptnObjList = getMS().getPartitionsByNames(tableObj.getCatName(), tableObj.getDbName(),
tableObj.getTableName(), partNameList);
} else {
ptnObjList = new ArrayList<>();
for (WriteNotificationLogRequest ignored : batchRequest.getRequestList()) {
ptnObjList.add(null);
}
}
int idx = 0;
for (Partition partObject : ptnObjList) {
String partition = ""; //Empty string is an invalid partition name. Can be used for non partitioned table.
WriteNotificationLogRequest request;
if (partObject != null) {
partition = Warehouse.makePartName(tableObj.getPartitionKeys(), partObject.getValues());
request = rqstMap.get(partition);
} else {
// for non partitioned table, we can get serially from the list.
request = batchRequest.getRequestList().get(idx++);
}
event.addNotification(partition, tableObj, partObject, request);
if (listeners != null && !listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.BATCH_ACID_WRITE,
new BatchAcidWriteEvent(partition, tableObj, partObject, request));
}
}
getTxnHandler().addWriteNotificationLog(event);
return new WriteNotificationLogBatchResponse();
}
@Override
public LockResponse lock(LockRequest rqst) throws TException {
return getTxnHandler().lock(rqst);
}
@Override
public LockResponse check_lock(CheckLockRequest rqst) throws TException {
return getTxnHandler().checkLock(rqst);
}
@Override
public void unlock(UnlockRequest rqst) throws TException {
getTxnHandler().unlock(rqst);
}
@Override
public ShowLocksResponse show_locks(ShowLocksRequest rqst) throws TException {
return getTxnHandler().showLocks(rqst);
}
@Override
public void heartbeat(HeartbeatRequest ids) throws TException {
getTxnHandler().heartbeat(ids);
}
@Override
public HeartbeatTxnRangeResponse heartbeat_txn_range(HeartbeatTxnRangeRequest rqst)
throws TException {
return getTxnHandler().heartbeatTxnRange(rqst);
}
@Deprecated
@Override
public void compact(CompactionRequest rqst) throws TException {
compact2(rqst);
}
@Override
public CompactionResponse compact2(CompactionRequest rqst) throws TException {
return getTxnHandler().compact(rqst);
}
@Override
public ShowCompactResponse show_compact(ShowCompactRequest rqst) throws TException {
ShowCompactResponse response = getTxnHandler().showCompact(rqst);
response.setCompacts(FilterUtils.filterCompactionsIfEnabled(isServerFilterEnabled,
filterHook, getDefaultCatalog(conf), response.getCompacts()));
return response;
}
@Override
public boolean submit_for_cleanup(CompactionRequest rqst, long highestWriteId, long txnId)
throws TException {
return getTxnHandler().submitForCleanup(rqst, highestWriteId, txnId);
}
@Override
public void flushCache() throws TException {
getMS().flushCache();
}
@Override
public void add_dynamic_partitions(AddDynamicPartitions rqst) throws TException {
getTxnHandler().addDynamicPartitions(rqst);
}
@Override
public GetPrincipalsInRoleResponse get_principals_in_role(GetPrincipalsInRoleRequest request)
throws TException {
incrementCounter("get_principals_in_role");
firePreEvent(new PreAuthorizationCallEvent(this));
Exception ex = null;
GetPrincipalsInRoleResponse response = null;
try {
response = new GetPrincipalsInRoleResponse(getMS().listRoleMembers(request.getRoleName()));
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_principals_in_role", ex == null, ex);
}
return response;
}
@Override
public GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(
GetRoleGrantsForPrincipalRequest request) throws TException {
incrementCounter("get_role_grants_for_principal");
firePreEvent(new PreAuthorizationCallEvent(this));
Exception ex = null;
List<RolePrincipalGrant> roleMaps = null;
try {
roleMaps = getMS().listRolesWithGrants(request.getPrincipal_name(), request.getPrincipal_type());
} catch (Exception e) {
ex = e;
rethrowException(e);
} finally {
endFunction("get_role_grants_for_principal", ex == null, ex);
}
//List<RolePrincipalGrant> roleGrantsList = getRolePrincipalGrants(roleMaps);
return new GetRoleGrantsForPrincipalResponse(roleMaps);
}
@Override
public AggrStats get_aggr_stats_for(PartitionsStatsRequest request) throws TException {
String catName = request.isSetCatName() ? request.getCatName().toLowerCase() :
getDefaultCatalog(conf);
String dbName = request.getDbName().toLowerCase();
String tblName = request.getTblName().toLowerCase();
startFunction("get_aggr_stats_for", ": table=" +
TableName.getQualified(catName, dbName, tblName));
List<String> lowerCaseColNames = new ArrayList<>(request.getColNames().size());
for (String colName : request.getColNames()) {
lowerCaseColNames.add(colName.toLowerCase());
}
List<String> lowerCasePartNames = new ArrayList<>(request.getPartNames().size());
for (String partName : request.getPartNames()) {
lowerCasePartNames.add(lowerCaseConvertPartName(partName));
}
AggrStats aggrStats = null;
try {
aggrStats = getMS().get_aggr_stats_for(catName, dbName, tblName,
lowerCasePartNames, lowerCaseColNames, request.getEngine(), request.getValidWriteIdList());
return aggrStats;
} finally {
endFunction("get_aggr_stats_for", aggrStats == null, null, request.getTblName());
}
}
@Override
public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) throws TException {
boolean ret = true;
List<ColumnStatistics> csNews = request.getColStats();
if (csNews == null || csNews.isEmpty()) {
return ret;
}
// figure out if it is table level or partition level
ColumnStatistics firstColStats = csNews.get(0);
ColumnStatisticsDesc statsDesc = firstColStats.getStatsDesc();
String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
String dbName = statsDesc.getDbName();
String tableName = statsDesc.getTableName();
List<String> colNames = new ArrayList<>();
for (ColumnStatisticsObj obj : firstColStats.getStatsObj()) {
colNames.add(obj.getColName());
}
if (statsDesc.isIsTblLevel()) {
// there should be only one ColumnStatistics
if (request.getColStatsSize() != 1) {
throw new MetaException(
"Expecting only 1 ColumnStatistics for table's column stats, but find "
+ request.getColStatsSize());
}
if (request.isSetNeedMerge() && request.isNeedMerge()) {
return updateTableColumnStatsWithMerge(catName, dbName, tableName, colNames, request);
} else {
// This is the overwrite case, we do not care about the accuracy.
return updateTableColumnStatsInternal(firstColStats,
request.getValidWriteIdList(), request.getWriteId());
}
} else {
// partition level column stats merging
// note that we may have two or more duplicate partition names.
// see autoColumnStats_2.q under TestMiniLlapLocalCliDriver
Map<String, ColumnStatistics> newStatsMap = new HashMap<>();
for (ColumnStatistics csNew : csNews) {
String partName = csNew.getStatsDesc().getPartName();
if (newStatsMap.containsKey(partName)) {
MetaStoreServerUtils.mergeColStats(csNew, newStatsMap.get(partName));
}
newStatsMap.put(partName, csNew);
}
if (request.isSetNeedMerge() && request.isNeedMerge()) {
ret = updatePartColumnStatsWithMerge(catName, dbName, tableName,
colNames, newStatsMap, request);
} else { // No merge.
Table t = getTable(catName, dbName, tableName);
MTable mTable = getMS().ensureGetMTable(catName, dbName, tableName);
// We don't short-circuit on errors here anymore. That can leave acid stats invalid.
if (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL)) {
ret = updatePartitionColStatsInBatch(t, newStatsMap,
request.getValidWriteIdList(), request.getWriteId());
} else {
for (Map.Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
// We don't short-circuit on errors here anymore. That can leave acid stats invalid.
ret = updatePartitonColStatsInternal(t, mTable, entry.getValue(),
request.getValidWriteIdList(), request.getWriteId()) && ret;
}
}
}
}
return ret;
}
private boolean updatePartColumnStatsWithMerge(String catName, String dbName, String tableName,
List<String> colNames, Map<String, ColumnStatistics> newStatsMap, SetPartitionsStatsRequest request)
throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
RawStore ms = getMS();
ms.openTransaction();
boolean isCommitted = false, result = false;
try {
// a single call to get all column stats for all partitions
List<String> partitionNames = new ArrayList<>();
partitionNames.addAll(newStatsMap.keySet());
List<ColumnStatistics> csOlds = ms.getPartitionColumnStatistics(catName, dbName, tableName,
partitionNames, colNames, request.getEngine(), request.getValidWriteIdList());
if (newStatsMap.values().size() != csOlds.size()) {
// some of the partitions miss stats.
LOG.debug("Some of the partitions miss stats.");
}
Map<String, ColumnStatistics> oldStatsMap = new HashMap<>();
for (ColumnStatistics csOld : csOlds) {
oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld);
}
// another single call to get all the partition objects
List<Partition> partitions = ms.getPartitionsByNames(catName, dbName, tableName, partitionNames);
Map<String, Partition> mapToPart = new HashMap<>();
for (int index = 0; index < partitionNames.size(); index++) {
mapToPart.put(partitionNames.get(index), partitions.get(index));
}
Table t = getTable(catName, dbName, tableName);
MTable mTable = getMS().ensureGetMTable(catName, dbName, tableName);
Map<String, ColumnStatistics> statsMap = new HashMap<>();
boolean useDirectSql = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL);
for (Map.Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
ColumnStatistics csNew = entry.getValue();
ColumnStatistics csOld = oldStatsMap.get(entry.getKey());
boolean isInvalidTxnStats = csOld != null
&& csOld.isSetIsStatsCompliant() && !csOld.isIsStatsCompliant();
Partition part = mapToPart.get(entry.getKey());
if (isInvalidTxnStats) {
// No columns can be merged; a shortcut for getMergableCols.
csNew.setStatsObj(Lists.newArrayList());
} else {
// we first use getParameters() to prune the stats
MetaStoreServerUtils.getMergableCols(csNew, part.getParameters());
// we merge those that can be merged
if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) {
MetaStoreServerUtils.mergeColStats(csNew, csOld);
}
}
if (!csNew.getStatsObj().isEmpty()) {
// We don't short-circuit on errors here anymore. That can leave acid stats invalid.
if (useDirectSql) {
statsMap.put(csNew.getStatsDesc().getPartName(), csNew);
} else {
result = updatePartitonColStatsInternal(t, mTable, csNew,
request.getValidWriteIdList(), request.getWriteId()) && result;
}
} else if (isInvalidTxnStats) {
// For now because the stats state is such as it is, we will invalidate everything.
// Overall the sematics here are not clear - we could invalide only some columns, but does
// that make any physical sense? Could query affect some columns but not others?
part.setWriteId(request.getWriteId());
StatsSetupConst.clearColumnStatsState(part.getParameters());
StatsSetupConst.setBasicStatsState(part.getParameters(), StatsSetupConst.FALSE);
ms.alterPartition(catName, dbName, tableName, part.getValues(), part,
request.getValidWriteIdList());
result = false;
} else {
// TODO: why doesn't the original call for non acid tables invalidate the stats?
LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName()
+ " are not accurate to merge.");
}
}
ms.commitTransaction();
isCommitted = true;
// updatePartitionColStatsInBatch starts/commit transaction internally. As there is no write or select for update
// operations is done in this transaction, it is safe to commit it before calling updatePartitionColStatsInBatch.
if (!statsMap.isEmpty()) {
updatePartitionColStatsInBatch(t, statsMap, request.getValidWriteIdList(), request.getWriteId());
}
} finally {
if (!isCommitted) {
ms.rollbackTransaction();
}
}
return result;
}
private boolean updateTableColumnStatsWithMerge(String catName, String dbName, String tableName,
List<String> colNames, SetPartitionsStatsRequest request) throws MetaException,
NoSuchObjectException, InvalidObjectException, InvalidInputException {
ColumnStatistics firstColStats = request.getColStats().get(0);
RawStore ms = getMS();
ms.openTransaction();
boolean isCommitted = false, result = false;
try {
ColumnStatistics csOld = ms.getTableColumnStatistics(catName, dbName, tableName, colNames,
request.getEngine(), request.getValidWriteIdList());
// we first use the valid stats list to prune the stats
boolean isInvalidTxnStats = csOld != null
&& csOld.isSetIsStatsCompliant() && !csOld.isIsStatsCompliant();
if (isInvalidTxnStats) {
// No columns can be merged; a shortcut for getMergableCols.
firstColStats.setStatsObj(Lists.newArrayList());
} else {
Table t = getTable(catName, dbName, tableName);
MetaStoreServerUtils.getMergableCols(firstColStats, t.getParameters());
// we merge those that can be merged
if (csOld != null && csOld.getStatsObjSize() != 0 && !firstColStats.getStatsObj().isEmpty()) {
MetaStoreServerUtils.mergeColStats(firstColStats, csOld);
}
}
if (!firstColStats.getStatsObj().isEmpty()) {
result = updateTableColumnStatsInternal(firstColStats,
request.getValidWriteIdList(), request.getWriteId());
} else if (isInvalidTxnStats) {
// For now because the stats state is such as it is, we will invalidate everything.
// Overall the sematics here are not clear - we could invalide only some columns, but does
// that make any physical sense? Could query affect some columns but not others?
Table t = getTable(catName, dbName, tableName);
t.setWriteId(request.getWriteId());
StatsSetupConst.clearColumnStatsState(t.getParameters());
StatsSetupConst.setBasicStatsState(t.getParameters(), StatsSetupConst.FALSE);
ms.alterTable(catName, dbName, tableName, t, request.getValidWriteIdList());
} else {
// TODO: why doesn't the original call for non acid tables invalidate the stats?
LOG.debug("All the column stats are not accurate to merge.");
result = true;
}
ms.commitTransaction();
isCommitted = true;
} finally {
if (!isCommitted) {
ms.rollbackTransaction();
}
}
return result;
}
private Table getTable(String catName, String dbName, String tableName)
throws MetaException, InvalidObjectException {
return getTable(catName, dbName, tableName, null);
}
private Table getTable(String catName, String dbName, String tableName,
String writeIdList)
throws MetaException, InvalidObjectException {
Table t = getMS().getTable(catName, dbName, tableName, writeIdList);
if (t == null) {
throw new InvalidObjectException(TableName.getQualified(catName, dbName, tableName)
+ " table not found");
}
return t;
}
@Override
public NotificationEventResponse get_next_notification(NotificationEventRequest rqst)
throws TException {
authorizeProxyPrivilege();
RawStore ms = getMS();
return ms.getNextNotification(rqst);
}
@Override
public CurrentNotificationEventId get_current_notificationEventId() throws TException {
authorizeProxyPrivilege();
RawStore ms = getMS();
return ms.getCurrentNotificationEventId();
}
@Override
public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
throws TException {
authorizeProxyPrivilege();
RawStore ms = getMS();
return ms.getNotificationEventsCount(rqst);
}
private void authorizeProxyPrivilege() throws TException {
// Skip the auth in embedded mode or if the auth is disabled
if (!HiveMetaStore.isMetaStoreRemote() ||
!MetastoreConf.getBoolVar(conf, ConfVars.EVENT_DB_NOTIFICATION_API_AUTH) || conf.getBoolean(HIVE_IN_TEST.getVarname(),
false)) {
return;
}
String user = null;
try {
user = SecurityUtils.getUGI().getShortUserName();
} catch (Exception ex) {
LOG.error("Cannot obtain username", ex);
throw new TException(ex);
}
if (!MetaStoreServerUtils.checkUserHasHostProxyPrivileges(user, conf, getIPAddress())) {
LOG.error("Not authorized to make the get_notification_events_count call. You can try to disable " +
ConfVars.EVENT_DB_NOTIFICATION_API_AUTH.toString());
throw new TException("User " + user + " is not allowed to perform this API call");
}
}
@Override
public FireEventResponse fire_listener_event(FireEventRequest rqst) throws TException {
switch (rqst.getData().getSetField()) {
case INSERT_DATA:
case INSERT_DATAS:
String catName =
rqst.isSetCatName() ? rqst.getCatName() : getDefaultCatalog(conf);
String dbName = rqst.getDbName();
String tblName = rqst.getTableName();
boolean isSuccessful = rqst.isSuccessful();
List<InsertEvent> events = new ArrayList<>();
if (rqst.getData().isSetInsertData()) {
events.add(new InsertEvent(catName, dbName, tblName,
rqst.getPartitionVals(),
rqst.getData().getInsertData(), isSuccessful, this));
} else {
// this is a bulk fire insert event operation
// we use the partition values field from the InsertEventRequestData object
// instead of the FireEventRequest object
for (InsertEventRequestData insertData : rqst.getData().getInsertDatas()) {
if (!insertData.isSetPartitionVal()) {
throw new MetaException(
"Partition values must be set when firing multiple insert events");
}
events.add(new InsertEvent(catName, dbName, tblName,
insertData.getPartitionVal(),
insertData, isSuccessful, this));
}
}
FireEventResponse response = new FireEventResponse();
for (InsertEvent event : events) {
/*
* The transactional listener response will be set already on the event, so there is not need
* to pass the response to the non-transactional listener.
*/
MetaStoreListenerNotifier
.notifyEvent(transactionalListeners, EventType.INSERT, event);
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.INSERT, event);
if (event.getParameters() != null && event.getParameters()
.containsKey(
MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) {
response.addToEventIds(Long.valueOf(event.getParameters()
.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)));
} else {
String msg = "Insert event id not generated for ";
if (event.getPartitionObj() != null) {
msg += "partition " + Arrays
.toString(event.getPartitionObj().getValues().toArray()) + " of ";
}
msg +=
" of table " + event.getTableObj().getDbName() + "." + event.getTableObj()
.getTableName();
LOG.warn(msg);
}
}
return response;
case REFRESH_EVENT:
response = new FireEventResponse();
catName = rqst.isSetCatName() ? rqst.getCatName() : getDefaultCatalog(conf);
dbName = rqst.getDbName();
tblName = rqst.getTableName();
List<String> partitionVals = rqst.getPartitionVals();
Map<String, String> tableParams = rqst.getTblParams();
ReloadEvent event = new ReloadEvent(catName, dbName, tblName, partitionVals, rqst.isSuccessful(),
rqst.getData().getRefreshEvent(), tableParams, this);
MetaStoreListenerNotifier
.notifyEvent(transactionalListeners, EventType.RELOAD, event);
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.RELOAD, event);
if (event.getParameters() != null && event.getParameters()
.containsKey(
MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) {
response.addToEventIds(Long.valueOf(event.getParameters()
.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)));
} else {
String msg = "Reload event id not generated for ";
if (event.getPartitionObj() != null) {
msg += "partition " + Arrays
.toString(event.getPartitionObj().getValues().toArray()) + " of ";
}
msg +=
" of table " + event.getTableObj().getDbName() + "." + event.getTableObj()
.getTableName();
LOG.warn(msg);
}
return response;
default:
throw new TException("Event type " + rqst.getData().getSetField().toString()
+ " not currently supported.");
}
}
@Override
public GetFileMetadataByExprResult get_file_metadata_by_expr(GetFileMetadataByExprRequest req)
throws TException {
GetFileMetadataByExprResult result = new GetFileMetadataByExprResult();
RawStore ms = getMS();
if (!ms.isFileMetadataSupported()) {
result.setIsSupported(false);
result.setMetadata(Collections.emptyMap()); // Set the required field.
return result;
}
result.setIsSupported(true);
List<Long> fileIds = req.getFileIds();
boolean needMetadata = !req.isSetDoGetFooters() || req.isDoGetFooters();
FileMetadataExprType type = req.isSetType() ? req.getType() : FileMetadataExprType.ORC_SARG;
ByteBuffer[] metadatas = needMetadata ? new ByteBuffer[fileIds.size()] : null;
ByteBuffer[] ppdResults = new ByteBuffer[fileIds.size()];
boolean[] eliminated = new boolean[fileIds.size()];
getMS().getFileMetadataByExpr(fileIds, type, req.getExpr(), metadatas, ppdResults, eliminated);
for (int i = 0; i < fileIds.size(); ++i) {
if (!eliminated[i] && ppdResults[i] == null)
{
continue; // No metadata => no ppd.
}
MetadataPpdResult mpr = new MetadataPpdResult();
ByteBuffer ppdResult = eliminated[i] ? null : handleReadOnlyBufferForThrift(ppdResults[i]);
mpr.setIncludeBitset(ppdResult);
if (needMetadata) {
ByteBuffer metadata = eliminated[i] ? null : handleReadOnlyBufferForThrift(metadatas[i]);
mpr.setMetadata(metadata);
}
result.putToMetadata(fileIds.get(i), mpr);
}
if (!result.isSetMetadata()) {
result.setMetadata(Collections.emptyMap()); // Set the required field.
}
return result;
}
@Override
public GetFileMetadataResult get_file_metadata(GetFileMetadataRequest req) throws TException {
GetFileMetadataResult result = new GetFileMetadataResult();
RawStore ms = getMS();
if (!ms.isFileMetadataSupported()) {
result.setIsSupported(false);
result.setMetadata(Collections.emptyMap()); // Set the required field.
return result;
}
result.setIsSupported(true);
List<Long> fileIds = req.getFileIds();
ByteBuffer[] metadatas = ms.getFileMetadata(fileIds);
assert metadatas.length == fileIds.size();
for (int i = 0; i < metadatas.length; ++i) {
ByteBuffer bb = metadatas[i];
if (bb == null) {
continue;
}
bb = handleReadOnlyBufferForThrift(bb);
result.putToMetadata(fileIds.get(i), bb);
}
if (!result.isSetMetadata()) {
result.setMetadata(Collections.emptyMap()); // Set the required field.
}
return result;
}
private ByteBuffer handleReadOnlyBufferForThrift(ByteBuffer bb) {
if (!bb.isReadOnly()) {
return bb;
}
// Thrift cannot write read-only buffers... oh well.
// TODO: actually thrift never writes to the buffer, so we could use reflection to
// unset the unnecessary read-only flag if allocation/copy perf becomes a problem.
ByteBuffer copy = ByteBuffer.allocate(bb.capacity());
copy.put(bb);
copy.flip();
return copy;
}
@Override
public PutFileMetadataResult put_file_metadata(PutFileMetadataRequest req) throws TException {
RawStore ms = getMS();
if (ms.isFileMetadataSupported()) {
ms.putFileMetadata(req.getFileIds(), req.getMetadata(), req.getType());
}
return new PutFileMetadataResult();
}
@Override
public ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req)
throws TException {
getMS().putFileMetadata(req.getFileIds(), null, null);
return new ClearFileMetadataResult();
}
@Override
public CacheFileMetadataResult cache_file_metadata(
CacheFileMetadataRequest req) throws TException {
RawStore ms = getMS();
if (!ms.isFileMetadataSupported()) {
return new CacheFileMetadataResult(false);
}
String dbName = req.getDbName(), tblName = req.getTblName(),
partName = req.isSetPartName() ? req.getPartName() : null;
boolean isAllPart = req.isSetIsAllParts() && req.isIsAllParts();
ms.openTransaction();
boolean success = false;
try {
Table tbl = ms.getTable(getDefaultCatalog(conf), dbName, tblName);
if (tbl == null) {
throw new NoSuchObjectException(dbName + "." + tblName + " not found");
}
boolean isPartitioned = tbl.isSetPartitionKeys() && tbl.getPartitionKeysSize() > 0;
String tableInputFormat = tbl.isSetSd() ? tbl.getSd().getInputFormat() : null;
if (!isPartitioned) {
if (partName != null || isAllPart) {
throw new MetaException("Table is not partitioned");
}
if (!tbl.isSetSd() || !tbl.getSd().isSetLocation()) {
throw new MetaException(
"Table does not have storage location; this operation is not supported on views");
}
FileMetadataExprType type = expressionProxy.getMetadataType(tableInputFormat);
if (type == null) {
throw new MetaException("The operation is not supported for " + tableInputFormat);
}
fileMetadataManager.queueCacheMetadata(tbl.getSd().getLocation(), type);
success = true;
} else {
List<String> partNames;
if (partName != null) {
partNames = Lists.newArrayList(partName);
} else if (isAllPart) {
partNames = ms.listPartitionNames(getDefaultCatalog(conf), dbName, tblName, (short)-1);
} else {
throw new MetaException("Table is partitioned");
}
int batchSize = MetastoreConf.getIntVar(
conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
int index = 0;
int successCount = 0, failCount = 0;
HashSet<String> failFormats = null;
while (index < partNames.size()) {
int currentBatchSize = Math.min(batchSize, partNames.size() - index);
List<String> nameBatch = partNames.subList(index, index + currentBatchSize);
index += currentBatchSize;
List<Partition> parts = ms.getPartitionsByNames(getDefaultCatalog(conf), dbName, tblName, nameBatch);
for (Partition part : parts) {
if (!part.isSetSd() || !part.getSd().isSetLocation()) {
throw new MetaException("Partition does not have storage location;" +
" this operation is not supported on views");
}
String inputFormat = part.getSd().isSetInputFormat()
? part.getSd().getInputFormat() : tableInputFormat;
FileMetadataExprType type = expressionProxy.getMetadataType(inputFormat);
if (type == null) {
++failCount;
if (failFormats == null) {
failFormats = new HashSet<>();
}
failFormats.add(inputFormat);
} else {
++successCount;
fileMetadataManager.queueCacheMetadata(part.getSd().getLocation(), type);
}
}
}
success = true; // Regardless of the following exception
if (failCount > 0) {
String errorMsg = "The operation failed for " + failCount + " partitions and "
+ "succeeded for " + successCount + " partitions; unsupported formats: ";
boolean isFirst = true;
for (String s : failFormats) {
if (!isFirst) {
errorMsg += ", ";
}
isFirst = false;
errorMsg += s;
}
throw new MetaException(errorMsg);
}
}
} finally {
if (success) {
if (!ms.commitTransaction()) {
throw new MetaException("Failed to commit");
}
} else {
ms.rollbackTransaction();
}
}
return new CacheFileMetadataResult(true);
}
@VisibleForTesting
void updateMetrics() throws MetaException {
if (Metrics.getRegistry() != null) {
LOG.info("Begin calculating metadata count metrics.");
Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).set(getMS().getTableCount());
Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).set(getMS().getPartitionCount());
Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).set(getMS().getDatabaseCount());
}
}
@Override
public PrimaryKeysResponse get_primary_keys(PrimaryKeysRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startTableFunction("get_primary_keys", request.getCatName(), request.getDb_name(), request.getTbl_name());
List<SQLPrimaryKey> ret = null;
Exception ex = null;
try {
ret = getMS().getPrimaryKeys(request);
} catch (Exception e) {
ex = e;
throwMetaException(e);
} finally {
endFunction("get_primary_keys", ret != null, ex, request.getTbl_name());
}
return new PrimaryKeysResponse(ret);
}
@Override
public ForeignKeysResponse get_foreign_keys(ForeignKeysRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startFunction("get_foreign_keys",
" : parentdb=" + request.getParent_db_name() + " parenttbl=" + request.getParent_tbl_name() + " foreigndb="
+ request.getForeign_db_name() + " foreigntbl=" + request.getForeign_tbl_name());
List<SQLForeignKey> ret = null;
Exception ex = null;
try {
ret = getMS().getForeignKeys(request);
} catch (Exception e) {
ex = e;
throwMetaException(e);
} finally {
endFunction("get_foreign_keys", ret != null, ex, request.getForeign_tbl_name());
}
return new ForeignKeysResponse(ret);
}
@Override
public UniqueConstraintsResponse get_unique_constraints(UniqueConstraintsRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startTableFunction("get_unique_constraints", request.getCatName(), request.getDb_name(), request.getTbl_name());
List<SQLUniqueConstraint> ret = null;
Exception ex = null;
try {
ret = getMS().getUniqueConstraints(request);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_unique_constraints", ret != null, ex, request.getTbl_name());
}
return new UniqueConstraintsResponse(ret);
}
@Override
public NotNullConstraintsResponse get_not_null_constraints(NotNullConstraintsRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startTableFunction("get_not_null_constraints", request.getCatName(), request.getDb_name(), request.getTbl_name());
List<SQLNotNullConstraint> ret = null;
Exception ex = null;
try {
ret = getMS().getNotNullConstraints(request);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_not_null_constraints", ret != null, ex, request.getTbl_name());
}
return new NotNullConstraintsResponse(ret);
}
@Override
public DefaultConstraintsResponse get_default_constraints(DefaultConstraintsRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startTableFunction("get_default_constraints", request.getCatName(), request.getDb_name(), request.getTbl_name());
List<SQLDefaultConstraint> ret = null;
Exception ex = null;
try {
ret = getMS().getDefaultConstraints(request);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_default_constraints", ret != null, ex, request.getTbl_name());
}
return new DefaultConstraintsResponse(ret);
}
@Override
public CheckConstraintsResponse get_check_constraints(CheckConstraintsRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startTableFunction("get_check_constraints", request.getCatName(), request.getDb_name(), request.getTbl_name());
List<SQLCheckConstraint> ret = null;
Exception ex = null;
try {
ret = getMS().getCheckConstraints(request);
} catch (Exception e) {
ex = e;
throw newMetaException(e);
} finally {
endFunction("get_check_constraints", ret != null, ex, request.getTbl_name());
}
return new CheckConstraintsResponse(ret);
}
/**
* Api to fetch all table constraints at once.
* @param request it consist of catalog name, database name and table name to identify the table in metastore
* @return all constraints attached to given table
* @throws TException
*/
@Override
public AllTableConstraintsResponse get_all_table_constraints(AllTableConstraintsRequest request)
throws TException, MetaException, NoSuchObjectException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
startTableFunction("get_all_table_constraints", request.getCatName(), request.getDbName(), request.getTblName());
SQLAllTableConstraints ret = null;
Exception ex = null;
try {
ret = getMS().getAllTableConstraints(request);
} catch (Exception e) {
ex = e;
throwMetaException(e);
} finally {
endFunction("get_all_table_constraints", ret != null, ex, request.getTblName());
}
return new AllTableConstraintsResponse(ret);
}
@Override
public String get_metastore_db_uuid() throws TException {
try {
return getMS().getMetastoreDbUuid();
} catch (MetaException e) {
LOG.error("Exception thrown while querying metastore db uuid", e);
throw e;
}
}
@Override
public WMCreateResourcePlanResponse create_resource_plan(WMCreateResourcePlanRequest request)
throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
int defaultPoolSize = MetastoreConf.getIntVar(
conf, MetastoreConf.ConfVars.WM_DEFAULT_POOL_SIZE);
WMResourcePlan plan = request.getResourcePlan();
if (defaultPoolSize > 0 && plan.isSetQueryParallelism()) {
// If the default pool is not disabled, override the size with the specified parallelism.
defaultPoolSize = plan.getQueryParallelism();
}
try {
getMS().createResourcePlan(plan, request.getCopyFrom(), defaultPoolSize);
return new WMCreateResourcePlanResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to persist resource plan", e);
throw e;
}
}
@Override
public WMGetResourcePlanResponse get_resource_plan(WMGetResourcePlanRequest request)
throws NoSuchObjectException, MetaException, TException {
try {
WMFullResourcePlan rp = getMS().getResourcePlan(request.getResourcePlanName(), request.getNs());
WMGetResourcePlanResponse resp = new WMGetResourcePlanResponse();
resp.setResourcePlan(rp);
return resp;
} catch (MetaException e) {
LOG.error("Exception while trying to retrieve resource plan", e);
throw e;
}
}
@Override
public WMGetAllResourcePlanResponse get_all_resource_plans(WMGetAllResourcePlanRequest request)
throws MetaException, TException {
try {
WMGetAllResourcePlanResponse resp = new WMGetAllResourcePlanResponse();
resp.setResourcePlans(getMS().getAllResourcePlans(request.getNs()));
return resp;
} catch (MetaException e) {
LOG.error("Exception while trying to retrieve resource plans", e);
throw e;
}
}
@Override
public WMAlterResourcePlanResponse alter_resource_plan(WMAlterResourcePlanRequest request)
throws NoSuchObjectException, InvalidOperationException, MetaException, TException {
try {
if (((request.isIsEnableAndActivate() ? 1 : 0) + (request.isIsReplace() ? 1 : 0)
+ (request.isIsForceDeactivate() ? 1 : 0)) > 1) {
throw new MetaException("Invalid request; multiple flags are set");
}
WMAlterResourcePlanResponse response = new WMAlterResourcePlanResponse();
// This method will only return full resource plan when activating one,
// to give the caller the result atomically with the activation.
WMFullResourcePlan fullPlanAfterAlter = getMS().alterResourcePlan(
request.getResourcePlanName(), request.getNs(), request.getResourcePlan(),
request.isIsEnableAndActivate(), request.isIsForceDeactivate(), request.isIsReplace());
if (fullPlanAfterAlter != null) {
response.setFullResourcePlan(fullPlanAfterAlter);
}
return response;
} catch (MetaException e) {
LOG.error("Exception while trying to alter resource plan", e);
throw e;
}
}
@Override
public WMGetActiveResourcePlanResponse get_active_resource_plan(
WMGetActiveResourcePlanRequest request) throws MetaException, TException {
try {
WMGetActiveResourcePlanResponse response = new WMGetActiveResourcePlanResponse();
response.setResourcePlan(getMS().getActiveResourcePlan(request.getNs()));
return response;
} catch (MetaException e) {
LOG.error("Exception while trying to get active resource plan", e);
throw e;
}
}
@Override
public WMValidateResourcePlanResponse validate_resource_plan(WMValidateResourcePlanRequest request)
throws NoSuchObjectException, MetaException, TException {
try {
return getMS().validateResourcePlan(request.getResourcePlanName(), request.getNs());
} catch (MetaException e) {
LOG.error("Exception while trying to validate resource plan", e);
throw e;
}
}
@Override
public WMDropResourcePlanResponse drop_resource_plan(WMDropResourcePlanRequest request)
throws NoSuchObjectException, InvalidOperationException, MetaException, TException {
try {
getMS().dropResourcePlan(request.getResourcePlanName(), request.getNs());
return new WMDropResourcePlanResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to drop resource plan", e);
throw e;
}
}
@Override
public WMCreateTriggerResponse create_wm_trigger(WMCreateTriggerRequest request)
throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
try {
getMS().createWMTrigger(request.getTrigger());
return new WMCreateTriggerResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to create trigger", e);
throw e;
}
}
@Override
public WMAlterTriggerResponse alter_wm_trigger(WMAlterTriggerRequest request)
throws NoSuchObjectException, InvalidObjectException, MetaException, TException {
try {
getMS().alterWMTrigger(request.getTrigger());
return new WMAlterTriggerResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to alter trigger", e);
throw e;
}
}
@Override
public WMDropTriggerResponse drop_wm_trigger(WMDropTriggerRequest request)
throws NoSuchObjectException, InvalidOperationException, MetaException, TException {
try {
getMS().dropWMTrigger(request.getResourcePlanName(), request.getTriggerName(), request.getNs());
return new WMDropTriggerResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to drop trigger.", e);
throw e;
}
}
@Override
public WMGetTriggersForResourePlanResponse get_triggers_for_resourceplan(
WMGetTriggersForResourePlanRequest request)
throws NoSuchObjectException, MetaException, TException {
try {
List<WMTrigger> triggers =
getMS().getTriggersForResourcePlan(request.getResourcePlanName(), request.getNs());
WMGetTriggersForResourePlanResponse response = new WMGetTriggersForResourePlanResponse();
response.setTriggers(triggers);
return response;
} catch (MetaException e) {
LOG.error("Exception while trying to retrieve triggers plans", e);
throw e;
}
}
@Override
public WMAlterPoolResponse alter_wm_pool(WMAlterPoolRequest request)
throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException,
TException {
try {
getMS().alterPool(request.getPool(), request.getPoolPath());
return new WMAlterPoolResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to alter WMPool", e);
throw e;
}
}
@Override
public WMCreatePoolResponse create_wm_pool(WMCreatePoolRequest request)
throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException,
TException {
try {
getMS().createPool(request.getPool());
return new WMCreatePoolResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to create WMPool", e);
throw e;
}
}
@Override
public WMDropPoolResponse drop_wm_pool(WMDropPoolRequest request)
throws NoSuchObjectException, InvalidOperationException, MetaException, TException {
try {
getMS().dropWMPool(request.getResourcePlanName(), request.getPoolPath(), request.getNs());
return new WMDropPoolResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to drop WMPool", e);
throw e;
}
}
@Override
public WMCreateOrUpdateMappingResponse create_or_update_wm_mapping(
WMCreateOrUpdateMappingRequest request) throws AlreadyExistsException,
NoSuchObjectException, InvalidObjectException, MetaException, TException {
try {
getMS().createOrUpdateWMMapping(request.getMapping(), request.isUpdate());
return new WMCreateOrUpdateMappingResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to create or update WMMapping", e);
throw e;
}
}
@Override
public WMDropMappingResponse drop_wm_mapping(WMDropMappingRequest request)
throws NoSuchObjectException, InvalidOperationException, MetaException, TException {
try {
getMS().dropWMMapping(request.getMapping());
return new WMDropMappingResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to drop WMMapping", e);
throw e;
}
}
@Override
public WMCreateOrDropTriggerToPoolMappingResponse create_or_drop_wm_trigger_to_pool_mapping(
WMCreateOrDropTriggerToPoolMappingRequest request) throws AlreadyExistsException,
NoSuchObjectException, InvalidObjectException, MetaException, TException {
try {
if (request.isDrop()) {
getMS().dropWMTriggerToPoolMapping(request.getResourcePlanName(),
request.getTriggerName(), request.getPoolPath(), request.getNs());
} else {
getMS().createWMTriggerToPoolMapping(request.getResourcePlanName(),
request.getTriggerName(), request.getPoolPath(), request.getNs());
}
return new WMCreateOrDropTriggerToPoolMappingResponse();
} catch (MetaException e) {
LOG.error("Exception while trying to create or drop pool mappings", e);
throw e;
}
}
@Override
public void create_ischema(ISchema schema) throws TException {
startFunction("create_ischema", ": " + schema.getName());
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
try {
firePreEvent(new PreCreateISchemaEvent(this, schema));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.createISchema(schema);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.CREATE_ISCHEMA, new CreateISchemaEvent(true, this, schema));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.CREATE_ISCHEMA,
new CreateISchemaEvent(success, this, schema), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|AlreadyExistsException e) {
LOG.error("Caught exception creating schema", e);
ex = e;
throw e;
} finally {
endFunction("create_ischema", success, ex);
}
}
@Override
public void alter_ischema(AlterISchemaRequest rqst) throws TException {
startFunction("alter_ischema", ": " + rqst);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
try {
ISchema oldSchema = ms.getISchema(rqst.getName());
if (oldSchema == null) {
throw new NoSuchObjectException("Could not find schema " + rqst.getName());
}
firePreEvent(new PreAlterISchemaEvent(this, oldSchema, rqst.getNewSchema()));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.alterISchema(rqst.getName(), rqst.getNewSchema());
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_ISCHEMA, new AlterISchemaEvent(true, this, oldSchema, rqst.getNewSchema()));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ALTER_ISCHEMA,
new AlterISchemaEvent(success, this, oldSchema, rqst.getNewSchema()), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|NoSuchObjectException e) {
LOG.error("Caught exception altering schema", e);
ex = e;
throw e;
} finally {
endFunction("alter_ischema", success, ex);
}
}
@Override
public ISchema get_ischema(ISchemaName schemaName) throws TException {
startFunction("get_ischema", ": " + schemaName);
Exception ex = null;
ISchema schema = null;
try {
schema = getMS().getISchema(schemaName);
if (schema == null) {
throw new NoSuchObjectException("No schema named " + schemaName + " exists");
}
firePreEvent(new PreReadISchemaEvent(this, schema));
return schema;
} catch (MetaException e) {
LOG.error("Caught exception getting schema", e);
ex = e;
throw e;
} finally {
endFunction("get_ischema", schema != null, ex);
}
}
@Override
public void drop_ischema(ISchemaName schemaName) throws TException {
startFunction("drop_ischema", ": " + schemaName);
Exception ex = null;
boolean success = false;
RawStore ms = getMS();
try {
// look for any valid versions. This will also throw NoSuchObjectException if the schema
// itself doesn't exist, which is what we want.
SchemaVersion latest = ms.getLatestSchemaVersion(schemaName);
if (latest != null) {
ex = new InvalidOperationException("Schema " + schemaName + " cannot be dropped, it has" +
" at least one valid version");
throw (InvalidObjectException)ex;
}
ISchema schema = ms.getISchema(schemaName);
firePreEvent(new PreDropISchemaEvent(this, schema));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.dropISchema(schemaName);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_ISCHEMA, new DropISchemaEvent(true, this, schema));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.DROP_ISCHEMA,
new DropISchemaEvent(success, this, schema), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|NoSuchObjectException e) {
LOG.error("Caught exception dropping schema", e);
ex = e;
throw e;
} finally {
endFunction("drop_ischema", success, ex);
}
}
@Override
public void add_schema_version(SchemaVersion schemaVersion) throws TException {
startFunction("add_schema_version", ": " + schemaVersion);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
try {
// Make sure the referenced schema exists
if (ms.getISchema(schemaVersion.getSchema()) == null) {
throw new NoSuchObjectException("No schema named " + schemaVersion.getSchema());
}
firePreEvent(new PreAddSchemaVersionEvent(this, schemaVersion));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.addSchemaVersion(schemaVersion);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ADD_SCHEMA_VERSION, new AddSchemaVersionEvent(true, this, schemaVersion));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_SCHEMA_VERSION,
new AddSchemaVersionEvent(success, this, schemaVersion), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|AlreadyExistsException e) {
LOG.error("Caught exception adding schema version", e);
ex = e;
throw e;
} finally {
endFunction("add_schema_version", success, ex);
}
}
@Override
public SchemaVersion get_schema_version(SchemaVersionDescriptor version) throws TException {
startFunction("get_schema_version", ": " + version);
Exception ex = null;
SchemaVersion schemaVersion = null;
try {
schemaVersion = getMS().getSchemaVersion(version);
if (schemaVersion == null) {
throw new NoSuchObjectException("No schema version " + version + "exists");
}
firePreEvent(new PreReadhSchemaVersionEvent(this, Collections.singletonList(schemaVersion)));
return schemaVersion;
} catch (MetaException e) {
LOG.error("Caught exception getting schema version", e);
ex = e;
throw e;
} finally {
endFunction("get_schema_version", schemaVersion != null, ex);
}
}
@Override
public SchemaVersion get_schema_latest_version(ISchemaName schemaName) throws TException {
startFunction("get_latest_schema_version", ": " + schemaName);
Exception ex = null;
SchemaVersion schemaVersion = null;
try {
schemaVersion = getMS().getLatestSchemaVersion(schemaName);
if (schemaVersion == null) {
throw new NoSuchObjectException("No versions of schema " + schemaName + "exist");
}
firePreEvent(new PreReadhSchemaVersionEvent(this, Collections.singletonList(schemaVersion)));
return schemaVersion;
} catch (MetaException e) {
LOG.error("Caught exception getting latest schema version", e);
ex = e;
throw e;
} finally {
endFunction("get_latest_schema_version", schemaVersion != null, ex);
}
}
@Override
public List<SchemaVersion> get_schema_all_versions(ISchemaName schemaName) throws TException {
startFunction("get_all_schema_versions", ": " + schemaName);
Exception ex = null;
List<SchemaVersion> schemaVersions = null;
try {
schemaVersions = getMS().getAllSchemaVersion(schemaName);
if (schemaVersions == null) {
throw new NoSuchObjectException("No versions of schema " + schemaName + "exist");
}
firePreEvent(new PreReadhSchemaVersionEvent(this, schemaVersions));
return schemaVersions;
} catch (MetaException e) {
LOG.error("Caught exception getting all schema versions", e);
ex = e;
throw e;
} finally {
endFunction("get_all_schema_versions", schemaVersions != null, ex);
}
}
@Override
public void drop_schema_version(SchemaVersionDescriptor version) throws TException {
startFunction("drop_schema_version", ": " + version);
Exception ex = null;
boolean success = false;
RawStore ms = getMS();
try {
SchemaVersion schemaVersion = ms.getSchemaVersion(version);
if (schemaVersion == null) {
throw new NoSuchObjectException("No schema version " + version);
}
firePreEvent(new PreDropSchemaVersionEvent(this, schemaVersion));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.dropSchemaVersion(version);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.DROP_SCHEMA_VERSION, new DropSchemaVersionEvent(true, this, schemaVersion));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.DROP_SCHEMA_VERSION,
new DropSchemaVersionEvent(success, this, schemaVersion), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|NoSuchObjectException e) {
LOG.error("Caught exception dropping schema version", e);
ex = e;
throw e;
} finally {
endFunction("drop_schema_version", success, ex);
}
}
@Override
public FindSchemasByColsResp get_schemas_by_cols(FindSchemasByColsRqst rqst) throws TException {
startFunction("get_schemas_by_cols");
Exception ex = null;
List<SchemaVersion> schemaVersions = Collections.emptyList();
try {
schemaVersions = getMS().getSchemaVersionsByColumns(rqst.getColName(),
rqst.getColNamespace(), rqst.getType());
firePreEvent(new PreReadhSchemaVersionEvent(this, schemaVersions));
final List<SchemaVersionDescriptor> entries = new ArrayList<>(schemaVersions.size());
schemaVersions.forEach(schemaVersion -> entries.add(
new SchemaVersionDescriptor(schemaVersion.getSchema(), schemaVersion.getVersion())));
return new FindSchemasByColsResp(entries);
} catch (MetaException e) {
LOG.error("Caught exception doing schema version query", e);
ex = e;
throw e;
} finally {
endFunction("get_schemas_by_cols", !schemaVersions.isEmpty(), ex);
}
}
@Override
public void map_schema_version_to_serde(MapSchemaVersionToSerdeRequest rqst)
throws TException {
startFunction("map_schema_version_to_serde, :" + rqst);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
try {
SchemaVersion oldSchemaVersion = ms.getSchemaVersion(rqst.getSchemaVersion());
if (oldSchemaVersion == null) {
throw new NoSuchObjectException("No schema version " + rqst.getSchemaVersion());
}
SerDeInfo serde = ms.getSerDeInfo(rqst.getSerdeName());
if (serde == null) {
throw new NoSuchObjectException("No SerDe named " + rqst.getSerdeName());
}
SchemaVersion newSchemaVersion = new SchemaVersion(oldSchemaVersion);
newSchemaVersion.setSerDe(serde);
firePreEvent(new PreAlterSchemaVersionEvent(this, oldSchemaVersion, newSchemaVersion));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.alterSchemaVersion(rqst.getSchemaVersion(), newSchemaVersion);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_SCHEMA_VERSION, new AlterSchemaVersionEvent(true, this,
oldSchemaVersion, newSchemaVersion));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ALTER_SCHEMA_VERSION,
new AlterSchemaVersionEvent(success, this, oldSchemaVersion, newSchemaVersion), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|NoSuchObjectException e) {
LOG.error("Caught exception mapping schema version to serde", e);
ex = e;
throw e;
} finally {
endFunction("map_schema_version_to_serde", success, ex);
}
}
@Override
public void set_schema_version_state(SetSchemaVersionStateRequest rqst) throws TException {
startFunction("set_schema_version_state, :" + rqst);
boolean success = false;
Exception ex = null;
RawStore ms = getMS();
try {
SchemaVersion oldSchemaVersion = ms.getSchemaVersion(rqst.getSchemaVersion());
if (oldSchemaVersion == null) {
throw new NoSuchObjectException("No schema version " + rqst.getSchemaVersion());
}
SchemaVersion newSchemaVersion = new SchemaVersion(oldSchemaVersion);
newSchemaVersion.setState(rqst.getState());
firePreEvent(new PreAlterSchemaVersionEvent(this, oldSchemaVersion, newSchemaVersion));
Map<String, String> transactionalListenersResponses = Collections.emptyMap();
ms.openTransaction();
try {
ms.alterSchemaVersion(rqst.getSchemaVersion(), newSchemaVersion);
if (!transactionalListeners.isEmpty()) {
transactionalListenersResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventType.ALTER_SCHEMA_VERSION, new AlterSchemaVersionEvent(true, this,
oldSchemaVersion, newSchemaVersion));
}
success = ms.commitTransaction();
} finally {
if (!success) {
ms.rollbackTransaction();
}
if (!listeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ALTER_SCHEMA_VERSION,
new AlterSchemaVersionEvent(success, this, oldSchemaVersion, newSchemaVersion), null,
transactionalListenersResponses, ms);
}
}
} catch (MetaException|NoSuchObjectException e) {
LOG.error("Caught exception changing schema version state", e);
ex = e;
throw e;
} finally {
endFunction("set_schema_version_state", success, ex);
}
}
@Override
public void add_serde(SerDeInfo serde) throws TException {
startFunction("create_serde", ": " + serde.getName());
Exception ex = null;
boolean success = false;
RawStore ms = getMS();
try {
ms.openTransaction();
ms.addSerde(serde);
success = ms.commitTransaction();
} catch (MetaException|AlreadyExistsException e) {
LOG.error("Caught exception creating serde", e);
ex = e;
throw e;
} finally {
if (!success) {
ms.rollbackTransaction();
}
endFunction("create_serde", success, ex);
}
}
@Override
public SerDeInfo get_serde(GetSerdeRequest rqst) throws TException {
startFunction("get_serde", ": " + rqst);
Exception ex = null;
SerDeInfo serde = null;
try {
serde = getMS().getSerDeInfo(rqst.getSerdeName());
if (serde == null) {
throw new NoSuchObjectException("No serde named " + rqst.getSerdeName() + " exists");
}
return serde;
} catch (MetaException e) {
LOG.error("Caught exception getting serde", e);
ex = e;
throw e;
} finally {
endFunction("get_serde", serde != null, ex);
}
}
@Override
public LockResponse get_lock_materialization_rebuild(String dbName, String tableName, long txnId)
throws TException {
return getTxnHandler().lockMaterializationRebuild(dbName, tableName, txnId);
}
@Override
public boolean heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId)
throws TException {
return getTxnHandler().heartbeatLockMaterializationRebuild(dbName, tableName, txnId);
}
@Override
public void add_runtime_stats(RuntimeStat stat) throws TException {
startFunction("store_runtime_stats");
Exception ex = null;
boolean success = false;
RawStore ms = getMS();
try {
ms.openTransaction();
ms.addRuntimeStat(stat);
success = ms.commitTransaction();
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
if (!success) {
ms.rollbackTransaction();
}
endFunction("store_runtime_stats", success, ex);
}
}
@Override
public List<RuntimeStat> get_runtime_stats(GetRuntimeStatsRequest rqst) throws TException {
startFunction("get_runtime_stats");
Exception ex = null;
try {
List<RuntimeStat> res = getMS().getRuntimeStats(rqst.getMaxWeight(), rqst.getMaxCreateTime());
return res;
} catch (MetaException e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("get_runtime_stats", ex == null, ex);
}
}
@Override
public ScheduledQueryPollResponse scheduled_query_poll(ScheduledQueryPollRequest request)
throws MetaException, TException {
startFunction("scheduled_query_poll");
Exception ex = null;
try {
RawStore ms = getMS();
return ms.scheduledQueryPoll(request);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("scheduled_query_poll", ex == null, ex);
}
}
@Override
public void scheduled_query_maintenance(ScheduledQueryMaintenanceRequest request) throws MetaException, TException {
startFunction("scheduled_query_poll");
Exception ex = null;
try {
RawStore ms = getMS();
ms.scheduledQueryMaintenance(request);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("scheduled_query_poll", ex == null, ex);
}
}
@Override
public void scheduled_query_progress(ScheduledQueryProgressInfo info) throws MetaException, TException {
startFunction("scheduled_query_poll");
Exception ex = null;
try {
RawStore ms = getMS();
ms.scheduledQueryProgress(info);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("scheduled_query_poll", ex == null, ex);
}
}
@Override
public ScheduledQuery get_scheduled_query(ScheduledQueryKey scheduleKey) throws TException {
startFunction("get_scheduled_query");
Exception ex = null;
try {
return getMS().getScheduledQuery(scheduleKey);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("get_scheduled_query", ex == null, ex);
}
}
@Override
public void add_replication_metrics(ReplicationMetricList replicationMetricList) throws MetaException{
startFunction("add_replication_metrics");
Exception ex = null;
try {
getMS().addReplicationMetrics(replicationMetricList);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("add_replication_metrics", ex == null, ex);
}
}
@Override
public ReplicationMetricList get_replication_metrics(GetReplicationMetricsRequest
getReplicationMetricsRequest) throws MetaException{
startFunction("get_replication_metrics");
Exception ex = null;
try {
return getMS().getReplicationMetrics(getReplicationMetricsRequest);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("get_replication_metrics", ex == null, ex);
}
}
@Override
public void create_stored_procedure(StoredProcedure proc) throws NoSuchObjectException, MetaException {
startFunction("create_stored_procedure");
Exception ex = null;
throwUnsupportedExceptionIfRemoteDB(proc.getDbName(), "create_stored_procedure");
try {
getMS().createOrUpdateStoredProcedure(proc);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("create_stored_procedure", ex == null, ex);
}
}
public StoredProcedure get_stored_procedure(StoredProcedureRequest request) throws MetaException, NoSuchObjectException {
startFunction("get_stored_procedure");
Exception ex = null;
try {
StoredProcedure proc = getMS().getStoredProcedure(request.getCatName(), request.getDbName(), request.getProcName());
if (proc == null) {
throw new NoSuchObjectException(
"HPL/SQL StoredProcedure " + request.getDbName() + "." + request.getProcName() + " does not exist");
}
return proc;
} catch (Exception e) {
if (!(e instanceof NoSuchObjectException)) {
LOG.error("Caught exception", e);
}
ex = e;
throw e;
} finally {
endFunction("get_stored_procedure", ex == null, ex);
}
}
@Override
public void drop_stored_procedure(StoredProcedureRequest request) throws MetaException {
startFunction("drop_stored_procedure");
Exception ex = null;
try {
getMS().dropStoredProcedure(request.getCatName(), request.getDbName(), request.getProcName());
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("drop_stored_procedure", ex == null, ex);
}
}
@Override
public List<String> get_all_stored_procedures(ListStoredProcedureRequest request) throws MetaException {
startFunction("get_all_stored_procedures");
Exception ex = null;
try {
return getMS().getAllStoredProcedures(request);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("get_all_stored_procedures", ex == null, ex);
}
}
public Package find_package(GetPackageRequest request) throws MetaException, NoSuchObjectException {
startFunction("find_package");
Exception ex = null;
try {
Package pkg = getMS().findPackage(request);
if (pkg == null) {
throw new NoSuchObjectException(
"HPL/SQL package " + request.getDbName() + "." + request.getPackageName() + " does not exist");
}
return pkg;
} catch (Exception e) {
if (!(e instanceof NoSuchObjectException)) {
LOG.error("Caught exception", e);
}
ex = e;
throw e;
} finally {
endFunction("find_package", ex == null, ex);
}
}
public void add_package(AddPackageRequest request) throws MetaException, NoSuchObjectException {
startFunction("add_package");
Exception ex = null;
try {
getMS().addPackage(request);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("add_package", ex == null, ex);
}
}
public List<String> get_all_packages(ListPackageRequest request) throws MetaException {
startFunction("get_all_packages");
Exception ex = null;
try {
return getMS().listPackages(request);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("get_all_packages", ex == null, ex);
}
}
public void drop_package(DropPackageRequest request) throws MetaException {
startFunction("drop_package");
Exception ex = null;
try {
getMS().dropPackage(request);
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("drop_package", ex == null, ex);
}
}
@Override
public List<WriteEventInfo> get_all_write_event_info(GetAllWriteEventInfoRequest request)
throws MetaException {
startFunction("get_all_write_event_info");
Exception ex = null;
try {
List<WriteEventInfo> writeEventInfoList =
getMS().getAllWriteEventInfo(request.getTxnId(), request.getDbName(), request.getTableName());
return writeEventInfoList == null ? Collections.emptyList() : writeEventInfoList;
} catch (Exception e) {
LOG.error("Caught exception", e);
ex = e;
throw e;
} finally {
endFunction("get_all_write_event_info", ex == null, ex);
}
}
}