ATLAS-4339: Atlas should support skip temporary tables using config property
Signed-off-by: Sarath Subramanian <sarath@apache.org>
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index ab8e183..14cc2f2 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -58,19 +58,20 @@
private boolean isSkippedInputEntity;
private boolean isSkippedOutputEntity;
+ private boolean skipTempTables;
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HookContext hiveContext,
- HiveHookObjectNamesCache knownObjects) throws Exception {
- this(hook, hiveOperation, hiveContext, knownObjects, null, null);
+ HiveHookObjectNamesCache knownObjects, boolean skipTempTables) throws Exception {
+ this(hook, hiveOperation, hiveContext, knownObjects, null, null, skipTempTables);
}
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HiveHookObjectNamesCache knownObjects,
- HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent) throws Exception {
- this(hook, hiveOperation, null, knownObjects, metastoreHook, listenerEvent);
+ HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent, boolean skipTempTables) throws Exception {
+ this(hook, hiveOperation, null, knownObjects, metastoreHook, listenerEvent, skipTempTables);
}
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HookContext hiveContext, HiveHookObjectNamesCache knownObjects,
- HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent) throws Exception {
+ HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent, boolean skipTempTables) throws Exception {
this.hook = hook;
this.hiveOperation = hiveOperation;
this.hiveContext = hiveContext;
@@ -79,6 +80,7 @@
this.metastoreHook = metastoreHook;
this.metastoreEvent = listenerEvent;
this.metastoreHandler = (listenerEvent != null) ? metastoreEvent.getIHMSHandler() : null;
+ this.skipTempTables = skipTempTables;
init();
}
@@ -131,6 +133,10 @@
}
}
+ public boolean isSkipTempTables() {
+ return skipTempTables;
+ }
+
public LineageInfo getLineageInfo() {
return hiveContext != null ? hiveContext.getLinfo() : null;
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 37084d5..3cc7b3b 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -70,7 +70,8 @@
public static final String HOOK_HIVE_TABLE_PRUNE_PATTERN = CONF_PREFIX + "hive_table.prune.pattern";
public static final String HOOK_HIVE_TABLE_CACHE_SIZE = CONF_PREFIX + "hive_table.cache.size";
public static final String HOOK_HIVE_IGNORE_DDL_OPERATIONS = CONF_PREFIX + "hs2.ignore.ddl.operations";
- public static final String HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX + "hs2.filter.entity.additional.types.to.retain";
+ public static final String HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX + "hs2.filter.entity.additional.types.to.retain";
+ public static final String HOOK_HIVE_SKIP_TEMP_TABLES = CONF_PREFIX + "skip.temp.tables";
public static final String DEFAULT_HOST_NAME = "localhost";
private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>();
@@ -93,6 +94,7 @@
private static final boolean hiveProcessPopulateDeprecatedAttributes;
private static HiveHookObjectNamesCache knownObjects = null;
private static String hostName;
+ private static boolean skipTempTables = true;
static {
for (HiveOperation hiveOperation : HiveOperation.values()) {
@@ -154,6 +156,7 @@
ignoreDummyDatabaseName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name", defaultDummyDatabase);
ignoreDummyTableName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name", defaultDummyTable);
ignoreValuesTmpTableNamePrefix = atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix", "Values__Tmp__Table__");
+ skipTempTables = atlasProperties.getBoolean(HOOK_HIVE_SKIP_TEMP_TABLES, true);
try {
hostName = InetAddress.getLocalHost().getHostName();
@@ -182,7 +185,7 @@
try {
HiveOperation oper = OPERATION_MAP.get(hookContext.getOperationName());
- AtlasHiveHookContext context = new AtlasHiveHookContext(this, oper, hookContext, getKnownObjects());
+ AtlasHiveHookContext context = new AtlasHiveHookContext(this, oper, hookContext, getKnownObjects(), isSkipTempTables());
BaseHiveEvent event = null;
switch (oper) {
@@ -201,7 +204,7 @@
break;
case CREATETABLE:
- event = new CreateTable(context, true);
+ event = new CreateTable(context);
break;
case DROPTABLE:
@@ -218,7 +221,7 @@
case EXPORT:
case IMPORT:
case QUERY:
- event = new CreateHiveProcess(context, true);
+ event = new CreateHiveProcess(context);
break;
case ALTERTABLE_FILEFORMAT:
@@ -297,6 +300,10 @@
return hiveProcessPopulateDeprecatedAttributes;
}
+ public static boolean isSkipTempTables() {
+ return skipTempTables;
+ }
+
public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE;
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
index f01419c..6a492c2 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
@@ -115,7 +115,7 @@
try {
HiveOperation oper = operContext.getOperation();
- AtlasHiveHookContext context = new AtlasHiveHookContext(hiveHook, oper, hiveHook.getKnownObjects(), this, listenerEvent);
+ AtlasHiveHookContext context = new AtlasHiveHookContext(hiveHook, oper, hiveHook.getKnownObjects(), this, listenerEvent, hiveHook.isSkipTempTables());
BaseHiveEvent event = null;
switch (oper) {
@@ -132,7 +132,7 @@
break;
case CREATETABLE:
- event = new CreateTable(context, true);
+ event = new CreateTable(context);
break;
case DROPTABLE:
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
index e164370..d2f09cc 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
@@ -29,7 +29,7 @@
public class AlterTable extends CreateTable {
public AlterTable(AtlasHiveHookContext context) {
- super(context, true);
+ super(context);
}
@Override
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 2e8237b..3f35813 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -162,6 +162,7 @@
public static final Map<Integer, String> OWNER_TYPE_TO_ENUM_VALUE = new HashMap<>();
+ protected final boolean skipTempTables;
static {
OWNER_TYPE_TO_ENUM_VALUE.put(1, "USER");
@@ -173,7 +174,8 @@
protected BaseHiveEvent(AtlasHiveHookContext context) {
- this.context = context;
+ this.context = context;
+ this.skipTempTables = context.isSkipTempTables();
}
public AtlasHiveHookContext getContext() {
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index e5295f4..5787c93 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -49,12 +49,9 @@
public class CreateHiveProcess extends BaseHiveEvent {
private static final Logger LOG = LoggerFactory.getLogger(CreateHiveProcess.class);
- private final boolean skipTempTables;
- public CreateHiveProcess(AtlasHiveHookContext context, boolean skipTempTables) {
+ public CreateHiveProcess(AtlasHiveHookContext context) {
super(context);
-
- this.skipTempTables = skipTempTables;
}
@Override
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
index 63fc894..91611de 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
@@ -42,12 +42,9 @@
public class CreateTable extends BaseHiveEvent {
private static final Logger LOG = LoggerFactory.getLogger(CreateTable.class);
- private final boolean skipTempTables;
- public CreateTable(AtlasHiveHookContext context, boolean skipTempTables) {
+ public CreateTable(AtlasHiveHookContext context) {
super(context);
-
- this.skipTempTables = skipTempTables;
}
@Override
