ATLAS-4324: FS entity created for 'load data inpath' Part 2
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 94ef225..37084d5 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -70,7 +70,7 @@
public static final String HOOK_HIVE_TABLE_PRUNE_PATTERN = CONF_PREFIX + "hive_table.prune.pattern";
public static final String HOOK_HIVE_TABLE_CACHE_SIZE = CONF_PREFIX + "hive_table.cache.size";
public static final String HOOK_HIVE_IGNORE_DDL_OPERATIONS = CONF_PREFIX + "hs2.ignore.ddl.operations";
- public static final String HOOK_HIVE_FILTER_ENTITY_TYPES_TO_RETAIN = CONF_PREFIX + "hs2.filter.entity.types.to.retain";
+ public static final String HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX + "hs2.filter.entity.additional.types.to.retain";
public static final String DEFAULT_HOST_NAME = "localhost";
private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>();
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
index 737c637..0b0d4d6 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
@@ -18,11 +18,13 @@
package org.apache.atlas.hive.hook.utils;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.model.notification.HookNotification;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Arrays;
import java.util.List;
import static org.apache.atlas.hive.hook.HiveHook.HOOK_HIVE_IGNORE_DDL_OPERATIONS;
@@ -34,16 +36,32 @@
public static void init(Configuration configuration) {
boolean skipDdlOperations = configuration.getBoolean(HOOK_HIVE_IGNORE_DDL_OPERATIONS, false);
- init(skipDdlOperations);
+ List<String> additionalTypesToRetain = getConfiguredTypesToRetainForDDLEntityFilter(configuration);
+
+ init(skipDdlOperations, additionalTypesToRetain);
LOG.info("atlas.hook.hive.ignore.ddl.operations={} - {}", skipDdlOperations, entityFilter.getClass().getSimpleName());
}
@VisibleForTesting
- static void init(boolean lineageOnlyFilter) {
- entityFilter = lineageOnlyFilter ? new HiveDDLEntityFilter() : new PassthroughFilter();
+ static void init(boolean lineageOnlyFilter, List<String> additionalTypesToRetain) {
+ entityFilter = lineageOnlyFilter ? new HiveDDLEntityFilter(additionalTypesToRetain) : new PassthroughFilter();
}
public static List<HookNotification> apply(List<HookNotification> incoming) {
return entityFilter.apply(incoming);
}
+
+ private static List<String> getConfiguredTypesToRetainForDDLEntityFilter(Configuration configuration) {
+ try {
+ if (configuration.containsKey(HiveHook.HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN)) {
+ String[] configuredTypes = configuration.getStringArray(HiveHook.HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN);
+
+ return Arrays.asList(configuredTypes);
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to load application properties", e);
+ }
+
+ return null;
+ }
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
index 9163c47..3ce3942 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
@@ -18,8 +18,6 @@
package org.apache.atlas.hive.hook.utils;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.atlas.ApplicationProperties;
-import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.hive.hook.events.BaseHiveEvent;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
@@ -43,7 +41,7 @@
public class HiveDDLEntityFilter implements EntityFilter {
private static final Logger LOG = LoggerFactory.getLogger(HiveDDLEntityFilter.class);
- private static final Set<String> defaultPathTypes = new HashSet<String>() {{
+ private static final Set<String> defaultPathTypesToRetain = new HashSet<String>() {{
add(AtlasPathExtractorUtil.HDFS_TYPE_PATH);
add(AtlasPathExtractorUtil.ADLS_GEN2_DIRECTORY);
add(AtlasPathExtractorUtil.GCS_VIRTUAL_DIR);
@@ -57,20 +55,16 @@
add(BaseHiveEvent.HIVE_TYPE_COLUMN_LINEAGE);
add(BaseHiveEvent.HIVE_DB_DDL);
add(BaseHiveEvent.HIVE_TABLE_DDL);
- addAll(defaultPathTypes);
- addAll(getConfiguredTypesToRetain());
+ addAll(defaultPathTypesToRetain);
}};
- private static List<String> getConfiguredTypesToRetain() {
- String[] configuredTypesToRetain = null;
-
- try {
- configuredTypesToRetain = ApplicationProperties.get().getStringArray(HiveHook.HOOK_HIVE_FILTER_ENTITY_TYPES_TO_RETAIN);
- } catch (Exception e) {
- LOG.error("Failed to load application properties", e);
+ public HiveDDLEntityFilter(List<String> additionalTypesToRetain) {
+ if (CollectionUtils.isEmpty(additionalTypesToRetain)) {
+ return;
}
- return configuredTypesToRetain != null ? Arrays.asList(configuredTypesToRetain) : new ArrayList<>();
+ typesToRetain.addAll(additionalTypesToRetain);
+ LOG.info("Types retained: {}", typesToRetain.toArray());
}
public List<HookNotification> apply(List<HookNotification> incoming) {
@@ -197,7 +191,7 @@
AtlasObjectId oid = (AtlasObjectId) o;
String typeName = oid.getTypeName();
- if (oid.getUniqueAttributes() != null && !defaultPathTypes.contains(typeName)) {
+ if (oid.getUniqueAttributes() != null && !typesToRetain.contains(typeName)) {
oid.setGuid(null);
}
} else {
@@ -208,8 +202,8 @@
String typeName = hm.containsKey(AtlasObjectId.KEY_TYPENAME) ? (String) hm.get(AtlasObjectId.KEY_TYPENAME) : null;
- if (hm.containsKey(BaseHiveEvent.ATTRIBUTE_UNIQUE_ATTRIBUTES) && !defaultPathTypes.contains(typeName)) {
- hm.put(BaseHiveEvent.ATTRIBUTE_GUID, null);
+ if (hm.containsKey(BaseHiveEvent.ATTRIBUTE_UNIQUE_ATTRIBUTES) && !typesToRetain.contains(typeName)) {
+ hm.remove(BaseHiveEvent.ATTRIBUTE_GUID);
}
}
}
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
index 4dde1dc..a201214 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
@@ -21,13 +21,13 @@
import org.apache.atlas.model.notification.HookNotification;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.TestResourceFileUtils;
-import org.apache.commons.collections.MapUtils;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -35,16 +35,17 @@
import java.util.Set;
import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
public class ActiveEntityFilterTest {
- private static String FILE_SUFFIX = "-v2";
+ private static String FILE_SUFFIX_ACTUAL_RESULTS = "-v2";
+ private static String ADDITIONAL_TYPE_HDFS_PATH = "hdfs_path";
@BeforeClass
public void setup() {
- ActiveEntityFilter.init(true);
+ ActiveEntityFilter.init(true, Arrays.asList(new String[]{ADDITIONAL_TYPE_HDFS_PATH}));
}
@Test
@@ -56,14 +57,12 @@
assertMessageFromFile("hs2-alter-view");
assertMessageFromFile("hs2-drop-table");
assertAtlasEntitiesWithExtInfoFromFile("hs2-create-process");
- assertAtlasEntitiesWithExtInfoFromFile("hs2-load-inpath");
- assertAtlasEntitiesWithExtInfoFromFile("hs2-create-db-with-no-pathentities-to-retain", false);
- assertAtlasEntitiesWithExtInfoFromFile("hs2-load-inpath-with-no-pathentities-to-retain", false);
+ assertMessageFromFile("hs2-load-inpath");
}
private void assertMessageFromFile(String msgFile) throws IOException {
List incoming = loadList(msgFile);
- List expected = loadList(msgFile + FILE_SUFFIX);
+ List expected = loadList(msgFile + FILE_SUFFIX_ACTUAL_RESULTS);
int expectedSize = expected.size();
List<HookNotification> actual = ActiveEntityFilter.apply((List<HookNotification>) incoming);
@@ -139,51 +138,30 @@
}
private void assertAtlasEntitiesWithExtInfoFromFile(String entityFile) throws IOException {
- assertAtlasEntitiesWithExtInfoFromFile(entityFile, true);
- }
-
- private void assertAtlasEntitiesWithExtInfoFromFile(String entityFile, boolean retainPathEntities) throws IOException {
AtlasEntity.AtlasEntitiesWithExtInfo incoming = TestResourceFileUtils.readObjectFromJson("", entityFile, AtlasEntity.AtlasEntitiesWithExtInfo.class);
- AtlasEntity.AtlasEntitiesWithExtInfo expected = TestResourceFileUtils.readObjectFromJson("", entityFile + FILE_SUFFIX, AtlasEntity.AtlasEntitiesWithExtInfo.class);
+ AtlasEntity.AtlasEntitiesWithExtInfo expected = TestResourceFileUtils.readObjectFromJson("", entityFile + FILE_SUFFIX_ACTUAL_RESULTS, AtlasEntity.AtlasEntitiesWithExtInfo.class);
- HiveDDLEntityFilter hiveLineageEntityFilter = new HiveDDLEntityFilter();
+ HiveDDLEntityFilter hiveLineageEntityFilter = new HiveDDLEntityFilter(null);
AtlasEntity.AtlasEntitiesWithExtInfo actual = hiveLineageEntityFilter.apply(incoming);
-
- if (retainPathEntities) {
- assertAtlasEntitiesWithExtInfo(actual, expected);
- } else {
- assertAtlasEntitiesWithNoPathEntitiesToRetain(actual, expected);
- }
+ assertAtlasEntitiesWithExtInfo(actual, expected);
}
private void assertAtlasEntitiesWithExtInfo(AtlasEntity.AtlasEntitiesWithExtInfo actual, AtlasEntity.AtlasEntitiesWithExtInfo expected) {
assertNotNull(actual);
assertNotNull(expected);
- if (expected.getEntities() != null && actual.getEntities() != null) {
- assertEquals(actual.getEntities().size(), expected.getEntities().size());
- assertEntity(actual.getEntities(), expected.getEntities());
+ assertEquals(actual.getEntities().size(), expected.getEntities().size());
+ assertEntity(actual.getEntities(), expected.getEntities());
+
+ if (expected.getReferredEntities() == null && actual.getReferredEntities() != null) {
+ fail("expected.getReferredEntities() == null, but expected.getReferredEntities() != null");
}
- assertEquals(MapUtils.isEmpty(actual.getReferredEntities()), MapUtils.isEmpty(expected.getReferredEntities()));
if (expected.getReferredEntities() != null && actual.getReferredEntities() != null) {
assertEntity(actual.getReferredEntities(), expected.getReferredEntities());
}
}
- private void assertAtlasEntitiesWithNoPathEntitiesToRetain(AtlasEntity.AtlasEntitiesWithExtInfo actual, AtlasEntity.AtlasEntitiesWithExtInfo expected) {
- assertNotNull(actual);
- assertNotNull(expected);
-
- if (expected.getEntities() != null && actual.getEntities() != null) {
- assertNotEquals(actual.getEntities().size(), expected.getEntities().size());
- }
-
- if (expected.getReferredEntities() != null && actual.getReferredEntities() != null) {
- assertNotEquals(actual.getReferredEntities().size(), expected.getReferredEntities().size());
- }
- }
-
private void assertEntity(Map<String, AtlasEntity> actual, Map<String, AtlasEntity> expected) {
assertEquals(actual.size(), expected.size());
}
@@ -225,7 +203,7 @@
String actualJson = AtlasType.toJson(actualEntity);
String expectedJson = AtlasType.toJson(expectedEntity);
- assertEquals(AtlasType.fromJson(actualJson, LinkedHashMap.class), AtlasType.fromJson(expectedJson, LinkedHashMap.class));
+ Assert.assertEquals(actualJson, expectedJson, "Actual: " + actualJson);
}
}
}
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
index 881ee10..28d3b6b 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
@@ -11,7 +11,7 @@
"name": "create database cadb02",
"userName": "hive"
},
- "guid": "-14530890092409411",
+ "guid": "-14529329955589449",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
@@ -40,7 +40,6 @@
"version": 0,
"relationshipAttributes": {
"hiveDb": {
- "guid": "-14529329955589448",
"typeName": "hive_db",
"uniqueAttributes": {
"qualifiedName": "cadb02@cm"
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain-v2.json
deleted file mode 100644
index 42553b5..0000000
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain-v2.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "referredEntities": {},
- "entities": [
- {
- "typeName": "hive_db_ddl",
- "attributes": {
- "serviceType": "hive",
- "qualifiedName": "cadb02@cm:1616450673617",
- "execTime": 1616450673617,
- "queryText": "create database cadb02",
- "name": "create database cadb02",
- "userName": "hive"
- },
- "guid": "-14530890092409411",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "relationshipAttributes": {
- "db": {
- "typeName": "hive_db",
- "uniqueAttributes": {
- "qualifiedName": "cadb02@cm"
- },
- "relationshipType": "hive_db_ddl_queries"
- }
- },
- "proxy": false
- }
- ]
-}
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain.json b/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain.json
deleted file mode 100644
index a5b810f..0000000
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "referredEntities": {},
- "entities": [
- {
- "typeName": "hive_db",
- "attributes": {
- "owner": "hive",
- "ownerType": "USER",
- "managedLocation": null,
- "qualifiedName": "cadb02@cm",
- "clusterName": "cm",
- "name": "cadb02",
- "location": "hdfs://ve0126.halxg.cloudera.com:8020/warehouse/tablespace/external/hive/cadb02.db",
- "parameters": {}
- },
- "guid": "-14529329955589448",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
- },
- {
- "typeName": "hive_db_ddl",
- "attributes": {
- "serviceType": "hive",
- "qualifiedName": "cadb02@cm:1616450673617",
- "execTime": 1616450673617,
- "queryText": "create database cadb02",
- "name": "create database cadb02",
- "userName": "hive"
- },
- "guid": "-14529329955589449",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "relationshipAttributes": {
- "db": {
- "guid": "-14529329955589448",
- "typeName": "hive_db",
- "uniqueAttributes": {
- "qualifiedName": "cadb02@cm"
- },
- "relationshipType": "hive_db_ddl_queries"
- }
- },
- "proxy": false
- },
- {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/warehouse/tablespace/external/hive/cadb02.db",
- "qualifiedName": "hdfs://ve0126.halxg.cloudera.com:8020/warehouse/tablespace/external/hive/cadb02.db@cm",
- "clusterName": "cm",
- "name": "/warehouse/tablespace/external/hive/cadb02.db"
- },
- "guid": "-14529329955589450",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "relationshipAttributes": {
- "hiveDb": {
- "guid": "-14529329955589448",
- "typeName": "hive_db",
- "uniqueAttributes": {
- "qualifiedName": "cadb02@cm"
- },
- "relationshipType": "hive_db_location"
- }
- },
- "proxy": false
- }
- ]
-}
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
index 8e55b72..9291cde 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
@@ -17,7 +17,6 @@
"version": 0,
"relationshipAttributes": {
"table": {
- "guid": "-44808597128610",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb202.vw202@primary"
@@ -51,7 +50,6 @@
"relationshipAttributes": {
"outputs": [
{
- "guid": "-44808597128610",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb202.vw202@primary"
@@ -112,7 +110,6 @@
"relationshipAttributes": {
"outputs": [
{
- "guid": "-44808597128612",
"typeName": "hive_column",
"uniqueAttributes": {
"qualifiedName": "cadb202.vw202.col202@primary"
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
index 801918e..ebf9e51 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
@@ -40,7 +40,6 @@
"relationshipAttributes": {
"outputs": [
{
- "guid": "-14529329955589452",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb02.hh6@cm"
@@ -103,7 +102,6 @@
"version": 0,
"relationshipAttributes": {
"table": {
- "guid": "-14529329955589452",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb02.hh6@cm"
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
index dd31aa0..499a9c2 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
@@ -1,18 +1,96 @@
-{
- "referredEntities": {
- "-14529329955589455": {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv",
- "qualifiedName": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv@cm",
- "clusterName": "cm",
- "name": "/tmp/external/hh6.csv"
+[
+ {
+ "type": "ENTITY_CREATE_V2",
+ "user": "hive",
+ "entities": {
+ "referredEntities": {
+ "-98504074851374": {
+ "typeName": "hdfs_path",
+ "attributes": {
+ "path": "file:/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "qualifiedName": "file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "clusterName": "primary",
+ "name": "/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "guid": "-98504074851374",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "proxy": false
+ }
},
- "guid": "-14529329955589455",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
+ "entities": [
+ {
+ "typeName": "hive_process",
+ "attributes": {
+ "recentQueries": [
+ "load data local inpath 'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj' into table table_nrx8uoggc0 partition(dt = '2015-01-01')"
+ ],
+ "qualifiedName": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "clusterName": "primary",
+ "name": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "queryText": "",
+ "operationType": "LOAD",
+ "startTime": 1622738659471,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "",
+ "queryId": ""
+ },
+ "guid": "-98504074851381",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "outputs": [
+ {
+ "typeName": "hive_table",
+ "uniqueAttributes": {
+ "qualifiedName": "default.table_nrx8uoggc0@primary"
+ },
+ "relationshipType": "process_dataset_outputs"
+ }
+ ],
+ "inputs": [
+ {
+ "guid": "-98504074851374",
+ "typeName": "hdfs_path",
+ "uniqueAttributes": {
+ "qualifiedName": "file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "relationshipType": "dataset_process_inputs"
+ }
+ ]
+ },
+ "proxy": false
+ },
+ {
+ "typeName": "hive_process_execution",
+ "attributes": {
+ "hostName": "21806.local",
+ "qualifiedName": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "name": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "queryText": "load data local inpath 'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj' into table table_nrx8uoggc0 partition(dt = '2015-01-01')",
+ "startTime": 1622738658982,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "hive",
+ "queryId": "hive_20210603094308_ef789483-7de1-462b-ac74-bb0ebe7aeedf"
+ },
+ "guid": "-98504074851382",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "process": {
+ "guid": "-98504074851381",
+ "typeName": "hive_process",
+ "relationshipType": "hive_process_process_executions"
+ }
+ },
+ "proxy": false
+ }
+ ]
}
}
-}
\ No newline at end of file
+]
\ No newline at end of file
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain-v2.json
deleted file mode 100644
index 7f90d19..0000000
--- a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain-v2.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "referredEntities": {
- }
-}
\ No newline at end of file
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain.json b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain.json
deleted file mode 100644
index dd31aa0..0000000
--- a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "referredEntities": {
- "-14529329955589455": {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv",
- "qualifiedName": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv@cm",
- "clusterName": "cm",
- "name": "/tmp/external/hh6.csv"
- },
- "guid": "-14529329955589455",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
- }
- }
-}
\ No newline at end of file
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
index dd31aa0..499a9c2 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
@@ -1,18 +1,96 @@
-{
- "referredEntities": {
- "-14529329955589455": {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv",
- "qualifiedName": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv@cm",
- "clusterName": "cm",
- "name": "/tmp/external/hh6.csv"
+[
+ {
+ "type": "ENTITY_CREATE_V2",
+ "user": "hive",
+ "entities": {
+ "referredEntities": {
+ "-98504074851374": {
+ "typeName": "hdfs_path",
+ "attributes": {
+ "path": "file:/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "qualifiedName": "file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "clusterName": "primary",
+ "name": "/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "guid": "-98504074851374",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "proxy": false
+ }
},
- "guid": "-14529329955589455",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
+ "entities": [
+ {
+ "typeName": "hive_process",
+ "attributes": {
+ "recentQueries": [
+ "load data local inpath 'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj' into table table_nrx8uoggc0 partition(dt = '2015-01-01')"
+ ],
+ "qualifiedName": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "clusterName": "primary",
+ "name": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "queryText": "",
+ "operationType": "LOAD",
+ "startTime": 1622738659471,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "",
+ "queryId": ""
+ },
+ "guid": "-98504074851381",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "outputs": [
+ {
+ "typeName": "hive_table",
+ "uniqueAttributes": {
+ "qualifiedName": "default.table_nrx8uoggc0@primary"
+ },
+ "relationshipType": "process_dataset_outputs"
+ }
+ ],
+ "inputs": [
+ {
+ "guid": "-98504074851374",
+ "typeName": "hdfs_path",
+ "uniqueAttributes": {
+ "qualifiedName": "file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "relationshipType": "dataset_process_inputs"
+ }
+ ]
+ },
+ "proxy": false
+ },
+ {
+ "typeName": "hive_process_execution",
+ "attributes": {
+ "hostName": "21806.local",
+ "qualifiedName": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "name": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "queryText": "load data local inpath 'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj' into table table_nrx8uoggc0 partition(dt = '2015-01-01')",
+ "startTime": 1622738658982,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "hive",
+ "queryId": "hive_20210603094308_ef789483-7de1-462b-ac74-bb0ebe7aeedf"
+ },
+ "guid": "-98504074851382",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "process": {
+ "guid": "-98504074851381",
+ "typeName": "hive_process",
+ "relationshipType": "hive_process_process_executions"
+ }
+ },
+ "proxy": false
+ }
+ ]
}
}
-}
\ No newline at end of file
+]
\ No newline at end of file
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json b/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
index f2b2bf7..f133e7f 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
@@ -8,13 +8,13 @@
"typeName": "hive_table_ddl",
"attributes": {
"serviceType": "hive",
- "qualifiedName": "cadb02.hh6_renamed@cm:1616452234545",
- "execTime": 1616452234545,
+ "qualifiedName": "cadb02.hh6_renamed@cm:1616450674247",
+ "execTime": 1616450674247,
"queryText": "ALTER TABLE hh6 RENAME TO hh6_renamed",
"name": "ALTER TABLE hh6 RENAME TO hh6_renamed",
"userName": "hive"
},
- "guid": "-14530890092409428",
+ "guid": "-14529329955589467",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
