ATLAS-4125: JavaPatch to add new supertypes to existing entities
Signed-off-by: Madhan Neethiraj <madhan@apache.org>
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 2b809a6..3b2bbe0 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -83,6 +83,7 @@
public static final String ATTRIBUTE_OWNER = "owner";
public static final String ATTRIBUTE_CLUSTER_NAME = "clusterName";
public static final String ATTRIBUTE_LOCATION = "location";
+ public static final String ATTRIBUTE_LOCATION_PATH = "locationPath";
public static final String ATTRIBUTE_PARAMETERS = "parameters";
public static final String ATTRIBUTE_OWNER_TYPE = "ownerType";
public static final String ATTRIBUTE_COMMENT = "comment";
@@ -94,6 +95,7 @@
public static final String ATTRIBUTE_TEMPORARY = "temporary";
public static final String ATTRIBUTE_RETENTION = "retention";
public static final String ATTRIBUTE_DB = "db";
+ public static final String ATTRIBUTE_HIVE_DB = "hiveDb";
public static final String ATTRIBUTE_STORAGEDESC = "sd";
public static final String ATTRIBUTE_PARTITION_KEYS = "partitionKeys";
public static final String ATTRIBUTE_COLUMNS = "columns";
@@ -151,6 +153,7 @@
public static final String RELATIONSHIP_HIVE_TABLE_STORAGE_DESC = "hive_table_storagedesc";
public static final String RELATIONSHIP_HIVE_PROCESS_PROCESS_EXE = "hive_process_process_executions";
public static final String RELATIONSHIP_HIVE_DB_DDL_QUERIES = "hive_db_ddl_queries";
+ public static final String RELATIONSHIP_HIVE_DB_LOCATION = "hive_db_location";
public static final String RELATIONSHIP_HIVE_TABLE_DDL_QUERIES = "hive_table_ddl_queries";
public static final String RELATIONSHIP_HBASE_TABLE_NAMESPACE = "hbase_table_namespace";
@@ -689,6 +692,33 @@
return hiveDDL;
}
+ protected AtlasEntity createHiveLocationEntity(AtlasEntity dbEntity, AtlasEntitiesWithExtInfo extInfoEntity) {
+ AtlasEntity ret = null;
+ String locationUri = (String)dbEntity.getAttribute(ATTRIBUTE_LOCATION);
+
+ if (StringUtils.isNotEmpty(locationUri)) {
+ Path path = null;
+
+ try {
+ path = new Path(locationUri);
+ } catch (IllegalArgumentException excp) {
+ LOG.warn("failed to create Path from locationUri {}", locationUri, excp);
+ }
+
+ if (path != null) {
+ ret = getPathEntity(path, extInfoEntity);
+
+ if (ret != null) {
+ AtlasRelatedObjectId dbRelatedObjectId = AtlasTypeUtil.getAtlasRelatedObjectId(dbEntity, RELATIONSHIP_HIVE_DB_LOCATION);
+
+ ret.setRelationshipAttribute(ATTRIBUTE_HIVE_DB, dbRelatedObjectId);
+ }
+ }
+ }
+
+ return ret;
+ }
+
protected String getMetadataNamespace() {
return context.getMetadataNamespace();
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java
index c8a9ce3..8305a44 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java
@@ -97,6 +97,11 @@
if (dbDDLEntity != null) {
ret.addEntity(dbDDLEntity);
}
+
+ AtlasEntity dbLocationEntity = createHiveLocationEntity(dbEntity, ret);
+ if (dbLocationEntity != null) {
+ ret.addEntity(dbLocationEntity);
+ }
} else {
LOG.error("CreateDatabase.getEntities(): failed to retrieve db");
}
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 5c7777b..fccfc48 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -32,6 +32,7 @@
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
@@ -61,6 +62,7 @@
import static org.apache.atlas.AtlasClient.NAME;
import static org.apache.atlas.hive.hook.events.BaseHiveEvent.*;
import static org.testng.Assert.*;
+import static org.testng.AssertJUnit.assertEquals;
public class HiveHookIT extends HiveITBase {
private static final Logger LOG = LoggerFactory.getLogger(HiveHookIT.class);
@@ -118,6 +120,59 @@
}
@Test
+ public void testPathEntityDefAvailable() throws Exception {
+ //Check if Path entity definition created or not
+ AtlasEntityDef pathEntityDef = atlasClientV2.getEntityDefByName("Path");
+ assertNotNull(pathEntityDef);
+ }
+
+ @Test
+ public void testCreateDatabaseWithLocation() throws Exception {
+ String dbName = dbName();
+ String query = "CREATE DATABASE " + dbName;
+
+ runCommand(query);
+ String dbId = assertDatabaseIsRegistered(dbName);
+
+ //HDFS Location
+ String hdfsLocation = "hdfs://localhost:8020/warehouse/tablespace/external/hive/reports.db";
+ alterDatabaseLocation(dbName, hdfsLocation);
+ assertDatabaseLocationRelationship(dbId);
+
+ //AWS location
+ String s3Location = "s3://localhost:8020/warehouse/tablespace/external/hive/reports.db";
+ alterDatabaseLocation(dbName, s3Location);
+ assertDatabaseLocationRelationship(dbId);
+
+ //ABFS location
+ String abfsLocation = "abfs://localhost:8020/warehouse/tablespace/external/hive/reports.db";
+ alterDatabaseLocation(dbName, abfsLocation);
+ assertDatabaseLocationRelationship(dbId);
+ }
+
+ //alter database location
+ public void alterDatabaseLocation(String dbName, String location) throws Exception {
+ int timeDelay = 5000;
+ String query = String.format("ALTER DATABASE %s SET LOCATION \"%s\"", dbName, location);
+ runCommandWithDelay(query, timeDelay);
+ }
+
+ public void assertDatabaseLocationRelationship(String dbId) throws Exception {
+ AtlasEntity dbEntity = atlasClientV2.getEntityByGuid(dbId).getEntity();
+ AtlasEntityDef pathEntityDef = atlasClientV2.getEntityDefByName("Path");
+
+ //Check if dbEntity has location attribute
+ assertTrue(dbEntity.hasAttribute(ATTRIBUTE_LOCATION));
+ //Check if dbEntity has value for location attribute
+ assertNotNull(dbEntity.getAttribute(ATTRIBUTE_LOCATION));
+ //Check if dbEntity has location relationship attribute
+ assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_LOCATION_PATH)).size(), 1);
+ AtlasObjectId locationEntityObject = toAtlasObjectId(dbEntity.getRelationshipAttribute(ATTRIBUTE_LOCATION_PATH));
+ //Check if location relationship attribute is subtype of "Path"
+ assertTrue(pathEntityDef.getSubTypes().contains(locationEntityObject.getTypeName()));
+ }
+
+ @Test
public void testCreateTable() throws Exception {
String tableName = tableName();
String dbName = createDatabase();
diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json
index 78ba927..769d885 100644
--- a/addons/models/0000-Area0/0010-base_model.json
+++ b/addons/models/0000-Area0/0010-base_model.json
@@ -217,6 +217,14 @@
]
},
{
+ "name": "Path",
+ "superTypes": [
+ "DataSet"
+ ],
+ "serviceType": "atlas_core",
+ "typeVersion": "1.0"
+ },
+ {
"name": "AtlasServer",
"serviceType": "atlas_core",
"typeVersion": "1.0",
diff --git a/addons/models/1000-Hadoop/1020-fs_model.json b/addons/models/1000-Hadoop/1020-fs_model.json
index fc0e80d..6d7b95a 100644
--- a/addons/models/1000-Hadoop/1020-fs_model.json
+++ b/addons/models/1000-Hadoop/1020-fs_model.json
@@ -86,10 +86,11 @@
{
"name": "fs_path",
"superTypes": [
- "DataSet"
+ "DataSet",
+ "Path"
],
"serviceType": "file_system",
- "typeVersion": "1.0",
+ "typeVersion": "1.1",
"attributeDefs": [
{
"name": "path",
diff --git a/addons/models/1000-Hadoop/1030-hive_model.json b/addons/models/1000-Hadoop/1030-hive_model.json
index b44f724..79beee8 100644
--- a/addons/models/1000-Hadoop/1030-hive_model.json
+++ b/addons/models/1000-Hadoop/1030-hive_model.json
@@ -718,6 +718,23 @@
"cardinality": "SINGLE"
},
"propagateTags": "NONE"
+ },
+ {
+ "name": "hive_db_location",
+ "serviceType": "hive",
+ "typeVersion": "1.0",
+ "relationshipCategory": "ASSOCIATION",
+ "endDef1": {
+ "type": "hive_db",
+ "name": "locationPath",
+ "cardinality": "SINGLE"
+ },
+ "endDef2": {
+ "type": "Path",
+ "name": "hiveDb",
+ "cardinality": "SINGLE"
+ },
+ "propagateTags": "NONE"
}
]
}
diff --git a/addons/models/1000-Hadoop/patches/018-fs_path_add_supertypes.json b/addons/models/1000-Hadoop/patches/018-fs_path_add_supertypes.json
new file mode 100644
index 0000000..8923504
--- /dev/null
+++ b/addons/models/1000-Hadoop/patches/018-fs_path_add_supertypes.json
@@ -0,0 +1,16 @@
+{
+ "patches": [
+ {
+ "id": "TYPEDEF_PATCH_1000_018_001",
+ "description": "Add superTypes for fs_path",
+ "action": "ADD_SUPER_TYPES",
+ "typeName": "fs_path",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "params": null,
+ "superTypes": [
+ "Path"
+ ]
+ }
+ ]
+}
diff --git a/addons/models/3000-Cloud/3020-aws_s3_typedefs.json b/addons/models/3000-Cloud/3020-aws_s3_typedefs.json
index d41df92..5275c56 100644
--- a/addons/models/3000-Cloud/3020-aws_s3_typedefs.json
+++ b/addons/models/3000-Cloud/3020-aws_s3_typedefs.json
@@ -96,9 +96,9 @@
{
"name": "aws_s3_pseudo_dir",
"description": "Atlas Type representing a Pseudo-Directory (prefix) in an S3 Object Store Bucket",
- "superTypes": [ "DataSet" ],
+ "superTypes": [ "DataSet","Path" ],
"serviceType": "aws",
- "typeVersion": "1.0",
+ "typeVersion": "1.1",
"attributeDefs": [
{
"name": "objectPrefix",
diff --git a/addons/models/3000-Cloud/3030-aws_s3_v2_typedefs.json b/addons/models/3000-Cloud/3030-aws_s3_v2_typedefs.json
index 7236bc9..e656c31 100644
--- a/addons/models/3000-Cloud/3030-aws_s3_v2_typedefs.json
+++ b/addons/models/3000-Cloud/3030-aws_s3_v2_typedefs.json
@@ -146,10 +146,11 @@
"description": "Atlas Type representing a directory in an S3 Object Store",
"superTypes": [
"aws_s3_v2_object",
- "aws_s3_v2_container"
+ "aws_s3_v2_container",
+ "Path"
],
"serviceType": "aws",
- "typeVersion": "1.0",
+ "typeVersion": "1.1",
"attributeDefs": [
{
"name": "objectPrefix",
diff --git a/addons/models/3000-Cloud/3040-azure_adls_typedefs.json b/addons/models/3000-Cloud/3040-azure_adls_typedefs.json
index b297234..77c7494 100644
--- a/addons/models/3000-Cloud/3040-azure_adls_typedefs.json
+++ b/addons/models/3000-Cloud/3040-azure_adls_typedefs.json
@@ -317,10 +317,11 @@
"superTypes": [
"DataSet",
"adls_gen2_parent",
- "adls_gen2_child"
+ "adls_gen2_child",
+ "Path"
],
"serviceType": "azure",
- "typeVersion": "1.2",
+ "typeVersion": "1.3",
"attributeDefs": [
{
"name": "eTag",
diff --git a/addons/models/3000-Cloud/patches/006-cloud-model_add_supertypes.json b/addons/models/3000-Cloud/patches/006-cloud-model_add_supertypes.json
new file mode 100644
index 0000000..6e2ea57
--- /dev/null
+++ b/addons/models/3000-Cloud/patches/006-cloud-model_add_supertypes.json
@@ -0,0 +1,40 @@
+{
+ "patches": [
+ {
+ "id": "TYPEDEF_PATCH_3000_006_001",
+ "description": "Add superTypes for aws_s3_pseudo_dir",
+ "action": "ADD_SUPER_TYPES",
+ "typeName": "aws_s3_pseudo_dir",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "params": null,
+ "superTypes": [
+ "Path"
+ ]
+ },
+ {
+ "id": "TYPEDEF_PATCH_3000_006_002",
+ "description": "Add superTypes for aws_s3_v2_directory",
+ "action": "ADD_SUPER_TYPES",
+ "typeName": "aws_s3_v2_directory",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "params": null,
+ "superTypes": [
+ "Path"
+ ]
+ },
+ {
+ "id": "TYPEDEF_PATCH_3000_006_003",
+ "description": "Add superTypes for adls_gen2_directory",
+ "action": "ADD_SUPER_TYPES",
+ "typeName": "adls_gen2_directory",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "params": null,
+ "superTypes": [
+ "Path"
+ ]
+ }
+ ]
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
index 478376b..fae28c4 100644
--- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
@@ -18,7 +18,7 @@
package org.apache.atlas.repository.patches;
-import org.apache.atlas.model.patches.AtlasPatch;
+import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches;
import org.apache.atlas.model.patches.AtlasPatch.PatchStatus;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
@@ -28,7 +28,10 @@
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.SKIPPED;
@@ -37,26 +40,35 @@
public class AtlasPatchManager {
private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchManager.class);
- private final PatchContext context;
+ private final PatchContext context;
+ private final List<AtlasPatchHandler> handlers = new ArrayList<>();
@Inject
public AtlasPatchManager(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, GraphBackedSearchIndexer indexer, EntityGraphMapper entityGraphMapper) {
this.context = new PatchContext(atlasGraph, typeRegistry, indexer, entityGraphMapper);
}
- public AtlasPatch.AtlasPatches getAllPatches() {
+ @PostConstruct
+ public void init() {
+ LOG.info("==> AtlasPatchManager.init()");
+
+ // register all java patches here
+ handlers.add(new UniqueAttributePatch(context));
+ handlers.add(new ClassificationTextPatch(context));
+ handlers.add(new FreeTextRequestHandlerPatch(context));
+ handlers.add(new SuggestionsRequestHandlerPatch(context));
+ handlers.add(new IndexConsistencyPatch(context));
+ handlers.add(new ReIndexPatch(context));
+
+ LOG.info("<== AtlasPatchManager.init()");
+ }
+
+ public AtlasPatches getAllPatches() {
return context.getPatchRegistry().getAllPatches();
}
public void applyAll() {
- final AtlasPatchHandler handlers[] = {
- new UniqueAttributePatch(context),
- new ClassificationTextPatch(context),
- new FreeTextRequestHandlerPatch(context),
- new SuggestionsRequestHandlerPatch(context),
- new IndexConsistencyPatch(context),
- new ReIndexPatch(context)
- };
+ LOG.info("==> AtlasPatchManager.applyAll()");
try {
for (AtlasPatchHandler handler : handlers) {
@@ -70,9 +82,18 @@
handler.apply();
}
}
- }
- catch (Exception ex) {
+ } catch (Exception ex) {
LOG.error("Error applying patches.", ex);
}
+
+ LOG.info("<== AtlasPatchManager.applyAll()");
+ }
+
+ public void addPatchHandler(AtlasPatchHandler patchHandler) {
+ handlers.add(patchHandler);
+ }
+
+ public PatchContext getContext() {
+ return this.context;
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/SuperTypesUpdatePatch.java b/repository/src/main/java/org/apache/atlas/repository/patches/SuperTypesUpdatePatch.java
new file mode 100644
index 0000000..a77ca70
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/SuperTypesUpdatePatch.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.patches;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.pc.WorkItemManager;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED;
+import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
+
+public class SuperTypesUpdatePatch extends AtlasPatchHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(SuperTypesUpdatePatch.class);
+ private static final String PATCH_ID = "JAVA_PATCH_0000_007";
+ private static final String PATCH_DESCRIPTION = "Update supertypes for all existing entities for given typeName";
+
+ private final PatchContext context;
+ private final String typeName;
+
+ public SuperTypesUpdatePatch(PatchContext context, String typeDefPatchId, String typeName) {
+ super(context.getPatchRegistry(), PATCH_ID + "_" + typeDefPatchId, PATCH_DESCRIPTION);
+
+ this.context = context;
+ this.typeName = typeName;
+ }
+
+ @Override
+ public void apply() throws AtlasBaseException {
+ LOG.info("==> SuperTypesUpdatePatch.apply(): patchId={}", getPatchId());
+
+ ConcurrentPatchProcessor patchProcessor = new SuperTypesUpdatePatchProcessor(context, typeName);
+
+ patchProcessor.apply();
+
+ setStatus(APPLIED);
+
+ LOG.info("<== SuperTypesUpdatePatch.apply(): patchId={}, status={}", getPatchId(), getStatus());
+ }
+
+ public static class SuperTypesUpdatePatchProcessor extends ConcurrentPatchProcessor {
+ private final String typeName;
+ private final Set<String> typeAndAllSubTypes;
+
+ public SuperTypesUpdatePatchProcessor(PatchContext context, String typeName) {
+ super(context);
+
+ AtlasEntityType entityType = getTypeRegistry().getEntityTypeByName(typeName);
+
+ this.typeName = typeName;
+ this.typeAndAllSubTypes = entityType != null ? entityType.getTypeAndAllSubTypes() : Collections.emptySet();
+ }
+
+ @Override
+ public void submitVerticesToUpdate(WorkItemManager manager) {
+ if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
+ LOG.info("Entity types to be updated with supertypes :{}", typeAndAllSubTypes.size());
+
+ for (String typeName : typeAndAllSubTypes) {
+ LOG.info("finding entities of type {}", typeName);
+
+ AtlasGraph graph = getGraph();
+ Iterable<Object> vertexIds = graph.query().has(ENTITY_TYPE_PROPERTY_KEY, typeName).vertexIds();
+ int count = 0;
+
+ for (Iterator<Object> iterator = vertexIds.iterator(); iterator.hasNext(); ) {
+ Object vertexId = iterator.next();
+
+ manager.checkProduce(vertexId);
+
+ count++;
+ }
+
+ LOG.info("found {} entities of type {}", count, typeName);
+ }
+ }
+ }
+
+ @Override
+ protected void processVertexItem(Long vertexId, AtlasVertex vertex, String typeName, AtlasEntityType entityType) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("processVertexItem(typeName={}, vertexId={})", typeName, vertexId);
+ }
+
+ Set<String> allSuperTypes = entityType.getAllSuperTypes();
+
+ if (allSuperTypes != null) {
+ // remove and update all entity super types
+ vertex.removeProperty(SUPER_TYPES_PROPERTY_KEY);
+
+ for (String superType : allSuperTypes) {
+ AtlasGraphUtilsV2.addEncodedProperty(vertex, SUPER_TYPES_PROPERTY_KEY, superType);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updated superTypes for entity of typeName={}, vertexId={}): Done!", typeName, vertex.getId());
+ }
+ }
+ }
+
+ @Override
+ protected void prepareForExecution() {
+ //do nothing
+ }
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index c29888f..676a0aa 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -43,6 +43,8 @@
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.patches.SuperTypesUpdatePatch;
+import org.apache.atlas.repository.patches.AtlasPatchManager;
import org.apache.atlas.repository.patches.AtlasPatchRegistry;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
@@ -94,14 +96,16 @@
private final AtlasTypeRegistry typeRegistry;
private final Configuration conf;
private final AtlasGraph graph;
+ private final AtlasPatchManager patchManager;
@Inject
public AtlasTypeDefStoreInitializer(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
- AtlasGraph graph, Configuration conf) {
+ AtlasGraph graph, Configuration conf, AtlasPatchManager patchManager) {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.conf = conf;
this.graph = graph;
+ this.patchManager = patchManager;
}
@PostConstruct
@@ -449,7 +453,8 @@
new RemoveLegacyRefAttributesPatchHandler(typeDefStore, typeRegistry),
new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry),
new SetServiceTypePatchHandler(typeDefStore, typeRegistry),
- new UpdateAttributeMetadataHandler(typeDefStore, typeRegistry)
+ new UpdateAttributeMetadataHandler(typeDefStore, typeRegistry),
+ new AddSuperTypePatchHandler(typeDefStore, typeRegistry)
};
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
@@ -532,6 +537,7 @@
private Map<String, String> typeDefOptions;
private String serviceType;
private String attributeName;
+ private Set<String> superTypes;
public String getId() {
return id;
@@ -624,6 +630,14 @@
public String getAttributeName() { return attributeName; }
public void setAttributeName(String attributeName) { this.attributeName = attributeName; }
+
+ public Set<String> getSuperTypes() {
+ return superTypes;
+ }
+
+ public void setSuperTypes(Set<String> superTypes) {
+ this.superTypes = superTypes;
+ }
}
/**
@@ -1136,4 +1150,58 @@
}
}
}
+
+ class AddSuperTypePatchHandler extends PatchHandler {
+
+ public AddSuperTypePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+ super(typeDefStore, typeRegistry, new String[] {"ADD_SUPER_TYPES"});
+ }
+
+ @Override
+ public PatchStatus applyPatch(TypeDefPatch patch) throws AtlasBaseException {
+ String typeName = patch.getTypeName();
+ AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
+ PatchStatus ret;
+
+ if (typeDef == null) {
+ throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
+ }
+
+ Set<String> superTypesToBeAdded = patch.getSuperTypes();
+
+ if (CollectionUtils.isNotEmpty(superTypesToBeAdded) && isPatchApplicable(patch, typeDef)) {
+ if (typeDef.getClass().equals(AtlasEntityDef.class)) {
+ AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
+
+ for (String superType : superTypesToBeAdded) {
+ updatedDef.addSuperType(superType);
+ }
+
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateEntityDefByName(typeName, updatedDef);
+
+ LOG.info("Update entities of {} with new supertypes", typeName);
+
+ // add to java patch handlers to update entity supertypes
+ patchManager.addPatchHandler(new SuperTypesUpdatePatch(patchManager.getContext(), patch.getId(), typeName));
+
+ ret = APPLIED;
+ } else {
+ throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, patch.getAction(), typeDef.getClass().getSimpleName());
+ }
+ } else {
+ if (CollectionUtils.isEmpty(superTypesToBeAdded)) {
+ LOG.info("patch skipped: No superTypes provided to add for typeName={}", patch.getTypeName());
+ } else {
+ LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
+ patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
+ }
+
+ ret = SKIPPED;
+ }
+
+ return ret;
+ }
+ }
}
diff --git a/test-tools/src/main/resources/solr/core-template/solrconfig.xml b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
index 94f346f..4d7e444 100644
--- a/test-tools/src/main/resources/solr/core-template/solrconfig.xml
+++ b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
@@ -432,9 +432,30 @@
<!-- default values for query parameters can be specified, these
will be overridden by parameters in the request
-->
+ <!--
+ 35x_t __guid
+ f0l_t __typeName
+ i6d_l __timestamp
+ jr9_t __state
+ lc5_t __classificationsText
+ mx1_t __classificationNames
+ iyt_l __modificationTimestamp
+ zk5_t __customAttributes
+ 1151_t __labels
+ ohx_t __propagatedClassificationNames
+ 3thh_s Asset.__s_name
+ 3z0l_s Asset.__s_owner
+ 3wn9_t Asset.description
+ 3v2d_s Asset.__s_displayName
+ 3xfp_s Asset.__s_userDescription
+ 3mdh_t Referenceable.qualifiedName
+ c45h_t hive_serde.name
+ cb9h_t hive_process.queryText
+ c83p_t hive_process.userName
+ -->
<lst name="defaults">
<str name="defType">edismax</str>
- <str name="qf">3ll1_t 35x_t f0l_t i6d_l 7f2d_t 7gn9_t 3sp1_s jr9_t 3vut_t lc5_t mx1_t 7dhh_t iyt_l 3j7p_t 7klh_t 7hfp_t 7i85_t ohx_t 7bwl_l 7cp1_l</str>
+ <str name="qf">35x_t f0l_t i6d_l jr9_t lc5_t mx1_t iyt_l zk5_t 1151_t ohx_t 3thh_s 3z0l_s 3wn9_t 3v2d_s 3xfp_s 3mdh_t c45h_t cb9h_t c83p_t</str>
<str name="hl.fl">*</str>
<bool name="hl.requireFieldMatch">true</bool>
<bool name="lowercaseOperators">true</bool>
