| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.atlas.repository.graph; |
| |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableSet; |
| import org.apache.atlas.AtlasException; |
| import org.apache.atlas.CreateUpdateEntitiesResult; |
| import org.apache.atlas.GraphTransaction; |
| import org.apache.atlas.RequestContext; |
| import org.apache.atlas.TestOnlyModule; |
| import org.apache.atlas.TestUtils; |
| import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; |
| import org.apache.atlas.query.QueryParams; |
| import org.apache.atlas.repository.Constants; |
| import org.apache.atlas.repository.MetadataRepository; |
| import org.apache.atlas.repository.RepositoryException; |
| import org.apache.atlas.repository.graphdb.AtlasEdge; |
| import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; |
| import org.apache.atlas.repository.graphdb.AtlasGraph; |
| import org.apache.atlas.repository.graphdb.AtlasGraphQuery; |
| import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; |
| import org.apache.atlas.repository.graphdb.AtlasVertex; |
| import org.apache.atlas.type.AtlasTypeRegistry; |
| import org.apache.atlas.typesystem.IStruct; |
| import org.apache.atlas.typesystem.ITypedReferenceableInstance; |
| import org.apache.atlas.typesystem.ITypedStruct; |
| import org.apache.atlas.typesystem.Referenceable; |
| import org.apache.atlas.typesystem.Struct; |
| import org.apache.atlas.typesystem.exception.EntityNotFoundException; |
| import org.apache.atlas.typesystem.exception.TraitNotFoundException; |
| import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes; |
| import org.apache.atlas.typesystem.persistence.Id; |
| import org.apache.atlas.typesystem.types.AttributeDefinition; |
| import org.apache.atlas.typesystem.types.ClassType; |
| import org.apache.atlas.typesystem.types.DataTypes; |
| import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; |
| import org.apache.atlas.typesystem.types.Multiplicity; |
| import org.apache.atlas.typesystem.types.TraitType; |
| import org.apache.atlas.typesystem.types.TypeSystem; |
| import org.apache.atlas.typesystem.types.utils.TypesUtil; |
| import org.apache.commons.lang.RandomStringUtils; |
| import org.codehaus.jettison.json.JSONArray; |
| import org.codehaus.jettison.json.JSONObject; |
| import org.testng.Assert; |
| import org.testng.annotations.AfterClass; |
| import org.testng.annotations.BeforeClass; |
| import org.testng.annotations.BeforeMethod; |
| import org.testng.annotations.Guice; |
| import org.testng.annotations.Test; |
| |
| import javax.inject.Inject; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Date; |
| import java.util.HashMap; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.UUID; |
| import java.util.concurrent.Callable; |
| import java.util.concurrent.ExecutorService; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.Future; |
| |
| import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; |
| import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef; |
| import static org.testng.Assert.assertEquals; |
| import static org.testng.Assert.assertNotEquals; |
| import static org.testng.Assert.assertNotNull; |
| import static org.testng.Assert.assertTrue; |
| |
| /** |
| * GraphBackedMetadataRepository test |
| * |
| * Guice loads the dependencies and injects the necessary objects |
| * |
| */ |
| @Guice(modules = TestOnlyModule.class) |
| public class GraphBackedMetadataRepositoryTest { |
| |
| @Inject |
| private MetadataRepository repositoryService; |
| |
| @Inject |
| private GraphBackedDiscoveryService discoveryService; |
| |
| private TypeSystem typeSystem; |
| private String guid; |
| private QueryParams queryParams = new QueryParams(100, 0); |
| |
| @BeforeClass |
| public void setUp() throws Exception { |
| typeSystem = TypeSystem.getInstance(); |
| typeSystem.reset(); |
| |
| assertTrue(repositoryService instanceof GraphBackedMetadataRepository); |
| repositoryService = TestUtils.addTransactionWrapper(repositoryService); |
| new GraphBackedSearchIndexer(new AtlasTypeRegistry()); |
| |
| TestUtils.defineDeptEmployeeTypes(typeSystem); |
| TestUtils.createHiveTypes(typeSystem); |
| } |
| |
| @BeforeMethod |
| public void setupContext() { |
| TestUtils.resetRequestContext(); |
| } |
| |
| @AfterClass |
| public void tearDown() throws Exception { |
| TypeSystem.getInstance().reset(); |
| AtlasGraphProvider.cleanup(); |
| } |
| |
| @Test |
| //In some cases of parallel APIs, the edge is added, but get edge by label doesn't return the edge. ATLAS-1104 |
| public void testConcurrentCalls() throws Exception { |
| final HierarchicalTypeDefinition<ClassType> refType = |
| createClassTypeDef(randomString(), ImmutableSet.<String>of()); |
| HierarchicalTypeDefinition<ClassType> type = |
| createClassTypeDef(randomString(), ImmutableSet.<String>of(), |
| new AttributeDefinition("ref", refType.typeName, Multiplicity.OPTIONAL, true, null)); |
| typeSystem.defineClassType(refType); |
| typeSystem.defineClassType(type); |
| |
| String refId1 = createEntity(new Referenceable(refType.typeName)).get(0); |
| String refId2 = createEntity(new Referenceable(refType.typeName)).get(0); |
| |
| final Referenceable instance1 = new Referenceable(type.typeName); |
| instance1.set("ref", new Referenceable(refId1, refType.typeName, null)); |
| |
| final Referenceable instance2 = new Referenceable(type.typeName); |
| instance2.set("ref", new Referenceable(refId2, refType.typeName, null)); |
| |
| ExecutorService executor = Executors.newFixedThreadPool(3); |
| List<Future<Object>> futures = new ArrayList<>(); |
| futures.add(executor.submit(new Callable<Object>() { |
| @Override |
| public Object call() throws Exception { |
| return createEntity(instance1).get(0); |
| } |
| })); |
| futures.add(executor.submit(new Callable<Object>() { |
| @Override |
| public Object call() throws Exception { |
| return createEntity(instance2).get(0); |
| } |
| })); |
| futures.add(executor.submit(new Callable<Object>() { |
| @Override |
| public Object call() throws Exception { |
| return discoveryService.searchByDSL(TestUtils.TABLE_TYPE, new QueryParams(10, 0)); |
| } |
| })); |
| |
| String id1 = (String) futures.get(0).get(); |
| String id2 = (String) futures.get(1).get(); |
| futures.get(2).get(); |
| executor.shutdown(); |
| |
| boolean validated1 = assertEdge(id1, type.typeName); |
| boolean validated2 = assertEdge(id2, type.typeName); |
| assertTrue(validated1 | validated2); |
| } |
| |
| private boolean assertEdge(String id, String typeName) throws Exception { |
| AtlasGraph graph = TestUtils.getGraph(); |
| Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices(); |
| AtlasVertex AtlasVertex = vertices.iterator().next(); |
| Iterable<AtlasEdge> edges = AtlasVertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref"); |
| if (!edges.iterator().hasNext()) { |
| ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(id); |
| assertNotNull(entity.get("ref")); |
| return true; |
| } |
| return false; |
| } |
| |
| @Test |
| public void testSubmitEntity() throws Exception { |
| ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem); |
| |
| List<String> guids = repositoryService.createEntities(hrDept).getCreatedEntities(); |
| Assert.assertNotNull(guids); |
| Assert.assertEquals(guids.size(), 5); |
| guid = guids.get(4); |
| Assert.assertNotNull(guid); |
| } |
| |
| @Test |
| public void testCreateEntityWithOneNestingLevel() throws AtlasException { |
| |
| List<Referenceable> toValidate = new ArrayList<>(); |
| Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE); |
| toValidate.add(dept); |
| dept.set(TestUtils.NAME, "test1"); |
| |
| Referenceable mike = new Referenceable(TestUtils.PERSON_TYPE); |
| toValidate.add(mike); |
| |
| mike.set(TestUtils.NAME, "Mike"); |
| mike.set(TestUtils.DEPARTMENT_ATTR, dept); |
| |
| Referenceable mark = new Referenceable(TestUtils.PERSON_TYPE); |
| toValidate.add(mark); |
| mark.set(TestUtils.NAME, "Mark"); |
| mark.set(TestUtils.DEPARTMENT_ATTR, dept); |
| |
| dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(mike, mark)); |
| Map<String,Referenceable> positions = new HashMap<>(); |
| final String JANITOR = "janitor"; |
| final String RECEPTIONIST = "receptionist"; |
| positions.put(JANITOR, mike); |
| positions.put(RECEPTIONIST, mark); |
| dept.set(TestUtils.POSITIONS_ATTR, positions); |
| |
| |
| ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE); |
| ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED); |
| |
| CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance); |
| |
| validateGuidMapping(toValidate, result); |
| } |
| |
| |
| @Test |
| public void testCreateEntityWithTwoNestingLevels() throws AtlasException { |
| |
| List<Referenceable> toVerify = new ArrayList<>(); |
| Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE); |
| toVerify.add(dept); |
| dept.set(TestUtils.NAME, "test2"); |
| |
| Referenceable wallace = new Referenceable(TestUtils.PERSON_TYPE); |
| toVerify.add(wallace); |
| wallace.set(TestUtils.NAME, "Wallace"); |
| wallace.set(TestUtils.DEPARTMENT_ATTR, dept); |
| |
| Referenceable wallaceComputer = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(wallaceComputer); |
| wallaceComputer.set("name", "wallaceComputer"); |
| wallace.set(TestUtils.ASSETS_ATTR, ImmutableList.of(wallaceComputer)); |
| |
| Referenceable jordan = new Referenceable(TestUtils.PERSON_TYPE); |
| toVerify.add(jordan); |
| jordan.set(TestUtils.NAME, "Jordan"); |
| jordan.set(TestUtils.DEPARTMENT_ATTR, dept); |
| |
| Referenceable jordanComputer = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(jordanComputer); |
| jordanComputer.set("name", "jordanComputer"); |
| jordan.set(TestUtils.ASSETS_ATTR, ImmutableList.of(jordanComputer)); |
| |
| dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(wallace, jordan)); |
| Map<String,Referenceable> positions = new HashMap<>(); |
| final String JANITOR = "janitor"; |
| final String RECEPTIONIST = "receptionist"; |
| positions.put(JANITOR, wallace); |
| positions.put(RECEPTIONIST, jordan); |
| dept.set(TestUtils.POSITIONS_ATTR, positions); |
| |
| |
| ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE); |
| ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED); |
| |
| CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance); |
| validateGuidMapping(toVerify, result); |
| } |
| |
| |
| @Test |
| public void testCreateEntityWithThreeNestingLevels() throws AtlasException { |
| |
| List<Referenceable> toVerify = new ArrayList<>(); |
| |
| Referenceable dept = new Referenceable(TestUtils.DEPARTMENT_TYPE); |
| toVerify.add(dept); |
| dept.set(TestUtils.NAME, "test3"); |
| |
| Referenceable barry = new Referenceable(TestUtils.PERSON_TYPE); |
| toVerify.add(barry); |
| barry.set(TestUtils.NAME, "barry"); |
| barry.set(TestUtils.DEPARTMENT_ATTR, dept); |
| |
| Referenceable barryComputer = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(barryComputer); |
| barryComputer.set("name", "barryComputer"); |
| barry.set(TestUtils.ASSETS_ATTR, ImmutableList.of(barryComputer)); |
| |
| Referenceable barryHardDrive = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(barryHardDrive); |
| barryHardDrive.set("name", "barryHardDrive"); |
| |
| Referenceable barryCpuFan = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(barryCpuFan); |
| barryCpuFan.set("name", "barryCpuFan"); |
| |
| Referenceable barryVideoCard = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(barryVideoCard); |
| barryVideoCard.set("name", "barryVideoCard"); |
| |
| barryComputer.set("childAssets", ImmutableList.of(barryHardDrive, barryVideoCard, barryCpuFan)); |
| |
| |
| Referenceable jacob = new Referenceable(TestUtils.PERSON_TYPE); |
| toVerify.add(jacob); |
| jacob.set(TestUtils.NAME, "jacob"); |
| jacob.set(TestUtils.DEPARTMENT_ATTR, dept); |
| |
| Referenceable jacobComputer = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(jacobComputer); |
| jacobComputer.set("name", "jacobComputer"); |
| jacob.set(TestUtils.ASSETS_ATTR, ImmutableList.of(jacobComputer)); |
| |
| Referenceable jacobHardDrive = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(jacobHardDrive); |
| jacobHardDrive.set("name", "jacobHardDrive"); |
| |
| Referenceable jacobCpuFan = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(jacobCpuFan); |
| jacobCpuFan.set("name", "jacobCpuFan"); |
| |
| Referenceable jacobVideoCard = new Referenceable(TestUtils.ASSET_TYPE); |
| toVerify.add(jacobVideoCard); |
| jacobVideoCard.set("name", "jacobVideoCard"); |
| |
| jacobComputer.set("childAssets", ImmutableList.of(jacobHardDrive, jacobVideoCard, jacobCpuFan)); |
| |
| dept.set(TestUtils.EMPLOYEES_ATTR, ImmutableList.of(barry, jacob)); |
| Map<String,Referenceable> positions = new HashMap<>(); |
| final String JANITOR = "janitor"; |
| final String RECEPTIONIST = "receptionist"; |
| positions.put(JANITOR, barry); |
| positions.put(RECEPTIONIST, jacob); |
| dept.set(TestUtils.POSITIONS_ATTR, positions); |
| |
| |
| ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, TestUtils.DEPARTMENT_TYPE); |
| ITypedReferenceableInstance deptInstance = deptType.convert(dept, Multiplicity.REQUIRED); |
| |
| CreateUpdateEntitiesResult result = repositoryService.createEntities(deptInstance); |
| |
| assertEquals(result.getCreatedEntities().size(), toVerify.size()); |
| |
| validateGuidMapping(toVerify, result); |
| } |
| |
| private void validateGuidMapping(List<Referenceable> toVerify, CreateUpdateEntitiesResult result) |
| throws AtlasException { |
| Map<String,String> guids = result.getGuidMapping().getGuidAssignments(); |
| |
| TestUtils.assertContentsSame(result.getCreatedEntities(), guids.values()); |
| assertEquals(guids.size(), toVerify.size()); |
| for(Referenceable r : toVerify) { |
| loadAndDoSimpleValidation(guids.get(r.getId()._getId()), r); |
| } |
| } |
| |
| private ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable inst) throws AtlasException { |
| return TestUtils.loadAndDoSimpleValidation(guid, inst, repositoryService); |
| } |
| |
| |
| @Test(dependsOnMethods = "testSubmitEntity") |
| public void testGetEntityDefinitionForDepartment() throws Exception { |
| ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid); |
| Assert.assertNotNull(entity); |
| |
| //entity state should be active by default |
| Assert.assertEquals(entity.getId().getState(), Id.EntityState.ACTIVE); |
| |
| //System attributes created time and modified time should not be null |
| AtlasSystemAttributes systemAttributes = entity.getSystemAttributes(); |
| Assert.assertNotNull(systemAttributes.createdTime); |
| Assert.assertNotNull(systemAttributes.modifiedTime); |
| } |
| |
| @Test(expectedExceptions = EntityNotFoundException.class) |
| public void testGetEntityDefinitionNonExistent() throws Exception { |
| repositoryService.getEntityDefinition("blah"); |
| Assert.fail(); |
| } |
| |
| @Test(dependsOnMethods = "testSubmitEntity") |
| public void testGetEntityList() throws Exception { |
| List<String> entityList = repositoryService.getEntityList(TestUtils.DEPARTMENT_TYPE); |
| System.out.println("entityList = " + entityList); |
| Assert.assertNotNull(entityList); |
| Assert.assertTrue(entityList.contains(guid)); |
| } |
| |
| @Test |
| public void testGetTypeAttributeName() throws Exception { |
| Assert.assertEquals(repositoryService.getTypeAttributeName(), Constants.ENTITY_TYPE_PROPERTY_KEY); |
| } |
| |
| @Test(dependsOnMethods = "testSubmitEntity") |
| public void testGetTraitLabel() throws Exception { |
| Assert.assertEquals( |
| repositoryService.getTraitLabel(typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE), |
| TestUtils.CLASSIFICATION), TestUtils.CLASSIFICATION); |
| } |
| |
| @Test |
| public void testCreateEntity() throws Exception { |
| Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE); |
| databaseInstance.set("name", TestUtils.DATABASE_NAME); |
| databaseInstance.set("description", "foo database"); |
| databaseInstance.set("created", new Date(TestUtils.TEST_DATE_IN_LONG)); |
| |
| databaseInstance.set("namespace", "colo:cluster:hive:db"); |
| databaseInstance.set("cluster", "cluster-1"); |
| databaseInstance.set("colo", "colo-1"); |
| System.out.println("databaseInstance = " + databaseInstance); |
| |
| ClassType dbType = typeSystem.getDataType(ClassType.class, TestUtils.DATABASE_TYPE); |
| ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED); |
| System.out.println("db = " + db); |
| |
| //Reuse the same database instance without id, with the same unique attribute |
| ITypedReferenceableInstance table = createHiveTableInstance(databaseInstance); |
| List<String> guids = createEntities(db, table); |
| Assert.assertEquals(guids.size(), 7); //1 db + 5 columns + 1 table. Shouldn't create db again |
| System.out.println("added db = " + guids.get(0)); |
| System.out.println("added table = " + guids.get(6)); |
| } |
| |
| @Test(dependsOnMethods = "testCreateEntity") |
| public void testGetEntityDefinition() throws Exception { |
| String guid = getGUID(); |
| |
| ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid); |
| Assert.assertEquals(table.getDate("created"), new Date(TestUtils.TEST_DATE_IN_LONG)); |
| System.out.println("*** table = " + table); |
| } |
| |
| private List<String> createEntities(ITypedReferenceableInstance... instances) throws Exception { |
| RequestContext.createContext(); |
| return repositoryService.createEntities(instances).getCreatedEntities(); |
| } |
| |
| private List<String> createEntity(Referenceable entity) throws Exception { |
| ClassType type = typeSystem.getDataType(ClassType.class, entity.getTypeName()); |
| ITypedReferenceableInstance instance = type.convert(entity, Multiplicity.REQUIRED); |
| return createEntities(instance); |
| } |
| |
| @GraphTransaction |
| String getGUID() { |
| AtlasVertex tableVertex = getTableEntityVertex(); |
| |
| String guid = GraphHelper.getSingleValuedProperty(tableVertex, Constants.GUID_PROPERTY_KEY, String.class); |
| if (guid == null) { |
| Assert.fail(); |
| } |
| return guid; |
| } |
| |
| @GraphTransaction |
| AtlasVertex getTableEntityVertex() { |
| AtlasGraph graph = TestUtils.getGraph(); |
| AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, ComparisionOperator.EQUAL, TestUtils.TABLE_TYPE); |
| Iterator<AtlasVertex> results = query.vertices().iterator(); |
| // returning one since guid should be unique |
| AtlasVertex tableVertex = results.hasNext() ? results.next() : null; |
| if (tableVertex == null) { |
| Assert.fail(); |
| } |
| |
| return tableVertex; |
| } |
| |
| @Test(dependsOnMethods = "testCreateEntity") |
| public void testGetTraitNames() throws Exception { |
| final List<String> traitNames = repositoryService.getTraitNames(getGUID()); |
| Assert.assertEquals(traitNames.size(), 1); |
| Assert.assertEquals(traitNames, Arrays.asList(new String[]{TestUtils.CLASSIFICATION})); |
| } |
| |
| @Test |
| public void testGetTraitNamesForEmptyTraits() throws Exception { |
| final List<String> traitNames = repositoryService.getTraitNames(guid); |
| Assert.assertEquals(traitNames.size(), 0); |
| } |
| |
| @Test(expectedExceptions = EntityNotFoundException.class) |
| public void testGetTraitNamesForBadEntity() throws Exception { |
| repositoryService.getTraitNames(UUID.randomUUID().toString()); |
| Assert.fail(); |
| } |
| |
| @Test |
| public void testMultipleTypesWithSameUniqueAttribute() throws Exception { |
| //Two entities of different types(with same supertype that has the unique attribute) with same qualified name should succeed |
| HierarchicalTypeDefinition<ClassType> supertype = |
| createClassTypeDef(randomString(), ImmutableSet.<String>of(), |
| createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE)); |
| HierarchicalTypeDefinition<ClassType> t1 = |
| createClassTypeDef(randomString(), ImmutableSet.of(supertype.typeName)); |
| HierarchicalTypeDefinition<ClassType> t2 = |
| createClassTypeDef(randomString(), ImmutableSet.of(supertype.typeName)); |
| typeSystem.defineClassTypes(supertype, t1, t2); |
| |
| final String name = randomString(); |
| String id1 = createEntity(new Referenceable(t1.typeName) {{ |
| set("name", name); |
| }}).get(0); |
| String id2 = createEntity(new Referenceable(t2.typeName) {{ |
| set("name", name); |
| }}).get(0); |
| assertNotEquals(id1, id2); |
| |
| ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(t1.typeName, "name", name); |
| assertEquals(entity.getTypeName(), t1.typeName); |
| assertEquals(entity.getId()._getId(), id1); |
| |
| entity = repositoryService.getEntityDefinition(t2.typeName, "name", name); |
| assertEquals(entity.getTypeName(), t2.typeName); |
| assertEquals(entity.getId()._getId(), id2); |
| } |
| |
| @Test(dependsOnMethods = "testGetTraitNames") |
| public void testAddTrait() throws Exception { |
| final String aGUID = getGUID(); |
| AtlasVertex AtlasVertex = GraphHelper.getInstance().getVertexForGUID(aGUID); |
| Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); |
| Assert.assertNotNull(modificationTimestampPreUpdate); |
| |
| List<String> traitNames = repositoryService.getTraitNames(aGUID); |
| System.out.println("traitNames = " + traitNames); |
| Assert.assertEquals(traitNames.size(), 1); |
| Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); |
| Assert.assertFalse(traitNames.contains(TestUtils.PII)); |
| |
| TraitType traitType = typeSystem.getDataType(TraitType.class, TestUtils.PII); |
| ITypedStruct traitInstance = traitType.createInstance(); |
| |
| repositoryService.addTrait(aGUID, traitInstance); |
| |
| // refresh trait names |
| traitNames = repositoryService.getTraitNames(aGUID); |
| Assert.assertEquals(traitNames.size(), 2); |
| Assert.assertTrue(traitNames.contains(TestUtils.PII)); |
| Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); |
| |
| // Verify modification timestamp was updated. |
| GraphHelper.getInstance().getVertexForGUID(aGUID); |
| Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); |
| Assert.assertNotNull(modificationTimestampPostUpdate); |
| } |
| |
| @Test(dependsOnMethods = "testAddTrait") |
| public void testAddTraitWithAttribute() throws Exception { |
| final String aGUID = getGUID(); |
| final String traitName = "P_I_I"; |
| |
| HierarchicalTypeDefinition<TraitType> piiTrait = TypesUtil |
| .createTraitTypeDef(traitName, ImmutableSet.<String>of(), |
| TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE)); |
| TraitType traitType = typeSystem.defineTraitType(piiTrait); |
| ITypedStruct traitInstance = traitType.createInstance(); |
| traitInstance.set("type", "SSN"); |
| |
| repositoryService.addTrait(aGUID, traitInstance); |
| |
| TestUtils.dumpGraph(TestUtils.getGraph()); |
| |
| // refresh trait names |
| List<String> traitNames = repositoryService.getTraitNames(aGUID); |
| Assert.assertEquals(traitNames.size(), 3); |
| Assert.assertTrue(traitNames.contains(traitName)); |
| |
| ITypedReferenceableInstance instance = repositoryService.getEntityDefinition(aGUID); |
| IStruct traitInstanceRef = instance.getTrait(traitName); |
| String type = (String) traitInstanceRef.get("type"); |
| Assert.assertEquals(type, "SSN"); |
| } |
| |
| @Test(expectedExceptions = NullPointerException.class) |
| public void testAddTraitWithNullInstance() throws Exception { |
| repositoryService.addTrait(getGUID(), null); |
| Assert.fail(); |
| } |
| |
| @Test(dependsOnMethods = "testAddTrait", expectedExceptions = RepositoryException.class) |
| public void testAddTraitForBadEntity() throws Exception { |
| TraitType traitType = typeSystem.getDataType(TraitType.class, TestUtils.PII); |
| ITypedStruct traitInstance = traitType.createInstance(); |
| |
| repositoryService.addTrait(UUID.randomUUID().toString(), traitInstance); |
| Assert.fail(); |
| } |
| |
| @Test(dependsOnMethods = "testAddTrait") |
| public void testDeleteTrait() throws Exception { |
| final String aGUID = getGUID(); |
| AtlasVertex AtlasVertex = GraphHelper.getInstance().getVertexForGUID(aGUID); |
| Long modificationTimestampPreUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); |
| Assert.assertNotNull(modificationTimestampPreUpdate); |
| |
| List<String> traitNames = repositoryService.getTraitNames(aGUID); |
| Assert.assertEquals(traitNames.size(), 3); |
| Assert.assertTrue(traitNames.contains(TestUtils.PII)); |
| Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); |
| Assert.assertTrue(traitNames.contains("P_I_I")); |
| |
| repositoryService.deleteTrait(aGUID, TestUtils.PII); |
| |
| // refresh trait names |
| traitNames = repositoryService.getTraitNames(aGUID); |
| Assert.assertEquals(traitNames.size(), 2); |
| Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION)); |
| Assert.assertFalse(traitNames.contains(TestUtils.PII)); |
| |
| // Verify modification timestamp was updated. |
| GraphHelper.getInstance().getVertexForGUID(aGUID); |
| Long modificationTimestampPostUpdate = GraphHelper.getSingleValuedProperty(AtlasVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class); |
| Assert.assertNotNull(modificationTimestampPostUpdate); |
| Assert.assertTrue(modificationTimestampPostUpdate > modificationTimestampPreUpdate); |
| } |
| |
| @Test(expectedExceptions = EntityNotFoundException.class) |
| public void testDeleteTraitForNonExistentEntity() throws Exception { |
| repositoryService.deleteTrait(UUID.randomUUID().toString(), TestUtils.PII); |
| Assert.fail(); |
| } |
| |
| @Test(expectedExceptions = TraitNotFoundException.class) |
| public void testDeleteTraitForNonExistentTrait() throws Exception { |
| final String aGUID = getGUID(); |
| repositoryService.deleteTrait(aGUID, "PCI"); |
| Assert.fail(); |
| } |
| |
| @Test(dependsOnMethods = "testCreateEntity") |
| public void testGetIdFromVertex() throws Exception { |
| AtlasVertex tableVertex = getTableEntityVertex(); |
| |
| String guid = GraphHelper.getSingleValuedProperty(tableVertex, Constants.GUID_PROPERTY_KEY, String.class); |
| if (guid == null) { |
| Assert.fail(); |
| } |
| |
| Id expected = new Id(guid, GraphHelper.getSingleValuedProperty(tableVertex, Constants.VERSION_PROPERTY_KEY, Integer.class), TestUtils.TABLE_TYPE); |
| Assert.assertEquals(GraphHelper.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected); |
| } |
| |
| @Test(dependsOnMethods = "testCreateEntity") |
| public void testGetTypeName() throws Exception { |
| AtlasVertex tableVertex = getTableEntityVertex(); |
| Assert.assertEquals(GraphHelper.getTypeName(tableVertex), TestUtils.TABLE_TYPE); |
| } |
| |
| @Test(dependsOnMethods = "testCreateEntity") |
| public void testSearchByDSLQuery() throws Exception { |
| String dslQuery = "hive_database as PII"; |
| System.out.println("Executing dslQuery = " + dslQuery); |
| String jsonResults = discoveryService.searchByDSL(dslQuery, queryParams); |
| Assert.assertNotNull(jsonResults); |
| |
| JSONObject results = new JSONObject(jsonResults); |
| Assert.assertEquals(results.length(), 3); |
| System.out.println("results = " + results); |
| |
| Object query = results.get("query"); |
| Assert.assertNotNull(query); |
| |
| JSONObject dataType = results.getJSONObject("dataType"); |
| Assert.assertNotNull(dataType); |
| String typeName = dataType.getString("typeName"); |
| Assert.assertNotNull(typeName); |
| |
| JSONArray rows = results.getJSONArray("rows"); |
| Assert.assertNotNull(rows); |
| Assert.assertTrue(rows.length() > 0); |
| |
| for (int index = 0; index < rows.length(); index++) { |
| JSONObject row = rows.getJSONObject(index); |
| String type = row.getString("$typeName$"); |
| Assert.assertEquals(type, "hive_database"); |
| |
| String name = row.getString("name"); |
| Assert.assertEquals(name, TestUtils.DATABASE_NAME); |
| } |
| } |
| |
| @Test(dependsOnMethods = "testSubmitEntity") |
| public void testSearchByDSLWithInheritance() throws Exception { |
| String dslQuery = "Person where name = 'Jane'"; |
| System.out.println("Executing dslQuery = " + dslQuery); |
| String jsonResults = discoveryService.searchByDSL(dslQuery, queryParams); |
| Assert.assertNotNull(jsonResults); |
| |
| JSONObject results = new JSONObject(jsonResults); |
| Assert.assertEquals(results.length(), 3); |
| System.out.println("results = " + results); |
| |
| Object query = results.get("query"); |
| Assert.assertNotNull(query); |
| |
| JSONObject dataType = results.getJSONObject("dataType"); |
| Assert.assertNotNull(dataType); |
| String typeName = dataType.getString("typeName"); |
| Assert.assertEquals(typeName, "Person"); |
| |
| JSONArray rows = results.getJSONArray("rows"); |
| Assert.assertEquals(rows.length(), 1); |
| |
| JSONObject row = rows.getJSONObject(0); |
| Assert.assertEquals(row.getString("$typeName$"), "Manager"); |
| Assert.assertEquals(row.getString("name"), "Jane"); |
| } |
| |
| @Test(dependsOnMethods = "testCreateEntity") |
| public void testBug37860() throws Exception { |
| String dslQuery = "hive_table as t where name = 'bar' " |
| + "database where name = 'foo' and description = 'foo database' select t"; |
| |
| TestUtils.dumpGraph(TestUtils.getGraph()); |
| |
| System.out.println("Executing dslQuery = " + dslQuery); |
| String jsonResults = discoveryService.searchByDSL(dslQuery, queryParams); |
| Assert.assertNotNull(jsonResults); |
| |
| JSONObject results = new JSONObject(jsonResults); |
| Assert.assertEquals(results.length(), 3); |
| System.out.println("results = " + results); |
| |
| Object query = results.get("query"); |
| Assert.assertNotNull(query); |
| |
| JSONObject dataType = results.getJSONObject("dataType"); |
| Assert.assertNotNull(dataType); |
| |
| JSONArray rows = results.getJSONArray("rows"); |
| Assert.assertEquals(rows.length(), 1); |
| |
| } |
| |
| /** |
| * Full text search requires GraphBackedSearchIndexer, and GraphBackedSearchIndexer can't be enabled in |
| * GraphBackedDiscoveryServiceTest because of its test data. So, test for full text search is in |
| * GraphBackedMetadataRepositoryTest:( |
| */ |
| @Test(dependsOnMethods = "testSubmitEntity") |
| public void testFullTextSearch() throws Exception { |
| //todo fix this |
| //Weird: with lucene, the test passes without sleep |
| //but with elasticsearch, doesn't work without sleep. why?? |
| long sleepInterval = 1000; |
| |
| TestUtils.dumpGraph(TestUtils.getGraph()); |
| |
| //person in hr department whose name is john |
| Thread.sleep(sleepInterval); |
| String response = discoveryService.searchByFullText("john", queryParams); |
| Assert.assertNotNull(response); |
| JSONArray results = new JSONArray(response); |
| Assert.assertEquals(results.length(), 1); |
| JSONObject row = (JSONObject) results.get(0); |
| Assert.assertEquals(row.get("typeName"), "Person"); |
| |
| //person in hr department who lives in santa clara |
| response = discoveryService.searchByFullText("Jane AND santa AND clara", queryParams); |
| Assert.assertNotNull(response); |
| results = new JSONArray(response); |
| Assert.assertEquals(results.length(), 1); |
| row = (JSONObject) results.get(0); |
| Assert.assertEquals(row.get("typeName"), "Manager"); |
| |
| //search for person in hr department whose name starts is john/jahn |
| response = discoveryService.searchByFullText("hr AND (john OR jahn)", queryParams); |
| Assert.assertNotNull(response); |
| results = new JSONArray(response); |
| Assert.assertEquals(results.length(), 1); |
| row = (JSONObject) results.get(0); |
| Assert.assertEquals(row.get("typeName"), "Person"); |
| |
| //verify limit and offset |
| //higher limit should return all results |
| results = new JSONArray(discoveryService.searchByFullText("Department", queryParams)); |
| assertEquals(results.length(), 5); |
| |
| //smaller limit should return those many rows |
| results = new JSONArray(discoveryService.searchByFullText("Department", new QueryParams(2, 0))); |
| assertEquals(results.length(), 2); |
| |
| //offset should offset the results |
| results = new JSONArray(discoveryService.searchByFullText("Department", new QueryParams(5, 2))); |
| assertEquals(results.length(), 3); |
| |
| //higher offset shouldn't return any rows |
| results = new JSONArray(discoveryService.searchByFullText("Department", new QueryParams(2, 6))); |
| assertEquals(results.length(), 0); |
| } |
| |
| private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception { |
| Referenceable tableInstance = new Referenceable(TestUtils.TABLE_TYPE, TestUtils.CLASSIFICATION); |
| tableInstance.set("name", TestUtils.TABLE_NAME); |
| tableInstance.set("description", "bar table"); |
| tableInstance.set("type", "managed"); |
| tableInstance.set("created", new Date(TestUtils.TEST_DATE_IN_LONG)); |
| tableInstance.set("tableType", 1); // enum |
| |
| // super type |
| tableInstance.set("namespace", "colo:cluster:hive:db:table"); |
| tableInstance.set("cluster", "cluster-1"); |
| tableInstance.set("colo", "colo-1"); |
| |
| // refer to an existing class |
| tableInstance.set("database", databaseInstance); |
| |
| ArrayList<String> columnNames = new ArrayList<>(); |
| columnNames.add("first_name"); |
| columnNames.add("last_name"); |
| tableInstance.set("columnNames", columnNames); |
| |
| Struct traitInstance = (Struct) tableInstance.getTrait(TestUtils.CLASSIFICATION); |
| traitInstance.set("tag", "foundation_etl"); |
| |
| Struct serde1Instance = new Struct("serdeType"); |
| serde1Instance.set("name", "serde1"); |
| serde1Instance.set("serde", "serde1"); |
| tableInstance.set("serde1", serde1Instance); |
| |
| Struct serde2Instance = new Struct("serdeType"); |
| serde2Instance.set("name", "serde2"); |
| serde2Instance.set("serde", "serde2"); |
| tableInstance.set("serde2", serde2Instance); |
| |
| // HashMap<String, Referenceable> columnsMap = new HashMap<>(); |
| ArrayList<Referenceable> columns = new ArrayList<>(); |
| for (int index = 0; index < 5; index++) { |
| Referenceable columnInstance = new Referenceable("column_type"); |
| final String name = "column_" + index; |
| columnInstance.set("name", name); |
| columnInstance.set("type", "string"); |
| |
| columns.add(columnInstance); |
| // columnsMap.put(name, columnInstance); |
| } |
| tableInstance.set("columns", columns); |
| // tableInstance.set("columnsMap", columnsMap); |
| |
| // HashMap<String, Struct> partitionsMap = new HashMap<>(); |
| ArrayList<Struct> partitions = new ArrayList<>(); |
| for (int index = 0; index < 5; index++) { |
| Struct partitionInstance = new Struct(TestUtils.PARTITION_STRUCT_TYPE); |
| final String name = "partition_" + index; |
| partitionInstance.set("name", name); |
| |
| partitions.add(partitionInstance); |
| // partitionsMap.put(name, partitionInstance); |
| } |
| tableInstance.set("partitions", partitions); |
| // tableInstance.set("partitionsMap", partitionsMap); |
| |
| HashMap<String, String> parametersMap = new HashMap<>(); |
| parametersMap.put("foo", "bar"); |
| parametersMap.put("bar", "baz"); |
| parametersMap.put("some", "thing"); |
| tableInstance.set("parametersMap", parametersMap); |
| |
| ClassType tableType = typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE); |
| return tableType.convert(tableInstance, Multiplicity.REQUIRED); |
| } |
| |
| private String randomUTF() { |
| return RandomStringUtils.random(10); |
| } |
| |
| private String randomString() { |
| return RandomStringUtils.randomAlphanumeric(10); |
| } |
| |
| @Test |
| public void testUTFValues() throws Exception { |
| Referenceable hrDept = new Referenceable("Department"); |
| Referenceable john = new Referenceable("Person"); |
| john.set("name", randomUTF()); |
| john.set("department", hrDept); |
| |
| hrDept.set("name", randomUTF()); |
| hrDept.set("employees", ImmutableList.of(john)); |
| |
| ClassType deptType = typeSystem.getDataType(ClassType.class, "Department"); |
| ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); |
| |
| List<String> guids = repositoryService.createEntities(hrDept2).getCreatedEntities(); |
| Assert.assertNotNull(guids); |
| Assert.assertEquals(guids.size(), 2); |
| Assert.assertNotNull(guids.get(0)); |
| Assert.assertNotNull(guids.get(1)); |
| } |
| } |
