ATLAS-17 Parameterize schema API query per typeName (shwethags)
diff --git a/release-log.txt b/release-log.txt
index 56be318..0bfac32 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,18 +1,10 @@
Apache Atlas Release Notes
==========================
---trunk - unreleased
-
-INCOMPATIBLE CHANGES:
-
-ALL CHANGES:
-
-ATLAS-10 Update trunk version to 0.6-incubating-SNAPSHOT (shwethags)
-
--Release 0.5-incubating
ALL CHANGES:
-
+ATLAS-17 Parameterize schema API query per typeName (shwethags)
ATLAS-18 Build fails because of a missing artifact (Ajay Yadava via Venkatesh Seetharam)
ATLAS-13 Add project website (Venkatesh Seetharam)
ATLAS-12 Update the copyright in Notice and License files (Venkatesh Seetharam)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
index 9764f8f..76d94a3 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
@@ -32,6 +32,7 @@
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,26 +54,27 @@
private static final Option<List<String>> SELECT_ATTRIBUTES =
Some.<List<String>>apply(List.<String>fromArray(new String[]{"name"}));
+ public static final String HIVE_TABLE_SCHEMA_QUERY_PREFIX = "atlas.lineage.hive.table.schema.query.";
+
private static final String HIVE_TABLE_TYPE_NAME;
private static final String HIVE_PROCESS_TYPE_NAME;
private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME;
private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME;
- private static final String HIVE_TABLE_SCHEMA_QUERY;
private static final String HIVE_TABLE_EXISTS_QUERY;
+ private static final PropertiesConfiguration propertiesConf;
+
static {
// todo - externalize this using type system - dog food
try {
- PropertiesConfiguration conf = PropertiesUtil.getApplicationProperties();
- HIVE_TABLE_TYPE_NAME = conf.getString("atlas.lineage.hive.table.type.name", "DataSet");
- HIVE_PROCESS_TYPE_NAME = conf.getString("atlas.lineage.hive.process.type.name", "Process");
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.inputs.name", "inputs");
- HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = conf.getString("atlas.lineage.hive.process.outputs.name", "outputs");
+ propertiesConf = PropertiesUtil.getApplicationProperties();
+ HIVE_TABLE_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.table.type.name", "DataSet");
+ HIVE_PROCESS_TYPE_NAME = propertiesConf.getString("atlas.lineage.hive.process.type.name", "Process");
+ HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.inputs.name", "inputs");
+ HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = propertiesConf.getString("atlas.lineage.hive.process.outputs.name", "outputs");
- HIVE_TABLE_SCHEMA_QUERY =
- conf.getString("atlas.lineage.hive.table.schema.query", "hive_table where name=\"%s\", columns");
- HIVE_TABLE_EXISTS_QUERY = conf.getString("atlas.lineage.hive.table.exists.query",
+ HIVE_TABLE_EXISTS_QUERY = propertiesConf.getString("atlas.lineage.hive.table.exists.query",
"from " + HIVE_TABLE_TYPE_NAME + " where name=\"%s\"");
} catch (AtlasException e) {
throw new RuntimeException(e);
@@ -195,9 +197,10 @@
public String getSchema(String tableName) throws AtlasException {
LOG.info("Fetching schema for tableName={}", tableName);
ParamChecker.notEmpty(tableName, "table name cannot be null");
- validateTableExists(tableName);
+ String typeName = validateTableExists(tableName);
- final String schemaQuery = String.format(HIVE_TABLE_SCHEMA_QUERY, tableName);
+ final String schemaQuery =
+ String.format(propertiesConf.getString(HIVE_TABLE_SCHEMA_QUERY_PREFIX + typeName), tableName);
return discoveryService.searchByDSL(schemaQuery);
}
@@ -206,11 +209,14 @@
*
* @param tableName table name
*/
- private void validateTableExists(String tableName) throws AtlasException {
+ private String validateTableExists(String tableName) throws AtlasException {
final String tableExistsQuery = String.format(HIVE_TABLE_EXISTS_QUERY, tableName);
GremlinQueryResult queryResult = discoveryService.evaluate(tableExistsQuery);
if (!(queryResult.rows().length() > 0)) {
throw new EntityNotFoundException(tableName + " does not exist");
}
+
+ ReferenceableInstance referenceable = (ReferenceableInstance)queryResult.rows().apply(0);
+ return referenceable.getTypeName();
}
}
diff --git a/repository/src/test/resources/application.properties b/repository/src/test/resources/application.properties
index 32d40b0..20c0c51 100755
--- a/repository/src/test/resources/application.properties
+++ b/repository/src/test/resources/application.properties
@@ -36,7 +36,7 @@
#atlas.lineage.hive.process.outputs.name=outputs
## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
######### Security Properties #########
diff --git a/src/conf/application.properties b/src/conf/application.properties
index 5818701..6c4c7d2 100755
--- a/src/conf/application.properties
+++ b/src/conf/application.properties
@@ -36,7 +36,8 @@
#atlas.lineage.hive.process.outputs.name=outputs
## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
+atlas.lineage.hive.table.schema.query.Table=Table where name='%s'\, columns
######### Security Properties #########
diff --git a/webapp/src/main/resources/application.properties b/webapp/src/main/resources/application.properties
index 9ade2a7..c9b8408 100755
--- a/webapp/src/main/resources/application.properties
+++ b/webapp/src/main/resources/application.properties
@@ -36,7 +36,7 @@
#atlas.lineage.hive.process.outputs.name=outputs
## Schema
-#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
######### Security Properties #########
