ATLAS-4454 : Upgrade janusgraph version to 0.6.0
Signed-off-by: Pinal Shah <pinal.shah@freestoneinfotech.com>
diff --git a/graphdb/janus-hbase2/pom.xml b/graphdb/janus-hbase2/pom.xml
index 92a7104..883c344 100644
--- a/graphdb/janus-hbase2/pom.xml
+++ b/graphdb/janus-hbase2/pom.xml
@@ -34,7 +34,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-core</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>com.codahale.metrics</groupId>
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java
index 9aa552d..ffafc8c 100644
--- a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java
@@ -40,7 +40,9 @@
import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator;
import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
+import org.janusgraph.diskstorage.keycolumnvalue.KeySlicesIterator;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
+import org.janusgraph.diskstorage.keycolumnvalue.MultiSlicesQuery;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.util.RecordIterator;
import org.janusgraph.diskstorage.util.StaticArrayBuffer;
@@ -150,6 +152,11 @@
return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
}
+ @Override
+ public KeySlicesIterator getKeys(MultiSlicesQuery queries, StoreTransaction txh) throws BackendException {
+ throw new UnsupportedOperationException();
+ }
+
public static Filter getFilter(SliceQuery query) {
byte[] colStartBytes = query.getSliceStart().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java
index 40bcd1a..f98fa7b 100644
--- a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java
@@ -440,7 +440,7 @@
throw new TemporaryBackendException(e);
}
- sleepAfterWrite(txh, commitTime);
+ this.sleepAfterWrite(commitTime);
}
@Override
diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml
index 8f1ed54..6916097 100644
--- a/graphdb/janus/pom.xml
+++ b/graphdb/janus/pom.xml
@@ -65,15 +65,15 @@
</dependency>
<dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <scope>provided</scope>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-configuration2</artifactId>
+ <version>${commons-conf2.version}</version>
</dependency>
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-server</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>com.codahale.metrics</groupId>
@@ -101,7 +101,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-berkeleyje</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -113,7 +113,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-cassandra</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.cassandra.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -129,7 +129,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-cql</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -147,7 +147,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-es</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -159,7 +159,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-hbase</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -171,7 +171,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-solr</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -191,7 +191,13 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-lucene</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -269,7 +275,6 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
-
</dependencies>
<build>
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 0d47e38..3995cf9 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -29,6 +29,7 @@
import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.janusgraph.core.JanusGraph;
import org.janusgraph.core.JanusGraphException;
@@ -51,6 +52,7 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import static org.apache.atlas.ApplicationProperties.DEFAULT_INDEX_RECOVERY;
import static org.apache.atlas.ApplicationProperties.INDEX_RECOVERY_CONF;
@@ -100,18 +102,18 @@
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
//add serializers for non-standard property value types that Atlas uses
- janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
- janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
//not ideal, but avoids making large changes to Atlas
- janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
- janusConfig.addProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
- janusConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
- janusConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());
- janusConfig.addProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
- janusConfig.addProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
+ janusConfig.setProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());
return janusConfig;
}
@@ -188,14 +190,16 @@
@VisibleForTesting
static JanusGraph initJanusGraph(Configuration config) {
+
+ org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(config);
try {
- return JanusGraphFactory.open(config);
+ return JanusGraphFactory.open(conf2);
} catch (JanusGraphException e) {
LOG.warn("JanusGraphException: {}", e.getMessage());
if (e.getMessage().startsWith(OLDER_STORAGE_EXCEPTION)) {
LOG.info("Newer client is being used with older janus storage version. Setting allow-upgrade=true and reattempting connection");
config.addProperty("graph.allow-upgrade", true);
- return JanusGraphFactory.open(config);
+ return JanusGraphFactory.open(conf2);
} else {
throw new RuntimeException(e);
}
@@ -252,7 +256,9 @@
try {
Configuration cfg = getConfiguration();
cfg.setProperty("storage.batch-loading", true);
- return JanusGraphFactory.open(cfg);
+
+ org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(cfg);
+ return JanusGraphFactory.open(conf2);
} catch (IllegalArgumentException ex) {
LOG.error("getBulkLoadingGraphInstance: Failed!", ex);
} catch (AtlasException ex) {
@@ -262,6 +268,12 @@
return null;
}
+ private static org.apache.commons.configuration2.Configuration createConfiguration2(Configuration conf) {
+ Properties properties = ConfigurationConverter.getProperties(conf);
+
+ return org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
+ }
+
public static void unload() {
synchronized (AtlasJanusGraphDatabase.class) {
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
index 2d5bd8a..e026856 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
@@ -262,7 +262,7 @@
}
final GraphSONMapper.Builder builder = GraphSONMapper.build();
- final GraphSONMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create();
+ final GraphSONMapper mapper = builder.create();
return new AtlasGraphSONReader(mapper.createMapper(), relationshipCache, graph, bulkLoadGraph,
numWorkers, batchSize, suppliedStartIndex);
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
index 16fc445..a0f1460 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
@@ -236,7 +236,7 @@
@Override
public void orderBy(final String propertyName, final AtlasGraphQuery.SortOrder sortOrder) {
- Order order = sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.incr : Order.decr;
+ Order order = sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.asc : Order.desc;
query.orderBy(propertyName, order);
}
diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index 81ae202..23c11de 100644
--- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -672,6 +672,27 @@
doc -> doc.getFieldValue(keyIdField).toString());
}
+ @Override
+ public Long queryCount(IndexQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx) throws BackendException {
+ try {
+ String collection = query.getStore();
+ String keyIdField = this.getKeyFieldId(collection);
+ SolrQuery solrQuery = new SolrQuery("*:*");
+ solrQuery.set("fl", new String[]{keyIdField});
+ String queryFilter = this.buildQueryFilter(query.getCondition(), information.get(collection));
+ solrQuery.addFilterQuery(new String[]{queryFilter});
+ QueryResponse response = this.solrClient.query(collection, solrQuery);
+ logger.debug("Executed query [{}] in {} ms", query, response.getElapsedTime());
+ return response.getResults().getNumFound();
+ } catch (IOException ex) {
+ logger.error("Query did not complete : ", ex);
+ throw new PermanentBackendException(ex);
+ } catch (SolrServerException ex) {
+ logger.error("Unable to query Solr index.", ex);
+ throw new PermanentBackendException(ex);
+ }
+ }
+
private void addOrderToQuery(SolrQuery solrQuery, List<IndexQuery.OrderEntry> orders) {
for (final IndexQuery.OrderEntry order1 : orders) {
final String item = order1.getKey();
@@ -765,10 +786,12 @@
final String key = atom.getKey();
final JanusGraphPredicate predicate = atom.getPredicate();
- if (value instanceof Number) {
+ if (value == null && predicate == Cmp.NOT_EQUAL) {
+ return key + ":*";
+ } else if (value instanceof Number) {
final String queryValue = escapeValue(value);
Preconditions.checkArgument(predicate instanceof Cmp,
- "Relation not supported on numeric types: " + predicate);
+ "Relation not supported on numeric types: %s", predicate);
final Cmp numRel = (Cmp) predicate;
switch (numRel) {
case EQUAL:
@@ -804,18 +827,18 @@
return (key + ":" + escapeValue(value) + "*");
} else if (predicate == Text.REGEX || predicate == Text.CONTAINS_REGEX) {
return (key + ":/" + value + "/");
- } else if (predicate == Cmp.EQUAL) {
+ } else if (predicate == Cmp.EQUAL || predicate == Cmp.NOT_EQUAL) {
final String tokenizer =
ParameterType.STRING_ANALYZER.findParameter(information.get(key).getParameters(), null);
- if(tokenizer != null){
- return tokenize(information, value, key, predicate,tokenizer);
- } else {
+ if (tokenizer != null) {
+ return tokenize(information, value, key, predicate, tokenizer);
+ } else if (predicate == Cmp.EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
+ } else { // Cmp.NOT_EQUAL case
+ return ("-" + key + ":\"" + escapeValue(value) + "\"");
}
- } else if (predicate == Cmp.NOT_EQUAL) {
- return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else if (predicate == Text.FUZZY || predicate == Text.CONTAINS_FUZZY) {
- return (key + ":"+escapeValue(value)+"~");
+ return (key + ":"+escapeValue(value)+"~"+Text.getMaxEditDistance(value.toString()));
} else if (predicate == Cmp.LESS_THAN) {
return (key + ":[* TO \"" + escapeValue(value) + "\"}");
} else if (predicate == Cmp.LESS_THAN_EQUAL) {
@@ -830,9 +853,9 @@
} else if (value instanceof Geoshape) {
final Mapping map = Mapping.getMapping(information.get(key));
Preconditions.checkArgument(predicate instanceof Geo && predicate != Geo.DISJOINT,
- "Relation not supported on geo types: " + predicate);
+ "Relation not supported on geo types: %s", predicate);
Preconditions.checkArgument(map == Mapping.PREFIX_TREE || predicate == Geo.WITHIN || predicate == Geo.INTERSECT,
- "Relation not supported on geopoint types: " + predicate);
+ "Relation not supported on geopoint types: %s", predicate);
final Geoshape geo = (Geoshape)value;
if (geo.getType() == Geoshape.Type.CIRCLE && (predicate == Geo.INTERSECT || map == Mapping.DEFAULT)) {
final Geoshape.Point center = geo.getPoint();
@@ -852,8 +875,7 @@
} else if (value instanceof Date || value instanceof Instant) {
final String s = value.toString();
final String queryValue = escapeValue(value instanceof Date ? toIsoDate((Date) value) : value.toString());
- Preconditions.checkArgument(predicate instanceof Cmp, "Relation not supported on date types: "
- + predicate);
+ Preconditions.checkArgument(predicate instanceof Cmp, "Relation not supported on date types: %s", predicate);
final Cmp numRel = (Cmp) predicate;
switch (numRel) {
@@ -944,7 +966,11 @@
if (terms.isEmpty()) {
return "";
} else if (terms.size() == 1) {
- return (key + ":(" + escapeValue(terms.get(0)) + ")");
+ if (janusgraphPredicate == Cmp.NOT_EQUAL) {
+ return ("-" + key + ":(" + escapeValue(terms.get(0)) + ")");
+ } else {
+ return (key + ":(" + escapeValue(terms.get(0)) + ")");
+ }
} else {
final And<JanusGraphElement> andTerms = new And<>();
for (final String term : terms) {
@@ -999,7 +1025,7 @@
@Override
public void close() throws BackendException {
- logger.trace("Shutting down connection to Solr", solrClient);
+ logger.trace("Shutting down connection to Solr {}", solrClient);
try {
solrClient.close();
} catch (final IOException e) {
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
index 82ae675..2f311f2 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
@@ -73,7 +73,7 @@
private JsonNode getEntityNode(String json) throws IOException {
GraphSONMapper.Builder builder = GraphSONMapper.build();
- final ObjectMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create().createMapper();
+ final ObjectMapper mapper = builder.create().createMapper();
return mapper.readTree(json);
}
diff --git a/pom.xml b/pom.xml
index 220ffe0..0b6bb3d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -717,7 +717,8 @@
<httpcomponents-httpcore.version>4.4.13</httpcomponents-httpcore.version>
<jackson.databind.version>2.11.3</jackson.databind.version>
<jackson.version>2.11.3</jackson.version>
- <janus.version>0.5.3</janus.version>
+ <janusgraph.version>0.6.0</janusgraph.version>
+ <janusgraph.cassandra.version>0.5.3</janusgraph.cassandra.version>
<javax-inject.version>1</javax-inject.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
<jersey-spring.version>1.19.4</jersey-spring.version>
@@ -767,7 +768,7 @@
<surefire.forkCount>2C</surefire.forkCount>
<surefire.version>3.0.0-M5</surefire.version>
<testng.version>6.9.4</testng.version>
- <tinkerpop.version>3.4.10</tinkerpop.version>
+ <tinkerpop.version>3.5.1</tinkerpop.version>
<woodstox-core.version>5.0.3</woodstox-core.version>
<zookeeper.version>3.5.7</zookeeper.version>
</properties>
diff --git a/repository/src/test/java/org/apache/atlas/query/TraversalComposerTest.java b/repository/src/test/java/org/apache/atlas/query/TraversalComposerTest.java
index dd56ed1..8f76033 100644
--- a/repository/src/test/java/org/apache/atlas/query/TraversalComposerTest.java
+++ b/repository/src/test/java/org/apache/atlas/query/TraversalComposerTest.java
@@ -53,7 +53,7 @@
"[JanusGraphStep([],[__typeName.eq(DB)]), DedupGlobalStep@[d], RangeGlobalStep(0,25)]");
verify("Table groupby(owner) select name, owner, clusterName orderby name",
- "[JanusGraphStep([],[__typeName.eq(Table)]), TraversalFilterStep([JanusGraphPropertiesStep([Table.owner],property)]), GroupStep(value(Table.owner),[FoldStep]), DedupGlobalStep, RangeGlobalStep(0,25)]");
+ "[JanusGraphStep([],[__typeName.eq(Table), Table.owner.neq]), GroupStep(value(Table.owner),[FoldStep]), DedupGlobalStep, RangeGlobalStep(0,25)]");
}
private void verify(String dsl, String expected) {
diff --git a/tools/atlas-index-repair/pom.xml b/tools/atlas-index-repair/pom.xml
index a723bfa..ab55f4c 100644
--- a/tools/atlas-index-repair/pom.xml
+++ b/tools/atlas-index-repair/pom.xml
@@ -53,7 +53,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-core</artifactId>
- <version>${janus.version}</version>
+ <version>${janusgraph.version}</version>
</dependency>
</dependencies>
