OAK-9325 Tool to compare and manually merge index definitions
git-svn-id: https://svn.apache.org/repos/asf/jackrabbit/oak/trunk@1886381 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
index 04881e9..8689223 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
@@ -22,10 +22,8 @@
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Calendar;
import java.util.LinkedHashSet;
import java.util.List;
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexConsistencyCheckPrinter.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexConsistencyCheckPrinter.java
index 4e91236..9e87d37 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexConsistencyCheckPrinter.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexConsistencyCheckPrinter.java
@@ -19,7 +19,6 @@
package org.apache.jackrabbit.oak.index;
-import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDefinitionUpdater.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDefinitionUpdater.java
index bf380cd..c4d8e70 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDefinitionUpdater.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDefinitionUpdater.java
@@ -77,7 +77,7 @@
Preconditions.checkArgument(optionSet.has("initializer"), "initializer class must be provided");
String initializerClassName = optionSet.valueOf("initializer").toString();
- Class repoInitClazz = Class.forName(initializerClassName);
+ Class<?> repoInitClazz = Class.forName(initializerClassName);
Object obj = repoInitClazz.newInstance();
Preconditions.checkArgument(obj instanceof RepositoryInitializer, repoInitClazz + " is not a RepositoryInitializer");
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
index 708bf45..fbae7ba 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
@@ -37,7 +37,6 @@
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.FSDirectoryFactory;
import org.apache.jackrabbit.oak.plugins.index.progress.MetricRateEstimator;
import org.apache.jackrabbit.oak.plugins.index.progress.NodeCounterMBeanEstimator;
-import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.plugins.metric.MetricStatisticsProvider;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -55,7 +54,7 @@
import static java.util.Arrays.asList;
public class OutOfBandIndexer implements Closeable, IndexUpdateCallback, NodeTraversalCallback {
- private final Logger log = LoggerFactory.getLogger(getClass());
+ private final Logger LOG = LoggerFactory.getLogger(getClass());
/**
* Index lane name which is used for indexing
*/
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerUtils.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerUtils.java
index b530357..71d7719 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerUtils.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerUtils.java
@@ -30,6 +30,7 @@
import java.util.stream.Collectors;
import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.IndexName;
/**
@@ -39,21 +40,30 @@
private static HashSet<String> IGNORE_LEVEL_0 = new HashSet<>(Arrays.asList(
"reindex", "refresh", "seed", "reindexCount"));
+ private static HashSet<String> USE_PRODUCT_PROPERTY = new HashSet<>(Arrays.asList(
+ "jcr:created", "jcr:lastModified", "jcr:uuid"));
+ private static HashSet<String> USE_PRODUCT_CHILD_LEVEL_0 = new HashSet<>(Arrays.asList(
+ "tika"));
/**
* Merge index definition changes.
*
+ * @param ancestorName the name of the node of the ancestor index (e.g. /oak:index/lucene-1)
* @param ancestor the common ancestor (the old product index, e.g. lucene)
- * @param custom the latest customized version (e.g. lucene-custom-1)
+ * @param customName the name of the node of the customized index (e.g. /oak:index/lucene-1-custom-1)
+ * @param custom the latest customized version (e.g. lucene-1-custom-1)
* @param product the latest product index (e.g. lucene-2)
* @return the merged index definition (e.g. lucene-2-custom-1)
*/
- public static JsonObject merge(JsonObject ancestor, JsonObject custom, JsonObject product) {
+ public static JsonObject merge(String ancestorName, JsonObject ancestor, String customName, JsonObject custom, JsonObject product) {
ArrayList<String> conflicts = new ArrayList<>();
JsonObject merged = merge(0, ancestor, custom, product, conflicts);
if (!conflicts.isEmpty()) {
throw new UnsupportedOperationException("Conflicts detected: " + conflicts);
}
+ merged.getProperties().put("merges", "[" +
+ JsopBuilder.encode(ancestorName) + ", " +
+ JsopBuilder.encode(customName) + "]");
return merged;
}
@@ -95,9 +105,10 @@
}
}
LinkedHashMap<String, Boolean> children = new LinkedHashMap<>();
+ // first the (new) product index - to ensure the order of children matches the new product index
+ addAllChildren(product, children);
addAllChildren(ancestor, children);
addAllChildren(custom, children);
- addAllChildren(product, children);
for (String k : children.keySet()) {
if (k.startsWith(":")) {
// ignore hidden nodes
@@ -126,6 +137,9 @@
private static String mergeProperty(String property, JsonObject ancestor, JsonObject custom, JsonObject product,
ArrayList<String> conflicts) {
+ if (USE_PRODUCT_PROPERTY.contains(property)) {
+ return product.getProperties().get(property);
+ }
String ap = ancestor.getProperties().get(property);
String cp = custom.getProperties().get(property);
String pp = product.getProperties().get(property);
@@ -145,6 +159,9 @@
JsonObject a = ancestor.getChildren().get(child);
JsonObject c = custom.getChildren().get(child);
JsonObject p = product.getChildren().get(child);
+ if (level == 0 && USE_PRODUCT_CHILD_LEVEL_0.contains(child)) {
+ return p;
+ }
if (isSameJson(a, p) || isSameJson(c, p)) {
return c;
} else if (isSameJson(a, c)) {
@@ -200,8 +217,15 @@
JsonObject latestCustomized = allIndexes.getChildren().get(latest.getNodeName());
JsonObject latestAncestor = allIndexes.getChildren().get(ancestor.getNodeName());
JsonObject newProduct = newIndexes.getChildren().get(n.getNodeName());
- JsonObject merged = merge(latestAncestor, latestCustomized, newProduct);
- mergedMap.put(n.nextCustomizedName(), merged);
+ try {
+ JsonObject merged = merge(
+ ancestor.getNodeName(), latestAncestor,
+ latest.getNodeName(), latestCustomized,
+ newProduct);
+ mergedMap.put(n.nextCustomizedName(), merged);
+ } catch (UnsupportedOperationException e) {
+ throw new UnsupportedOperationException("Index: " + n.getNodeName() + ": " + e.getMessage(), e);
+ }
}
}
}
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDiff.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDiff.java
new file mode 100644
index 0000000..7b4a0b3
--- /dev/null
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDiff.java
@@ -0,0 +1,532 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.merge;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.plugins.index.search.spi.query.IndexName;
+
+/**
+ * The index diff tools allows to compare and merge indexes
+ */
+public class IndexDiff {
+
+ private static final String OAK_INDEX = "/oak:index/";
+
+ static JsonObject extract(String extractFile, String indexName) {
+ JsonObject indexDefs = parseIndexDefinitions(extractFile);
+ JsonObject index = indexDefs.getChildren().get(indexName);
+ removeUninterestingIndexProperties(indexDefs);
+ simplify(index);
+ return index;
+ }
+
+ static void extractAll(String extractFile, String extractTargetDirectory) {
+ new File(extractTargetDirectory).mkdirs();
+ JsonObject indexDefs = parseIndexDefinitions(extractFile);
+ removeUninterestingIndexProperties(indexDefs);
+ sortPropertiesByName(indexDefs);
+ for (String child : indexDefs.getChildren().keySet()) {
+ JsonObject index = indexDefs.getChildren().get(child);
+ simplify(index);
+ String fileName = child.replaceAll(OAK_INDEX, "");
+ fileName = fileName.replace(':', '-');
+ Path p = Paths.get(extractTargetDirectory, fileName + ".json");
+ writeFile(p, index);
+ }
+ }
+
+ private static void writeFile(Path p, JsonObject json) {
+ try {
+ Files.write(p, json.toString().getBytes());
+ } catch (IOException e) {
+ throw new IllegalStateException("Error writing file: " + p, e);
+ }
+ }
+
+ static JsonObject collectCustomizations(String directory) {
+ Path indexPath = Paths.get(directory);
+ JsonObject target = new JsonObject(true);
+ collectCustomizationsInDirectory(indexPath, target);
+ return target;
+ }
+
+ static JsonObject mergeIndexes(String directory, String newIndexFile) {
+ JsonObject newIndex = null;
+ if (newIndexFile != null && !newIndexFile.isEmpty()) {
+ newIndex = parseIndexDefinitions(newIndexFile);
+ }
+ Path indexPath = Paths.get(directory);
+ JsonObject target = new JsonObject(true);
+ mergeIndexesInDirectory(indexPath, newIndex, target);
+ for(String key : target.getChildren().keySet()) {
+ JsonObject c = target.getChildren().get(key);
+ removeUninterestingIndexProperties(c);
+ sortPropertiesByName(c);
+ simplify(c);
+ target.getChildren().put(key, c);
+ }
+ return target;
+ }
+
+ static void mergeIndex(String oldIndexFile, String newIndexFile, String targetDirectory) {
+ JsonObject oldIndexes = parseIndexDefinitions(oldIndexFile);
+ removeUninterestingIndexProperties(oldIndexes);
+ sortPropertiesByName(oldIndexes);
+ simplify(oldIndexes);
+
+ JsonObject newIndexes = parseIndexDefinitions(newIndexFile);
+ removeUninterestingIndexProperties(newIndexes);
+ sortPropertiesByName(newIndexes);
+ simplify(newIndexes);
+
+ List<IndexName> newNames = newIndexes.getChildren().keySet().stream().map(s -> IndexName.parse(s))
+ .collect(Collectors.toList());
+ List<IndexName> allNames = oldIndexes.getChildren().keySet().stream().map(s -> IndexName.parse(s))
+ .collect(Collectors.toList());
+
+ for (IndexName n : newNames) {
+ if (n.getCustomerVersion() == 0) {
+ IndexName latest = n.getLatestCustomized(allNames);
+ IndexName ancestor = n.getLatestProduct(allNames);
+ if (latest != null && ancestor != null) {
+ if (n.compareTo(latest) <= 0 || n.compareTo(ancestor) <= 0) {
+ // ignore older versions of indexes
+ continue;
+ }
+ JsonObject latestCustomized = oldIndexes.getChildren().get(latest.getNodeName());
+ String fileName = PathUtils.getName(latest.getNodeName());
+ writeFile(Paths.get(targetDirectory, fileName + ".json"),
+ addParent(latest.getNodeName(), latestCustomized));
+
+ JsonObject latestAncestor = oldIndexes.getChildren().get(ancestor.getNodeName());
+ fileName = PathUtils.getName(ancestor.getNodeName());
+ writeFile(Paths.get(targetDirectory, fileName + ".json"),
+ addParent(ancestor.getNodeName(), latestAncestor));
+
+ JsonObject newProduct = newIndexes.getChildren().get(n.getNodeName());
+ fileName = PathUtils.getName(n.getNodeName());
+ writeFile(Paths.get(targetDirectory, fileName + ".json"),
+ addParent(n.getNodeName(), newProduct));
+
+ JsonObject oldCustomizations = new JsonObject(true);
+ compareIndexes("", latestAncestor, latestCustomized, oldCustomizations);
+ // the old product index might be disabled
+ oldCustomizations.getChildren().remove("type");
+ writeFile(Paths.get(targetDirectory, "oldCustomizations.json"),
+ oldCustomizations);
+
+ JsonObject productChanges = new JsonObject(true);
+ compareIndexes("", latestAncestor, newProduct, productChanges);
+ writeFile(Paths.get(targetDirectory, "productChanges.json"),
+ productChanges);
+
+ try {
+ JsonObject merged = IndexDefMergerUtils.merge(
+ ancestor.getNodeName(), latestAncestor,
+ latest.getNodeName(), latestCustomized,
+ newProduct);
+ fileName = PathUtils.getName(n.nextCustomizedName());
+ writeFile(Paths.get(targetDirectory, fileName + ".json"),
+ addParent(n.nextCustomizedName(), merged));
+
+ JsonObject newCustomizations = new JsonObject(true);
+ compareIndexes("", newProduct, merged, newCustomizations);
+ writeFile(Paths.get(targetDirectory, "newCustomizations.json"),
+ newCustomizations);
+
+ JsonObject changes = new JsonObject(true);
+ compareIndexes("", oldCustomizations, newCustomizations, changes);
+ writeFile(Paths.get(targetDirectory, "changes.json"),
+ changes);
+
+ } catch (UnsupportedOperationException e) {
+ throw new UnsupportedOperationException("Index: " + n.getNodeName() + ": " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+ }
+
+ private static JsonObject addParent(String key, JsonObject obj) {
+ JsonObject result = new JsonObject(true);
+ result.getChildren().put(key, obj);
+ return result;
+ }
+
+ static JsonObject compareIndexes(String directory, String index1, String index2) {
+ Path indexPath = Paths.get(directory);
+ JsonObject target = new JsonObject(true);
+ compareIndexesIndexesInDirectory(indexPath, index1, index2, target);
+ return target;
+ }
+
+ private static Stream<Path> indexFiles(Path indexPath) {
+ try {
+ return Files.walk(indexPath).
+ filter(path -> Files.isRegularFile(path)).
+ filter(path -> path.toString().endsWith(".json")).
+ filter(path -> !path.toString().endsWith("allnamespaces.json")).
+ filter(path -> !path.toString().endsWith("-info.json")).
+ filter(path -> !path.toString().endsWith("-stats.json"));
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Error reading from " + indexPath, e);
+ }
+ }
+
+ private static void sortPropertiesByName(JsonObject obj) {
+ ArrayList<String> props = new ArrayList<>(obj.getProperties().keySet());
+ if (!props.isEmpty()) {
+ props.sort(null);
+ for(String key : props) {
+ String value = obj.getProperties().remove(key);
+ obj.getProperties().put(key, value);
+ }
+ }
+ for(String child : obj.getChildren().keySet()) {
+ JsonObject c = obj.getChildren().get(child);
+ sortPropertiesByName(c);
+ }
+ }
+
+ private static void compareIndexesIndexesInDirectory(Path indexPath, String index1, String index2,
+ JsonObject target) {
+ if (Files.isExecutable(indexPath)) {
+ indexFiles(indexPath).forEach(path -> {
+ JsonObject indexDefinitions = IndexDiff.parseIndexDefinitions(path.toString());
+ compareIndexes(indexDefinitions, indexPath.toString(), path.toString(), index1, index2, target);
+ });
+ } else {
+ JsonObject allIndexDefinitions = IndexDiff.parseIndexDefinitions(indexPath.toString());
+ for(String key : allIndexDefinitions.getChildren().keySet()) {
+ JsonObject indexDefinitions = allIndexDefinitions.getChildren().get(key);
+ compareIndexes(indexDefinitions, "", key, index1, index2, target);
+ }
+ }
+ }
+
+ private static void collectCustomizationsInDirectory(Path indexPath, JsonObject target) {
+ indexFiles(indexPath).forEach(path -> {
+ JsonObject indexDefinitions = IndexDiff.parseIndexDefinitions(path.toString());
+ showCustomIndexes(indexDefinitions, indexPath.toString(), path.toString(), target);
+ });
+ }
+
+ private static void mergeIndexesInDirectory(Path indexPath, JsonObject newIndex, JsonObject target) {
+ indexFiles(indexPath).forEach(path -> {
+ JsonObject indexDefinitions = IndexDiff.parseIndexDefinitions(path.toString());
+ mergeIndexes(indexDefinitions, indexPath.toString(), path.toString(), newIndex, target);
+ });
+ }
+
+ private static void mergeIndexes(JsonObject indexeDefinitions, String basePath, String fileName, JsonObject newIndexes, JsonObject target) {
+ JsonObject targetFile = new JsonObject(true);
+ if (newIndexes != null) {
+ for (String newIndexKey : newIndexes.getChildren().keySet()) {
+ if (indexeDefinitions.getChildren().containsKey(newIndexKey)) {
+ targetFile.getProperties().put(newIndexKey, JsopBuilder.encode("WARNING: already exists"));
+ }
+ }
+ } else {
+ newIndexes = new JsonObject(true);
+ }
+ // the superseded indexes of the old repository
+ List<String> supersededKeys = new ArrayList<>(IndexMerge.getSupersededIndexDefs(indexeDefinitions));
+ Collections.sort(supersededKeys);
+
+ // keep only new indexes that are not superseded
+ Map<String, JsonObject> indexMap = indexeDefinitions.getChildren();
+ for (String superseded : supersededKeys) {
+ indexMap.remove(superseded);
+ }
+ Set<String> indexKeys = indexeDefinitions.getChildren().keySet();
+ try {
+ IndexDefMergerUtils.merge(newIndexes, indexeDefinitions);
+ Set<String> newIndexKeys = new HashSet<>(newIndexes.getChildren().keySet());
+ newIndexKeys.removeAll(indexKeys);
+ for (String newIndexKey : newIndexKeys) {
+ JsonObject merged = newIndexes.getChildren().get(newIndexKey);
+ if (merged != null) {
+ targetFile.getChildren().put(newIndexKey, merged);
+ }
+ }
+ } catch (UnsupportedOperationException e) {
+ e.printStackTrace();
+ targetFile.getProperties().put("failed", JsopBuilder.encode(e.toString()));
+ }
+ addIfNotEmpty(basePath, fileName, targetFile, target);
+ }
+
+ private static void addIfNotEmpty(String basePath, String fileName, JsonObject targetFile, JsonObject target) {
+ if (!targetFile.getProperties().isEmpty() || !targetFile.getChildren().isEmpty()) {
+ String f = fileName;
+ if (f.startsWith(basePath)) {
+ f = f.substring(basePath.length());
+ }
+ target.getChildren().put(f, targetFile);
+ }
+ }
+
+ private static void showCustomIndexes(JsonObject indexDefinitions, String basePath, String fileName, JsonObject target) {
+ JsonObject targetFile = new JsonObject(true);
+ processAndRemoveIllegalIndexNames(indexDefinitions, targetFile);
+ removeUninterestingIndex(indexDefinitions);
+ removeUninterestingIndexProperties(indexDefinitions);
+ removeUnusedIndexes(indexDefinitions);
+ for(String k : indexDefinitions.getChildren().keySet()) {
+ if (!k.startsWith(OAK_INDEX)) {
+ targetFile.getProperties().put(k, JsopBuilder.encode("WARNING: Index not under " + OAK_INDEX));
+ continue;
+ }
+ if (!k.contains("-custom-")) {
+ continue;
+ }
+ listNewAndCustomizedIndexes(indexDefinitions, k, targetFile);
+ }
+ addIfNotEmpty(basePath, fileName, targetFile, target);
+ }
+
+ private static void compareIndexes(JsonObject indexDefinitions, String basePath, String fileName, String index1, String index2, JsonObject target) {
+ JsonObject targetFile = new JsonObject(true);
+ JsonObject i1 = indexDefinitions.getChildren().get(index1);
+ JsonObject i2 = indexDefinitions.getChildren().get(index2);
+ if (i1 != null && i2 != null) {
+ compareIndexes("", i1, i2, targetFile);
+ }
+ addIfNotEmpty(basePath, fileName, targetFile, target);
+ }
+
+ private static void listNewAndCustomizedIndexes(JsonObject indexDefinitions, String indexNodeName, JsonObject target) {
+ JsonObject index = indexDefinitions.getChildren().get(indexNodeName);
+ String nodeName = indexNodeName.substring(OAK_INDEX.length());
+ IndexName indexName = IndexName.parse(nodeName);
+ String ootb = indexName.getBaseName();
+ if (indexName.getProductVersion() > 1) {
+ ootb += "-" + indexName.getProductVersion();
+ }
+ simplify(indexDefinitions);
+ JsonObject ootbIndex = indexDefinitions.getChildren().get(OAK_INDEX + ootb);
+ if (ootbIndex != null) {
+ JsonObject targetCustom = new JsonObject(true);
+ targetCustom.getProperties().put("customizes", JsopBuilder.encode(OAK_INDEX + ootb));
+ target.getChildren().put(indexNodeName, targetCustom);
+ compareIndexes("", ootbIndex, index, targetCustom);
+ } else {
+ target.getProperties().put(indexNodeName, JsopBuilder.encode("new"));
+ }
+ }
+
+ private static void processAndRemoveIllegalIndexNames(JsonObject indexDefinitions, JsonObject target) {
+ Set<String> indexes = new HashSet<>(indexDefinitions.getChildren().keySet());
+ for(String k : indexes) {
+ if (!k.startsWith("/oak:index/")) {
+ continue;
+ }
+ String nodeName = k.substring("/oak:index/".length());
+ IndexName indexName = IndexName.parse(nodeName);
+ if (!indexName.isLegal()) {
+ target.getProperties().put(k, JsopBuilder.encode("WARNING: Invalid name"));
+ indexDefinitions.getChildren().remove(k);
+ }
+ }
+ }
+
+ private static void removeUninterestingIndex(JsonObject indexDefinitions) {
+ }
+
+ private static void compareIndexes(String path, JsonObject ootb, JsonObject custom, JsonObject target) {
+ LinkedHashMap<String, Boolean> properties = new LinkedHashMap<>();
+ addAllProperties(ootb, properties);
+ addAllProperties(custom, properties);
+ for (String k : properties.keySet()) {
+ String op = ootb.getProperties().get(k);
+ String cp = custom.getProperties().get(k);
+ if (!Objects.equals(op, cp)) {
+ JsonObject change = new JsonObject(true);
+ if (op != null) {
+ change.getProperties().put("old", op);
+ }
+ if (cp != null) {
+ change.getProperties().put("new", cp);
+ }
+ target.getChildren().put(path + k, change);
+ }
+ }
+ LinkedHashMap<String, Boolean> children = new LinkedHashMap<>();
+ addAllChildren(ootb, children);
+ addAllChildren(custom, children);
+ for (String k : children.keySet()) {
+ JsonObject oc = ootb.getChildren().get(k);
+ JsonObject cc = custom.getChildren().get(k);
+ if (!isSameJson(oc, cc)) {
+ if (oc == null) {
+ target.getProperties().put(path + k, JsopBuilder.encode("added"));
+ } else if (cc == null) {
+ target.getProperties().put(path + k, JsopBuilder.encode("removed"));
+ } else {
+ compareIndexes(path + k + "/", oc, cc, target);
+ }
+ }
+ }
+ compareOrder(path, ootb, custom, target);
+ }
+
+ private static void addAllChildren(JsonObject source, LinkedHashMap<String, Boolean> target) {
+ for(String k : source.getChildren().keySet()) {
+ target.put(k, true);
+ }
+ }
+
+ private static void compareOrder(String path, JsonObject ootb, JsonObject custom, JsonObject target) {
+ // list of entries, sorted by how they appear in the ootb case
+ ArrayList<String> bothSortedByOotb = new ArrayList<>();
+ for(String k : ootb.getChildren().keySet()) {
+ if (custom.getChildren().containsKey(k)) {
+ bothSortedByOotb.add(k);
+ }
+ }
+ // list of entries, sorted by how they appear in the custom case
+ ArrayList<String> bothSortedByCustom = new ArrayList<>();
+ for(String k : custom.getChildren().keySet()) {
+ if (ootb.getChildren().containsKey(k)) {
+ bothSortedByCustom.add(k);
+ }
+ }
+ if (!bothSortedByOotb.toString().equals(bothSortedByCustom.toString())) {
+ JsonObject change = new JsonObject(true);
+ change.getProperties().put("warning", JsopBuilder.encode("WARNING: Order is different"));
+ change.getProperties().put("ootb", JsopBuilder.encode(bothSortedByOotb.toString()));
+ change.getProperties().put("custom", JsopBuilder.encode(bothSortedByCustom.toString()));
+ target.getChildren().put(path + "<order>", change);
+ }
+ }
+
+ private static boolean isSameJson(JsonObject a, JsonObject b) {
+ if (a == null || b == null) {
+ return a == null && b == null;
+ }
+ return a.toString().equals(b.toString());
+ }
+
+ private static void addAllProperties(JsonObject source, LinkedHashMap<String, Boolean> target) {
+ for(String k : source.getProperties().keySet()) {
+ target.put(k, true);
+ }
+ }
+
+ private static void removeUnusedIndexes(JsonObject indexDefinitions) {
+ HashMap<String, IndexName> latest = new HashMap<>();
+ Set<String> indexes = new HashSet<>(indexDefinitions.getChildren().keySet());
+ for(String k : indexes) {
+ if (!k.startsWith("/oak:index/")) {
+ continue;
+ }
+ String nodeName = k.substring("/oak:index/".length());
+ IndexName indexName = IndexName.parse(nodeName);
+ String baseName = indexName.getBaseName();
+ IndexName old = latest.get(baseName);
+ if (old == null) {
+ latest.put(baseName, indexName);
+ } else {
+ if (old.compareTo(indexName) < 0) {
+ if (old.getCustomerVersion() > 0) {
+ indexDefinitions.getChildren().remove("/oak:index/" + old.getNodeName());
+ }
+ latest.put(baseName, indexName);
+ } else {
+ indexDefinitions.getChildren().remove("/oak:index/" + nodeName);
+ }
+ }
+ }
+ }
+
+ private static void removeUninterestingIndexProperties(JsonObject indexDefinitions) {
+ for(String k : indexDefinitions.getChildren().keySet()) {
+ JsonObject indexDef = indexDefinitions.getChildren().get(k);
+ indexDef.getProperties().remove("reindexCount");
+ indexDef.getProperties().remove("reindex");
+ indexDef.getProperties().remove("seed");
+ indexDef.getProperties().remove(":version");
+ }
+ }
+
+ private static void simplify(JsonObject json) {
+ for(String k : json.getChildren().keySet()) {
+ JsonObject child = json.getChildren().get(k);
+ simplify(child);
+
+ // the following properties are not strictly needed for display,
+ // but we keep them to avoid issues with validation
+ // child.getProperties().remove("jcr:created");
+ // child.getProperties().remove("jcr:createdBy");
+ // child.getProperties().remove("jcr:lastModified");
+ // child.getProperties().remove("jcr:lastModifiedBy");
+
+ // the UUID we remove, because duplicate UUIDs are not allowed
+ child.getProperties().remove("jcr:uuid");
+ for(String p : child.getProperties().keySet()) {
+ String v = child.getProperties().get(p);
+ if (v.startsWith("\"str:") || v.startsWith("\"nam:")) {
+ v = "\"" + v.substring(5);
+ child.getProperties().put(p, v);
+ } else if (v.startsWith("[") && v.contains("\"nam:")) {
+ v = v.replaceAll("\\[\"nam:", "\\[\"");
+ v = v.replaceAll(", \"nam:", ", \"");
+ child.getProperties().put(p, v);
+ } else if (v.startsWith("\":blobId:")) {
+ String base64 = v.substring(9, v.length() - 1);
+ String clear = new String(java.util.Base64.getDecoder().decode(base64), StandardCharsets.UTF_8);
+ v = JsopBuilder.encode(clear);
+ // we don't update the property, otherwise importing the index
+ // would change the type
+ // child.getProperties().put(p, v);
+ }
+ }
+ }
+ }
+
+ private static JsonObject parseIndexDefinitions(String jsonFileName) {
+ try {
+ String json = new String(Files.readAllBytes(Paths.get(jsonFileName)));
+ return JsonObject.fromJson(json, true);
+ } catch (Exception e) {
+ throw new IllegalStateException("Error parsing file: " + jsonFileName, e);
+ }
+ }
+
+}
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDiffCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDiffCommand.java
new file mode 100644
index 0000000..cc2b326
--- /dev/null
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDiffCommand.java
@@ -0,0 +1,114 @@
+package org.apache.jackrabbit.oak.index.merge;
+
+import static java.util.Arrays.asList;
+
+import org.apache.jackrabbit.oak.run.commons.Command;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+public class IndexDiffCommand implements Command {
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+ OptionSpec<String> customDirOption = parser
+ .accepts("custom", "Path to index definition files (.json). " +
+ "List differences between out-of-the-box and customized indexes").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> mergeDirectoryOption = parser
+ .accepts("merge", "Path to index definition files (.json). " +
+ "This is the existing indexes when merging").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> mergeAddFileOption = parser
+ .accepts("merge-add", "Path to index definition file (.json) to merge. " +
+ "Adds the new out-of-the-box index to each index definition.").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> compareDirectoryOption = parser
+ .accepts("compare", "Path to index definition files (.json). " +
+ "Compare index1 and index2").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> index1Option = parser
+ .accepts("index1", "Index 1").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> index2Option = parser
+ .accepts("index2", "Index 2").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> extractFileOption = parser
+ .accepts("extract", "File from where to extract (an) index definition(s) (.json). " +
+ "This will extract index1, or all indexes").withOptionalArg()
+ .defaultsTo("");
+ OptionSpec<String> targetDirectoryOption = parser
+ .accepts("target", "Target directory where to store results.")
+ .withOptionalArg()
+ .defaultsTo("");
+
+ OptionSpec<?> helpSpec = parser.acceptsAll(
+ asList("h", "?", "help"), "show help").forHelp();
+
+ OptionSet options = parser.parse(args);
+ String customDir = customDirOption.value(options);
+ String mergeDirectory = mergeDirectoryOption.value(options);
+ String compareDirectory = compareDirectoryOption.value(options);
+ String extractFile = extractFileOption.value(options);
+ String targetDirectory = targetDirectoryOption.value(options);
+ if (options.has(helpSpec) || (customDir.isEmpty() &&
+ mergeDirectory.isEmpty() &&
+ compareDirectory.isEmpty() &&
+ extractFile.isEmpty())) {
+ parser.printHelpOn(System.out);
+ return;
+ }
+ if (!customDir.isEmpty()) {
+ System.out.println("Listing differences between out-of-the-box and customized indexes " +
+ "for directory \"" + customDir + "\"");
+ System.out.println(IndexDiff.collectCustomizations(customDir));
+ }
+ if (!mergeDirectory.isEmpty()) {
+ String mergeAdd = mergeAddFileOption.value(options);
+ if (mergeAdd.isEmpty()) {
+ parser.printHelpOn(System.out);
+ return;
+ }
+ if (targetDirectory.isEmpty()) {
+ System.out.println("Merging indexes " +
+ "for directory \"" +
+ mergeDirectory +
+ "\" with \"" + mergeAdd + "\"");
+ System.out.println(IndexDiff.mergeIndexes(mergeDirectory, mergeAdd));
+ } else {
+ System.out.println("Merging index " +
+ "for \"" +
+ mergeDirectory +
+ "\" with \"" + mergeAdd + "\"");
+ IndexDiff.mergeIndex(mergeDirectory, mergeAdd, targetDirectory);
+ }
+ }
+ if (!compareDirectory.isEmpty()) {
+ String index1 = index1Option.value(options);
+ String index2 = index2Option.value(options);
+ if (index1.isEmpty() || index2.isEmpty()) {
+ parser.printHelpOn(System.out);
+ return;
+ }
+ System.out.println("Comparing indexes " + index1 + " and " + index2 +
+ " for directory \"" +
+ compareDirectory + "\"");
+ System.out.println(IndexDiff.compareIndexes(compareDirectory, index1, index2));
+ }
+ if (!extractFile.isEmpty()) {
+ String index1 = index1Option.value(options);
+ if (!index1.isEmpty()) {
+ System.out.println("Extracting index " + index1 + " from \"" +
+ extractFile + "\"");
+ System.out.println(IndexDiff.extract(extractFile, index1));
+ } else if (!targetDirectory.isEmpty()) {
+ System.out.println("Extracting indexes to \"" + targetDirectory + "\" from \"" +
+ extractFile + "\"");
+ IndexDiff.extractAll(extractFile, targetDirectory);
+ }
+ }
+ }
+
+}
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
index 76d608e..936774c 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
@@ -22,6 +22,7 @@
import com.google.common.collect.ImmutableMap;
import org.apache.jackrabbit.oak.exporter.NodeStateExportCommand;
import org.apache.jackrabbit.oak.index.IndexCommand;
+import org.apache.jackrabbit.oak.index.merge.IndexDiffCommand;
import org.apache.jackrabbit.oak.run.commons.Command;
import org.apache.jackrabbit.oak.run.commons.Modes;
@@ -48,6 +49,7 @@
.put("help", new HelpCommand())
.put("history", new HistoryCommand())
.put("index-merge", new IndexMergeCommand())
+ .put("index-diff", new IndexDiffCommand())
.put(IndexCommand.NAME, new IndexCommand())
.put(IOTraceCommand.NAME, new IOTraceCommand())
.put(JsonIndexCommand.INDEX, new JsonIndexCommand())
diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java
index b3009e1..fcf74bc 100644
--- a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java
+++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java
@@ -68,7 +68,10 @@
JsonObject custom = e.getChildren().get("custom");
JsonObject product = e.getChildren().get("product");
try {
- JsonObject got = IndexDefMergerUtils.merge(ancestor, custom, product);
+ JsonObject got = IndexDefMergerUtils.merge(
+ "/oak:index/test-1", ancestor,
+ "/oak:index/test-1-custom-1", custom,
+ product);
JsonObject expected = e.getChildren().get("expected");
assertEquals(expected.toString(), got.toString());
} catch (UnsupportedOperationException e2) {
diff --git a/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt b/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt
index 489d515..b8e3618 100644
--- a/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt
+++ b/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt
@@ -4,13 +4,14 @@
"ancestor": {"value": 1, "a-old": 0},
"custom": {"value": 2, "b-new": 3},
"product": {"value": 1, "c-new": 4},
- "expected": {"value": 2, "b-new": 3, "c-new": 4}
+ "expected": {"value": 2, "b-new": 3, "c-new": 4, "merges": ["/oak:index/test-1", "/oak:index/test-1-custom-1"]
+ }
},
{
"ancestor": {"o": {"a": 1}},
"custom": {"o": {"a": 2, "c": 10}},
"product": {"o": {"a": 1, "p": 20}},
- "expected": {"o": {"a": 2, "c": 10, "p": 20}}
+ "expected": {"merges": ["/oak:index/test-1", "/oak:index/test-1-custom-1"], "o": {"a": 2, "c": 10, "p": 20 }}
},
{
"ancestor": {"o": {"a": 1}},
@@ -70,7 +71,7 @@
, "custom": {
-// changed: facetc
+// changed: facets
"jcr:primaryType": "nam:oak:QueryIndexDefinition",
"compatVersion": 2,
"includedPaths": ["/content/dam"],
@@ -195,7 +196,8 @@
}
}
}
- }
+ },
+ "merges": ["/oak:index/test-1", "/oak:index/test-1-custom-1"]
}
diff --git a/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt b/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt
index 94fe60c..20ed5f0 100644
--- a/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt
+++ b/oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt
@@ -10,7 +10,7 @@
},
"expectedNew": {
"/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 },
- "/oak:index/lucene-2-custom-1": { "a": 2, "b": 11, "c": 1, "d": 100 }
+ "/oak:index/lucene-2-custom-1": { "a": 2, "b": 11, "c": 1, "d": 100, "merges": ["/oak:index/lucene", "/oak:index/lucene-custom-1"] }
}
}
]
