[VXQUERY-196][VXQUERY-204][VXQUERY-228] Cleaning up indexing query statements
1) Move XPath out of arguments for collection-from-index, leave only the collection path.
2) Hide the index usage from the user by creating a rewrite rule to figure out the existence of index.
3) Remove the unnecessary header tags that collection-from-index creates.
diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
index e0e3843..25ff9c4 100644
--- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
+++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
@@ -14,6 +14,7 @@
*/
package org.apache.vxquery.cli;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -23,6 +24,7 @@
import java.net.InetAddress;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
@@ -66,12 +68,13 @@
public class VXQuery {
private final CmdLineOptions opts;
+ private final CmdLineOptions indexOpts;
private ClusterControllerService cc;
private NodeControllerService[] ncs;
private IHyracksClientConnection hcc;
private IHyracksDataset hds;
-
+ private List<String> collectionList;
private ResultSetId resultSetId;
private static List<String> timingMessages = new ArrayList<>();
private static long sumTiming;
@@ -87,6 +90,16 @@
*/
public VXQuery(CmdLineOptions opts) {
this.opts = opts;
+ // The index query returns only the result, without any other information.
+ this.indexOpts = opts;
+ indexOpts.showAST = false;
+ indexOpts.showOET = false;
+ indexOpts.showQuery = false;
+ indexOpts.showRP = false;
+ indexOpts.showTET = false;
+ indexOpts.timing = false;
+ indexOpts.compileOnly = false;
+ this.collectionList = new ArrayList<String>();
}
/**
@@ -168,71 +181,87 @@
* @throws SystemException
* @throws Exception
*/
+
private void runQueries() throws Exception {
- Date start;
- Date end;
- for (String query : opts.arguments) {
- String qStr = slurp(query);
- if (opts.showQuery) {
- System.err.println(qStr);
- }
+ List<String> queries = opts.arguments;
+ // Run the showIndexes query before executing any target query, to store the index metadata
+ List<String> queriesIndex = new ArrayList<String>();
+ queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq");
+ OutputStream resultStream = new ByteArrayOutputStream();
+ executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts);
+ ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream;
+ String result = new String(bos.toByteArray());
+ String[] collections = result.split("\n");
+ this.collectionList = Arrays.asList(collections);
+ executeQueries(queries);
+ }
- VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET,
- opts.showOET, opts.showRP);
-
- start = opts.timing ? new Date() : null;
-
- Map<String, NodeControllerInfo> nodeControllerInfos = null;
- if (hcc != null) {
- nodeControllerInfos = hcc.getNodeControllerInfos();
- }
- XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
- opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
- resultSetId = createResultSetId();
- CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
- resultSetId, null);
- compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel);
- // if -timing argument passed, show the starting and ending times
- if (opts.timing) {
- end = new Date();
- timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
- }
- if (opts.compileOnly) {
- continue;
- }
-
- Module module = compiler.getModule();
- JobSpecification js = module.getHyracksJobSpecification();
-
- DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
- js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
-
+ public void executeQueries(List<String> queries) throws Exception {
+ for (String query : queries) {
OutputStream resultStream = System.out;
if (opts.resultFile != null) {
resultStream = new FileOutputStream(new File(opts.resultFile));
}
+ executeQuery(query, opts.repeatExec, resultStream, opts);
+ }
+ }
- PrintWriter writer = new PrintWriter(resultStream, true);
- // Repeat execution for number of times provided in -repeatexec argument
- for (int i = 0; i < opts.repeatExec; ++i) {
- start = opts.timing ? new Date() : null;
- runJob(js, writer);
- // if -timing argument passed, show the starting and ending times
- if (opts.timing) {
- end = new Date();
- long currentRun = end.getTime() - start.getTime();
- if ((i + 1) > opts.timingIgnoreQueries) {
- sumTiming += currentRun;
- sumSquaredTiming += currentRun * currentRun;
- if (currentRun < minTiming) {
- minTiming = currentRun;
- }
- if (maxTiming < currentRun) {
- maxTiming = currentRun;
- }
+ public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options)
+ throws Exception {
+ PrintWriter writer = new PrintWriter(resultStream, true);
+ String qStr = slurp(query);
+ if (opts.showQuery) {
+ writer.println(qStr);
+ }
+ VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET,
+ opts.showRP);
+
+ Date start = opts.timing ? new Date() : null;
+
+ Map<String, NodeControllerInfo> nodeControllerInfos = null;
+ if (hcc != null) {
+ nodeControllerInfos = hcc.getNodeControllerInfos();
+ }
+ XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
+ opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
+ resultSetId = createResultSetId();
+ CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
+ resultSetId, null);
+ compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList);
+ // if -timing argument passed, show the starting and ending times
+ Date end = opts.timing ? new Date() : null;
+ if (opts.timing) {
+ timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
+ }
+ if (opts.compileOnly) {
+ return;
+ }
+
+ Module module = compiler.getModule();
+ JobSpecification js = module.getHyracksJobSpecification();
+
+ DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
+ js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
+
+ // Repeat execution for number of times provided in -repeatexec argument
+ for (int i = 0; i < repeatedExecution; ++i) {
+ start = opts.timing ? new Date() : null;
+ runJob(js, writer);
+ // if -timing argument passed, show the starting and ending times
+ if (opts.timing) {
+ end = new Date();
+ long currentRun = end.getTime() - start.getTime();
+ if ((i + 1) > opts.timingIgnoreQueries) {
+ sumTiming += currentRun;
+ sumSquaredTiming += currentRun * currentRun;
+ if (currentRun < minTiming) {
+ minTiming = currentRun;
}
- timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
+ if (maxTiming < currentRun) {
+ maxTiming = currentRun;
+ }
}
+ timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
}
}
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
index ceaf3c7..400fb15 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java
@@ -35,7 +35,8 @@
static {
indexingFunctions.add(BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier());
- indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier());
+ indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier());
indexingFunctions.add(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier());
indexingFunctions.add(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier());
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
index 20283d8..11d3795 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java
@@ -16,12 +16,16 @@
*/
package org.apache.vxquery.compiler.rewriter.rules;
+import java.util.ArrayList;
+
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.vxquery.common.VXQueryCommons;
import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext;
import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
+import org.apache.vxquery.metadata.VXQueryIndexingDataSource;
+import org.apache.vxquery.metadata.VXQueryMetadataProvider;
import org.apache.vxquery.types.AnyItemType;
import org.apache.vxquery.types.Quantifier;
import org.apache.vxquery.types.SequenceType;
@@ -61,11 +65,35 @@
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
String[] args = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions);
-
+ VXQueryMetadataProvider metadata = (VXQueryMetadataProvider) context.getMetadataProvider();
if (args != null) {
String collectionName = args[0];
// Build the new operator and update the query plan.
int collectionId = vxqueryContext.newCollectionId();
+ ArrayList<String> collectionTempName = new ArrayList<String>();
+ collectionTempName.add(collectionName);
+ if (collectionName.contains("|")) {
+ collectionTempName.remove(0);
+ int index = collectionName.indexOf("|");
+ int start = 0;
+ while (index >= 0) {
+ collectionTempName.add(collectionName.substring(start, index));
+ start = index + 1;
+ index = collectionName.indexOf("|", index + 1);
+ if (index == -1) {
+ collectionTempName.add(collectionName.substring(start));
+ }
+ }
+ }
+ if (metadata.hasIndex(collectionTempName)) {
+ VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collectionName,
+ SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR),
+ functionCall.getFunctionIdentifier().getName());
+ if (ids != null) {
+ ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
+ return setDataSourceScan(ids, opRef);
+ }
+ }
VXQueryCollectionDataSource ds = VXQueryCollectionDataSource.create(collectionId, collectionName,
SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR));
if (ds != null) {
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
index 5b96131..6e60d75 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java
@@ -33,19 +33,19 @@
public class IntroduceIndexingRule extends AbstractCollectionRule {
@Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context;
String args[] = getFunctionalArguments(opRef, VXQueryCommons.indexingFunctions);
if (args != null) {
String collection = args[0];
- String elementPath = args.length > 1?args[1]:null;
-
// Build the new operator and update the query plan.
int collectionId = vxqueryContext.newCollectionId();
- VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, elementPath,
- SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), functionCall.getFunctionIdentifier().getName());
+ VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection,
+ SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR),
+ functionCall.getFunctionIdentifier().getName());
if (ids != null) {
ids.setTotalDataSources(vxqueryContext.getTotalDataSources());
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
index dbcce54..2773154 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java
@@ -55,12 +55,9 @@
@Override
boolean updateDataSource(IVXQueryDataSource datasource, Mutable<ILogicalExpression> expression) {
- //TODO: indexing needs to be extended to support push child into datascan
- if (datasource.usingIndex()) {
- return false;
- }
- boolean added = false;
List<Mutable<ILogicalExpression>> finds = new ArrayList<Mutable<ILogicalExpression>>();
+ boolean added = false;
+
ExpressionToolbox.findAllFunctionExpressions(expression, BuiltinOperators.CHILD.getFunctionIdentifier(), finds);
for (int i = finds.size(); i > 0; --i) {
int typeId = ExpressionToolbox.getTypeExpressionTypeArgument(finds.get(i - 1));
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
index 1d8a55d..b901469 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java
@@ -59,6 +59,9 @@
@Override
boolean updateDataSource(IVXQueryDataSource datasource, Mutable<ILogicalExpression> expression) {
+ if (datasource.usingIndex()) {
+ return false;
+ }
VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) datasource;
boolean added = false;
List<Mutable<ILogicalExpression>> finds = new ArrayList<Mutable<ILogicalExpression>>();
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
index d64f423..4932a78 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
+++ b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml
@@ -152,7 +152,6 @@
<!-- fn:collection-from-index($indexfolder as xs:string?, $elementpath as xs:string?) as node()* -->
<function name="fn:collection-from-index">
<param name="index-folder" type="xs:string?"/>
- <param name="element-path" type="xs:string?"/>
<return type="node()*"/>
<property type="DocumentOrder" class="org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.InputPropertyPropagationPolicy">
<argument value="0"/>
@@ -166,7 +165,7 @@
<!-- fn:show-indexes as node()* -->
<function name="fn:show-indexes">
<return type="node()*"/>
- <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexScalarEvaluatorFactory"/>
+ <runtime type="scalar" class="org.apache.vxquery.runtime.functions.index.ShowIndexesScalarEvaluatorFactory"/>
</function>
<!-- fn:collection-with-tag($arg1 as xs:string?, $arg2 as xs:string?) as node()* -->
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
index 7524da4..1df31dd 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java
@@ -22,7 +22,6 @@
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.BooleanPointable;
import org.apache.hyracks.data.std.primitive.BytePointable;
import org.apache.hyracks.data.std.primitive.DoublePointable;
@@ -56,12 +55,10 @@
import org.apache.vxquery.datamodel.accessors.nodes.TextOrCommentNodePointable;
import org.apache.vxquery.datamodel.values.ValueTag;
import org.apache.vxquery.runtime.functions.cast.CastToStringOperation;
-import org.apache.vxquery.runtime.functions.index.updateIndex.Constants;
+import org.apache.vxquery.runtime.functions.index.update.Constants;
import org.apache.vxquery.serializer.XMLSerializer;
public class IndexDocumentBuilder extends XMLSerializer {
- private final IPointable treePointable;
-
private final PointablePool pp;
private NodeTreePointable ntp;
@@ -88,16 +85,11 @@
}
//TODO: Handle Processing Instructions, PrefixedNames, and Namepsace entries
- public IndexDocumentBuilder(IPointable tree, IndexWriter inWriter, String file) {
- this.treePointable = tree;
+ public IndexDocumentBuilder(TaggedValuePointable tvp, IndexWriter inWriter, String file) {
writer = inWriter;
this.filePath = file;
- //convert to tagged value pointable
- TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
- tvp.set(treePointable.getByteArray(), 0, treePointable.getLength());
-
//get bytes and info from doc pointer
bstart = tvp.getByteArray();
sstart = tvp.getStartOffset();
@@ -105,7 +97,7 @@
doc = new Document();
- results = new ArrayList<ComplexItem>();
+ results = new ArrayList<>();
pp = PointablePoolFactory.INSTANCE.createPointablePool();
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
index 623b48c..a3756d5 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java
@@ -59,7 +59,6 @@
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.hdfs.ContextFactory;
@@ -129,38 +128,14 @@
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
fta.reset(buffer);
String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
- Reader input;
+
if (!collectionModifiedName.contains("hdfs:/")) {
File collectionDirectory = new File(collectionModifiedName);
// check if directory is in the local file system
if (collectionDirectory.exists()) {
// Go through each tuple.
if (collectionDirectory.isDirectory()) {
- for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
- Iterator<File> it = FileUtils.iterateFiles(collectionDirectory,
- new VXQueryIOFileFilter(), TrueFileFilter.INSTANCE);
- while (it.hasNext()) {
- File file = it.next();
- String fileName = file.getName().toLowerCase();
- if (fileName.endsWith(".xml")) {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath());
- }
- parser.parseElements(file, writer, tupleIndex);
- } else if (fileName.endsWith(".json")) {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath());
- }
- try {
- jsonAbvs.reset();
- input = new InputStreamReader(new FileInputStream(file));
- jparser.parse(input, jsonAbvs, writer, appender);
- } catch (FileNotFoundException e) {
- throw new HyracksDataException(e.toString());
- }
- }
- }
- }
+ xmlAndJsonCollection(collectionDirectory);
} else {
throw new HyracksDataException("Invalid directory parameter (" + nodeId + ":"
+ collectionDirectory.getAbsolutePath() + ") passed to collection.");
@@ -272,6 +247,35 @@
}
}
+ public void xmlAndJsonCollection(File directory) throws HyracksDataException {
+ Reader input;
+ for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
+ Iterator<File> it = FileUtils.iterateFiles(directory, new VXQueryIOFileFilter(),
+ TrueFileFilter.INSTANCE);
+ while (it.hasNext()) {
+ File file = it.next();
+ String fileName = file.getName().toLowerCase();
+ if (fileName.endsWith(".xml")) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath());
+ }
+ parser.parseElements(file, writer, tupleIndex);
+ } else if (fileName.endsWith(".json")) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath());
+ }
+ try {
+ jsonAbvs.reset();
+ input = new InputStreamReader(new FileInputStream(file));
+ jparser.parse(input, jsonAbvs, writer, appender);
+ } catch (FileNotFoundException e) {
+ throw new HyracksDataException(e.toString());
+ }
+ }
+ }
+ }
+ }
+
@Override
public void fail() throws HyracksDataException {
writer.fail();
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
index ea69cfd..d55530d 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java
@@ -31,14 +31,11 @@
*/
public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource {
- private String elementPath;
private String function;
- private VXQueryIndexingDataSource(int id, String collection, String elementPath, Object[] types,
- String functionCall) {
+ private VXQueryIndexingDataSource(int id, String collection, Object[] types, String functionCall) {
this.dataSourceId = id;
this.collectionName = collection;
- this.elementPath = elementPath;
this.function = functionCall;
this.collectionPartitions = collectionName.split(DELIMITER);
this.types = types;
@@ -56,13 +53,8 @@
this.valueSeq = new ArrayList<>();
}
- public static VXQueryIndexingDataSource create(int id, String collection, String index, Object type,
- String function) {
- return new VXQueryIndexingDataSource(id, collection, index, new Object[] { type }, function);
- }
-
- public String getElementPath() {
- return elementPath;
+ public static VXQueryIndexingDataSource create(int id, String collection, Object type, String function) {
+ return new VXQueryIndexingDataSource(id, collection, new Object[] { type }, function);
}
public String getFunctionCall() {
@@ -71,7 +63,7 @@
@Override
public String toString() {
- return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + elementPath
+ return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + this.childSeq
+ ", function=" + function + "]";
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
index ac92a0e..3563713 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java
@@ -16,15 +16,13 @@
*/
package org.apache.vxquery.metadata;
-import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Map;
+import java.util.List;
import java.util.logging.Logger;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameFieldAppender;
import org.apache.hyracks.api.comm.VSizeFrame;
@@ -35,23 +33,20 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
import org.apache.vxquery.datamodel.values.XDMConstants;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.functions.BuiltinFunctions;
import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
import org.apache.vxquery.runtime.functions.index.VXQueryIndexReader;
-import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater;
+import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.update.IndexUpdater;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
@@ -61,18 +56,18 @@
private short dataSourceId;
private short totalDataSources;
private String[] collectionPartitions;
- private String elementPath;
private final String functionCall;
+ private List<Integer> childSeq;
public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQueryIndexingDataSource ds,
- RecordDescriptor rDesc, String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+ RecordDescriptor rDesc) {
super(spec, 1, 1);
this.functionCall = ds.getFunctionCall();
collectionPartitions = ds.getPartitions();
dataSourceId = (short) ds.getDataSourceId();
totalDataSources = (short) ds.getTotalDataSources();
recordDescriptors[0] = rDesc;
- this.elementPath = ds.getElementPath();
+ childSeq = ds.getChildSeq();
}
@Override
@@ -87,10 +82,11 @@
final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources);
final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
final String collectionName = collectionPartitions[partition % collectionPartitions.length];
- String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
+ final String collectionModifiedName = collectionName.replace("${nodeId}", nodeId);
IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil(
ctx.getIOManager().getIODevices().get(0).getMount());
indexCentralizerUtil.readIndexDirectory();
+ final IPointable result = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
@Override
@@ -103,100 +99,115 @@
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
fta.reset(buffer);
- IPointable result = new TaggedValuePointable();
-
- final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
- final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY
- .createPointable();
-
- final ByteBufferInputStream bbis = new ByteBufferInputStream();
- final DataInputStream di = new DataInputStream(bbis);
- final SequenceBuilder sb = new SequenceBuilder();
final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage();
- String indexModifiedName;
+ abvs.reset();
+ abvsFileNode.reset();
+
if (collectionModifiedName.contains("hdfs://")) {
throw new HyracksDataException("Indexing support for HDFS not yet implemented.");
} else {
if (functionCall.equals(
BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
- File collectionDirectory = new File(collectionModifiedName);
-
- //check if directory is in the local file system
- if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
- IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
- try {
- indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result,
- stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode, nodep, false,
- nodeId);
- XDMConstants.setTrue(result);
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- } catch (SystemException e) {
- throw new HyracksDataException("Could not create index for collection: "
- + collectionName + " in dir: " + indexModifiedName + " " + e.getMessage());
- }
- } else {
- throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
- + collectionDirectory.getAbsolutePath() + ")");
+ try {
+ createIndex(result, abvs, abvsFileNode);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
}
} else if (functionCall
.equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
- IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
- nodeIdProvider, abvsFileNode, nodep, nodeId);
- try {
- updater.setup();
- updater.updateIndex();
- updater.updateMetadataFile();
- updater.exit();
- XDMConstants.setTrue(result);
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- } catch (IOException e) {
- throw new HyracksDataException(
- "Could not update index in " + indexModifiedName + " " + e.getMessage());
- }
+ updateIndex(result, abvs, abvsFileNode);
} else if (functionCall
.equals(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
- IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs,
- nodeIdProvider, abvsFileNode, nodep, nodeId);
- indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
- try {
- updater.setup();
- updater.deleteAllIndexes();
- XDMConstants.setTrue(result);
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- } catch (IOException e) {
- throw new HyracksDataException(
- "Could not delete index in " + indexModifiedName + " " + e.getMessage());
- }
-
+ deleteIndex(result, abvs, abvsFileNode);
} else if (functionCall
- .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier().getName())) {
- indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
- VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, elementPath);
- try {
- indexReader.init();
- while (indexReader.step(result)) {
- FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(),
- result.getStartOffset(), result.getLength());
- }
- } catch (AlgebricksException e) {
- throw new HyracksDataException("Could not read index.");
- }
-
+ .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier().getName())
+ || functionCall
+ .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier().getName())) {
+ usingIndex(result);
} else {
throw new HyracksDataException("Unsupported function call (" + functionCall + ")");
}
}
}
+ public void createIndex(IPointable result, ArrayBackedValueStorage abvs,
+ ArrayBackedValueStorage abvsFileNode) throws IOException {
+ String indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName);
+ File collectionDirectory = new File(collectionModifiedName);
+
+ //check if directory is in the local file system
+ if (collectionDirectory.exists() && collectionDirectory.isDirectory()) {
+ IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
+ try {
+ indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result, abvs,
+ nodeIdProvider, abvsFileNode, false, nodeId);
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+ result.getLength());
+ } catch (SystemException e) {
+ throw new HyracksDataException("Could not create index for collection: " + collectionName
+ + " in dir: " + indexModifiedName + " " + e.getMessage(), e);
+ }
+ } else {
+ throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":"
+ + collectionDirectory.getAbsolutePath() + ")");
+ }
+ }
+
+ public void updateIndex(IPointable result, ArrayBackedValueStorage abvs,
+ ArrayBackedValueStorage abvsFileNode) throws HyracksDataException {
+ String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode,
+ nodeId);
+ try {
+ updater.setup();
+ updater.updateIndex();
+ updater.updateMetadataFile();
+ updater.exit();
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+ result.getLength());
+ } catch (IOException e) {
+ throw new HyracksDataException(
+ "Could not update index in " + indexModifiedName + " " + e.getMessage(), e);
+ }
+ }
+
+ public void deleteIndex(IPointable result, ArrayBackedValueStorage abvs,
+ ArrayBackedValueStorage abvsFileNode) throws HyracksDataException {
+ String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode,
+ nodeId);
+ indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName);
+ try {
+ updater.setup();
+ updater.deleteAllIndexes();
+ XDMConstants.setTrue(result);
+ FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(),
+ result.getLength());
+ } catch (IOException e) {
+ throw new HyracksDataException(
+ "Could not delete index in " + indexModifiedName + " " + e.getMessage(), e);
+ }
+ }
+
+ public void usingIndex(IPointable result) throws HyracksDataException {
+ String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName);
+ VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, childSeq, appender);
+ try {
+ indexReader.init();
+ for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) {
+ while (indexReader.step(result, writer, tupleIndex)) {
+ }
+ }
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException("Could not read index.", e);
+ }
+ }
+
@Override
public void fail() throws HyracksDataException {
writer.fail();
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
index f6644d6..5bb9d1a 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
@@ -59,15 +59,17 @@
private final Map<String, File> sourceFileMap;
private final StaticContext staticCtx;
private final String hdfsConf;
+ private final List<String> collections;
private final Map<String, NodeControllerInfo> nodeControllerInfos;
public VXQueryMetadataProvider(String[] nodeList, Map<String, File> sourceFileMap, StaticContext staticCtx,
- String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos) {
+ String hdfsConf, Map<String, NodeControllerInfo> nodeControllerInfos, List<String> collections) {
this.nodeList = nodeList;
this.sourceFileMap = sourceFileMap;
this.staticCtx = staticCtx;
this.hdfsConf = hdfsConf;
this.nodeControllerInfos = nodeControllerInfos;
+ this.collections = collections;
}
@Override
@@ -111,8 +113,7 @@
constraint = getClusterLocations(nodeList, ds.getPartitionCount());
} else {
rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
- scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc,
- this.hdfsConf, this.nodeControllerInfos);
+ scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc);
constraint = getClusterLocations(nodeList, ds.getPartitionCount());
}
@@ -142,7 +143,7 @@
@Override
public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
- throws AlgebricksException {
+ throws AlgebricksException {
throw new UnsupportedOperationException();
}
@@ -168,7 +169,7 @@
IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
- throws AlgebricksException {
+ throws AlgebricksException {
throw new UnsupportedOperationException();
}
@@ -234,7 +235,7 @@
JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
@@ -243,7 +244,7 @@
JobSpecification jobSpec, boolean bulkload) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getUpsertRuntime(IDataSource<String> dataSource,
IOperatorSchema inputSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
@@ -252,7 +253,7 @@
JobSpecification jobSpec) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexUpsertRuntime(
IDataSourceIndex<String, String> dataSourceIndex, IOperatorSchema propagatedSchema,
@@ -263,11 +264,26 @@
JobSpecification spec) throws AlgebricksException {
throw new UnsupportedOperationException();
}
-
+
@Override
public Map<String, String> getConfig() {
return new HashMap<>();
}
+ public List<String> getIndexCollections() {
+ return collections;
+
+ }
+
+ public boolean hasIndex(ArrayList<String> collections) {
+ boolean indexExists = false;
+ for (String collection : collections) {
+ indexExists = getIndexCollections().contains(collection);
+ if (!indexExists) {
+ break;
+ }
+ }
+ return indexExists;
+ }
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
index 4706496..2c25752 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java
@@ -16,10 +16,14 @@
*/
package org.apache.vxquery.runtime.functions.index;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.concurrent.ConcurrentHashMap;
+
import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -31,38 +35,31 @@
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexDocumentBuilder;
-import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
+import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
import org.apache.vxquery.runtime.functions.util.FunctionHelper;
import org.apache.vxquery.xmlparser.IParser;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.XMLParser;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
-import java.util.concurrent.ConcurrentHashMap;
-
public class IndexConstructorUtil {
- boolean isMetaFilePresent = false;
- MetaFileUtil metaFileUtil;
- ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
+ private final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+ private final SequenceBuilder sb = new SequenceBuilder();
+ private boolean isMetaFilePresent = false;
+ private MetaFileUtil metaFileUtil;
+ private ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>();
- public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable
- stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs,
- ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep,
- boolean isElementPath, String nodeId) throws SystemException {
+ public void evaluate(String collectioFolder, String indexFolder, IPointable result, ArrayBackedValueStorage abvs,
+ ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, boolean isElementPath,
+ String nodeId) throws IOException {
- metaFileUtil = new MetaFileUtil(indexFolder);
-// metaFileUtil = .create(indexFolder);
- isMetaFilePresent = metaFileUtil.isMetaFilePresent();
- metaFileUtil.setCollection(collectioFolder);
+ metaFileUtil = new MetaFileUtil(indexFolder);
+ isMetaFilePresent = metaFileUtil.isMetaFilePresent();
+ metaFileUtil.setCollection(collectioFolder);
File collectionDirectory = new File(collectioFolder);
if (!collectionDirectory.exists()) {
- throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist.");
+ throw new IOException("The collection directory (" + collectioFolder + ") does not exist.");
}
try {
@@ -80,8 +77,7 @@
IndexWriter writer = new IndexWriter(dir, iwc);
//Add files to index
- indexXmlFiles(collectionDirectory, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di,
- nodeId);
+ indexXmlFiles(collectionDirectory, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId);
if (!isMetaFilePresent) {
// Write metadata map to a file.
@@ -101,14 +97,13 @@
}
}
- /*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
+ /*
+ * This function goes recursively one file at a time. First it turns the file into an ABVS document node, then
* it indexes that document node.
*/
public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath,
- TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider,
- SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId)
- throws SystemException, IOException {
-
+ ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, SequenceBuilder sb, String nodeId)
+ throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy, HH:mm:ss");
for (File file : collectionDirectory.listFiles()) {
@@ -116,8 +111,7 @@
if (readableXmlFile(file.getPath())) {
abvsFileNode.reset();
- IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, nodep, abvsFileNode, nodeIdProvider, bbis,
- di, nodeId);
+ IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, abvsFileNode, nodeIdProvider, nodeId);
ibuilder.printStart();
if (!isMetaFilePresent) {
@@ -131,22 +125,21 @@
} else if (file.isDirectory()) {
// Consider all XML file in sub directories.
- indexXmlFiles(file, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di, nodeId);
+ indexXmlFiles(file, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId);
}
}
}
public boolean readableXmlFile(String path) {
- return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"));
+ return path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz");
}
- public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep,
- ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis,
- DataInputStream di, String nodeId) throws IOException {
+ public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, ArrayBackedValueStorage abvsFileNode,
+ ITreeNodeIdProvider nodeIdProvider, String nodeId) throws IOException {
//Get the document node
IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
- FunctionHelper.readInDocFromString(file.getPath(), bbis, di, abvsFileNode, parser);
+ FunctionHelper.readInDocFromString(file.getPath(), abvsFileNode, parser);
nodep.set(abvsFileNode.getByteArray(), abvsFileNode.getStartOffset(), abvsFileNode.getLength());
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java
similarity index 91%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java
index 6677bd9..6b18b33 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java
@@ -30,12 +30,12 @@
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
-import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil;
+import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil;
-public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+public class ShowIndexesScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
private static final long serialVersionUID = 1L;
- public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ public ShowIndexesScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) {
super(args);
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
index 8750849..cf781ab 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java
@@ -16,11 +16,17 @@
*/
package org.apache.vxquery.runtime.functions.index;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.comm.IFrameFieldAppender;
+import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
@@ -32,21 +38,21 @@
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.context.DynamicContext;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
import org.apache.vxquery.index.IndexAttributes;
+import org.apache.vxquery.runtime.functions.util.FunctionHelper;
+import org.apache.vxquery.types.ElementType;
+import org.apache.vxquery.types.NameTest;
+import org.apache.vxquery.types.NodeType;
+import org.apache.vxquery.types.SequenceType;
import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
import org.apache.vxquery.xmlparser.SAXContentHandler;
import org.apache.vxquery.xmlparser.TreeNodeIdProvider;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
public class VXQueryIndexReader {
private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage();
@@ -55,10 +61,7 @@
private int indexLength;
private String elementPath;
private String indexName;
-
- private ByteBufferInputStream bbis = new ByteBufferInputStream();
- private DataInputStream di = new DataInputStream(bbis);
-
+ private List<SequenceType> childSequenceTypes;
private IndexReader reader;
private IndexSearcher searcher;
private QueryParser parser;
@@ -68,14 +71,40 @@
private Document doc;
private List<IndexableField> fields;
private IHyracksTaskContext ctx;
+ private String[] childLocalName = null;
+ private IFrameFieldAppender appender;
+ private boolean firstElement;
- public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) {
+ public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, List<Integer> childSeq,
+ IFrameFieldAppender appender) {
this.ctx = context;
this.indexName = indexPath;
- this.elementPath = elementPath;
+ this.appender = appender;
+ final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData();
+ childSequenceTypes = new ArrayList<>();
+ for (int typeCode : childSeq) {
+ childSequenceTypes.add(dCtx.getStaticContext().lookupSequenceType(typeCode));
+ }
+ childLocalName = new String[childSequenceTypes.size()];
+ int index = 0;
+ StringBuilder stb = new StringBuilder();
+ stb.append("/");
+ for (SequenceType sType : childSequenceTypes) {
+ NodeType nodeType = (NodeType) sType.getItemType();
+ ElementType eType = (ElementType) nodeType;
+ NameTest nameTest = eType.getNameTest();
+ childLocalName[index] = FunctionHelper.getStringFromBytes(nameTest.getLocalName());
+
+ stb.append(childLocalName[index]);
+ if (index != childSequenceTypes.size() - 1) {
+ stb.append("/");
+ }
+ ++index;
+ }
+ elementPath = stb.toString();
}
- public boolean step(IPointable result) throws AlgebricksException {
+ public boolean step(IPointable result, IFrameWriter writer, int tupleIndex) throws AlgebricksException {
/*each step will create a tuple for a single xml file
* This is done using the parse function
* checkoverflow is used throughout. This is because memory might not be
@@ -88,6 +117,8 @@
//TODO: now we get back the entire document
doc = searcher.doc(hits[indexPlace].doc);
fields = doc.getFields();
+ handler.setupElementWriter(writer, tupleIndex);
+ this.firstElement = true;
parse(nodeAbvs);
} catch (IOException e) {
throw new AlgebricksException(e);
@@ -103,7 +134,7 @@
int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition);
- handler = new SAXContentHandler(false, nodeIdProvider, true);
+ handler = new SAXContentHandler(false, nodeIdProvider, appender, childSequenceTypes);
nodeAbvs.reset();
indexPlace = 0;
@@ -125,7 +156,7 @@
String queryString = elementPath.replaceAll("/", ".");
queryString = "item:" + queryString + "*";
- int lastslash = elementPath.lastIndexOf("/");
+ int lastslash = elementPath.lastIndexOf('/');
elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1);
elementPath = elementPath.replaceAll("/", ".") + ".element";
@@ -135,31 +166,25 @@
//TODO: Right now it only returns 1000000 results
results = searcher.search(query, 1000000);
-
} catch (Exception e) {
- throw new SystemException(null);
+ throw new SystemException(null, e);
}
hits = results.scoreDocs;
- System.out.println("found: " + results.totalHits);
indexPlace = 0;
indexLength = hits.length;
-
}
public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException {
try {
- handler.startDocument();
-
for (int i = 0; i < fields.size(); i++) {
String fieldValue = fields.get(i).stringValue();
if (fieldValue.equals(elementPath)) {
+ handler.startDocument();
+ this.firstElement = true;
buildElement(abvsFileNode, i);
}
}
-
- handler.endDocument();
- handler.writeDocument(abvsFileNode);
} catch (Exception e) {
throw new IOException(e);
}
@@ -167,6 +192,7 @@
private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException {
int whereIFinish = fieldNum;
+ int firstFinish;
IndexableField field = fields.get(fieldNum);
String contents = field.stringValue();
String uri = "";
@@ -176,18 +202,37 @@
String type = contents.substring(lastDot + 1);
String lastBit = contents.substring(firstColon + 1, lastDot);
- if (type.equals("textnode")) {
+ if (this.firstElement) {
+ this.firstElement = false;
+ firstFinish = whereIFinish - this.childSequenceTypes.size() + 1;
+ String firstBit = contents.substring(1, firstColon);
+ List<String> names = new ArrayList<>();
+ List<String> values = new ArrayList<>();
+ List<String> uris = new ArrayList<>();
+ List<String> localNames = new ArrayList<>();
+ List<String> types = new ArrayList<>();
+ List<String> qNames = new ArrayList<>();
+ firstFinish = findAttributeChildren(firstFinish, names, values, uris, localNames, types, qNames);
+ Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
+
+ handler.startElement(uri, firstBit, firstBit, atts);
+ buildElement(abvsFileNode, firstFinish + 1);
+ handler.endElement(uri, firstBit, firstBit);
+
+ }
+
+ if ("textnode".equals(type)) {
char[] charContents = lastBit.toCharArray();
handler.characters(charContents, 0, charContents.length);
}
- if (type.equals("element")) {
- List<String> names = new ArrayList<String>();
- List<String> values = new ArrayList<String>();
- List<String> uris = new ArrayList<String>();
- List<String> localNames = new ArrayList<String>();
- List<String> types = new ArrayList<String>();
- List<String> qNames = new ArrayList<String>();
+ if ("element".equals(type)) {
+ List<String> names = new ArrayList<>();
+ List<String> values = new ArrayList<>();
+ List<String> uris = new ArrayList<>();
+ List<String> localNames = new ArrayList<>();
+ List<String> types = new ArrayList<>();
+ List<String> qNames = new ArrayList<>();
whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames);
Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames);
@@ -264,7 +309,7 @@
String adultPath = adultId.substring(0, lastDotAdult);
adultPath = adultPath.replaceFirst(":", ".");
- return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."));
+ return childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + ".");
}
boolean isDirectChildAttribute(IndexableField child, IndexableField adult) {
@@ -278,7 +323,7 @@
String childType = childSegments[childSegments.length - 1];
- return (childPath.startsWith(adultPath + ":") && childType.equals("attribute"));
+ return childPath.startsWith(adultPath + ":") && "attribute".equals(childType);
}
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java
similarity index 78%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java
index 51510d5..1d58433 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+package org.apache.vxquery.runtime.functions.index.centralizer;
import java.io.DataOutput;
import java.io.File;
@@ -49,46 +49,54 @@
*/
public class IndexCentralizerUtil {
- private final String FILE_NAME = "VXQuery-Index-Directory.xml";
+ private static final String FILE_NAME = "VXQuery-Index-Directory.xml";
private final List<String> collections = new ArrayList<>();
- private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
- private File XML_FILE;
- private String INDEX_LOCATION;
- private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+ private static final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil");
+ private File xmlFile;
+ private String indexPath;
+ public static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>();
+ private static final StringValueBuilder svb = new StringValueBuilder();
+ private final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ private final DataOutput output = abvs.getDataOutput();
public IndexCentralizerUtil(File index) {
- this.INDEX_LOCATION = index.getPath();
+ indexPath = index.getPath();
if (!index.exists()) {
try {
FileUtils.forceMkdir(index);
} catch (IOException e) {
- LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e);
+ LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + indexPath + " " + e);
}
}
- XML_FILE = new File(index.getPath() + "/" + FILE_NAME);
+ xmlFile = new File(index.getPath() + "/" + FILE_NAME);
}
/**
* Get the index directory containing index of the given collection
*
- * @param collection : Collection folder
+ * @param collection
+ * : Collection folder
* @return Index folder.
*/
public String getIndexForCollection(String collection) {
- return indexCollectionMap.get(collection).getIndex();
+ if (indexCollectionMap.size() > 0 && indexCollectionMap.containsKey(collection)) {
+ return indexCollectionMap.get(collection).getIndex();
+ }
+ return null;
}
/**
* Put the index location corresponding to given collection.
* Index location is created by using the last 100 characters of collection.
*
- * @param collection : Collection directory
+ * @param collection
+ * : Collection directory
* @return index
*/
public String putIndexForCollection(String collection) {
int length = collection.replaceAll("/", "").length();
String index = collection.replaceAll("/", "");
- index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index);
+ index = indexPath + "/" + (length > 100 ? index.substring(length - 100) : index);
IndexLocator il = new IndexLocator();
il.setCollection(collection);
il.setIndex(index);
@@ -102,7 +110,8 @@
/**
* Remove the entry for given collection directory.
*
- * @param collection : Collection directory
+ * @param collection
+ * : Collection directory
*/
public void deleteEntryForCollection(String collection) {
indexCollectionMap.remove(collection);
@@ -110,14 +119,15 @@
/**
* Prints all collections which have an index created.
- * @param sb : The output is stored in a sequence
- * @throws IOException : If writing the dataOutput generates {@link IOException}
+ *
+ * @param sb
+ * : The output is stored in a sequence
+ * @throws IOException
+ * : If writing the dataOutput generates {@link IOException}
*/
public void getAllCollections(SequenceBuilder sb) throws IOException {
for (String s : collections) {
- StringValueBuilder svb = new StringValueBuilder();
- ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
- DataOutput output = abvs.getDataOutput();
+ abvs.reset();
output.write(ValueTag.XS_STRING_TAG);
svb.write(s, output);
sb.addItem(abvs);
@@ -128,11 +138,11 @@
* Read the collection, index directory file and populate the HashMap.
*/
public void readIndexDirectory() {
- if (this.XML_FILE.exists()) {
+ if (xmlFile.exists()) {
try {
JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class);
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
- IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE);
+ IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(xmlFile);
for (IndexLocator il : indexDirectory.getDirectory()) {
indexCollectionMap.put(il.getCollection(), il);
@@ -153,7 +163,7 @@
List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values());
id.setDirectory(indexLocators);
try {
- FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE);
+ FileOutputStream fileOutputStream = new FileOutputStream(this.xmlFile);
JAXBContext context = JAXBContext.newInstance(IndexDirectory.class);
Marshaller jaxbMarshaller = context.createMarshaller();
jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java
similarity index 89%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java
index 54d9ad9..d118926 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java
@@ -14,19 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
@XmlRootElement(name = "indexes")
@XmlAccessorType(XmlAccessType.FIELD)
-public class IndexDirectory implements Serializable{
+public class IndexDirectory implements Serializable {
+ private static final long serialVersionUID = 1L;
@XmlElement(name = "index", type = IndexLocator.class)
private List<IndexLocator> directory = new ArrayList<>();
@@ -35,7 +37,6 @@
return directory;
}
-
public void setDirectory(List<IndexLocator> directory) {
this.directory = directory;
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java
similarity index 89%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java
index 1a33c8b..49dbecd 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java
@@ -14,17 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.indexCentralizer;
+package org.apache.vxquery.runtime.functions.index.centralizer;
+
+import java.io.Serializable;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
@XmlAccessorType(XmlAccessType.FIELD)
@XmlRootElement(name = "Entry")
-public class IndexLocator implements Serializable{
+public class IndexLocator implements Serializable {
+ private static final long serialVersionUID = 1L;
@XmlAttribute
private String collection;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java
similarity index 80%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java
index 2a45747..0346a62 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java
@@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
/**
* Constants used in updating index
*/
public class Constants {
- public static String FIELD_PATH = "path";
- public static String META_FILE_NAME = "vxquery_index.xml";
+ public static final String FIELD_PATH = "path";
+ public static final String META_FILE_NAME = "vxquery_index.xml";
+
+ private Constants() {
+ }
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java
similarity index 87%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java
index d3b9fdf..65a8325 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java
@@ -14,29 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
-import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
-import org.apache.vxquery.exceptions.ErrorCode;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.index.IndexDocumentBuilder;
-import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
-import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
-import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
-
-import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -46,6 +25,23 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.index.IndexDocumentBuilder;
+import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer;
+import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil;
+import org.apache.vxquery.xmlparser.ITreeNodeIdProvider;
+
/**
* Update the index if the source files are changed.
*/
@@ -53,48 +49,37 @@
private MetaFileUtil metaFileUtil;
private ConcurrentHashMap<String, XmlMetadata> metadataMap;
private IPointable result;
- private ByteBufferInputStream bbis;
- private DataInputStream di;
- private SequenceBuilder sb;
+ private final SequenceBuilder sb = new SequenceBuilder();
private ArrayBackedValueStorage abvs;
private ITreeNodeIdProvider nodeIdProvider;
private ArrayBackedValueStorage abvsFileNode;
- private TaggedValuePointable nodep;
private String nodeId;
private IndexWriter indexWriter;
private Set<String> pathsFromFileList;
private String collectionFolder;
private String indexFolder;
- private Logger LOGGER = Logger.getLogger("Index Updater");
+ private final Logger LOGGER = Logger.getLogger("Index Updater");
private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss");
private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil();
- public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis,
- DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider,
- ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) {
+ public IndexUpdater(String indexFolder, IPointable result, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider,
+ ArrayBackedValueStorage abvsFileNode, String nodeId) {
this.indexFolder = indexFolder;
this.result = result;
- this.bbis = bbis;
- this.di = di;
- this.sb = sb;
this.abvs = abvs;
this.nodeIdProvider = nodeIdProvider;
this.abvsFileNode = abvsFileNode;
- this.nodep = nodep;
this.nodeId = nodeId;
this.pathsFromFileList = new HashSet<>();
}
/**
* Perform the initial configuration for index update/ delete processes.
- *
- * @throws SystemException
- * : If getting the index folder generates {@link SystemException}
+ *
* @throws IOException
* : If getting the index folder generates {@link IOException}
*/
- public void setup() throws SystemException, IOException {
-
+ public void setup() throws IOException {
// Read the metadata file and load the metadata map into memory.
metaFileUtil = new MetaFileUtil(indexFolder);
metaFileUtil.readMetadataFile();
@@ -114,14 +99,14 @@
/**
* Wrapper for update index function.
- *
+ *
* @throws IOException
* : If the directory doesn't exist
*/
public void updateIndex() throws IOException {
File collectionDirectory = new File(collectionFolder);
if (!collectionDirectory.exists()) {
- throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist.");
+ throw new IOException("The collection directory (" + collectionFolder + ") does not exist.");
}
//Execute update index process
@@ -134,7 +119,7 @@
/**
* Close opened IndexWriter and terminate the index update/ delete process.
- *
+ *
* @throws IOException
* : If exiting the index folder generates {@link IOException}
*/
@@ -149,7 +134,7 @@
/**
* Functional wrapper to update Metadata file.
- *
+ *
* @throws IOException
* : If updating metadata folder generates {@link IOException}
*/
@@ -190,8 +175,8 @@
//Update index corresponding to the xml file.
indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath()));
- indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep,
- abvsFileNode, nodeIdProvider, bbis, di, nodeId);
+ indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode,
+ nodeIdProvider, nodeId);
indexDocumentBuilder.printStart();
if (LOGGER.isDebugEnabled()) {
@@ -207,8 +192,8 @@
// In this case, the xml file has not added to the index. (It is a newly added file)
// Therefore generate a new index for this file and add it to the existing index.
- indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, abvsFileNode,
- nodeIdProvider, bbis, di, nodeId);
+ indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, nodeIdProvider,
+ nodeId);
indexDocumentBuilder.printStart();
if (LOGGER.isDebugEnabled()) {
@@ -235,8 +220,8 @@
* @throws IOException
* : If getting the file info generates {@link IOException}
*/
- private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException {
-
+ private XmlMetadata updateEntry(File file, XmlMetadata metadataArg) throws IOException {
+ XmlMetadata metadata = metadataArg;
if (metadata == null) {
metadata = new XmlMetadata();
}
@@ -286,7 +271,7 @@
* When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be
* restored.
* Otherwise the changes will be committed.
- *
+ *
* @throws SystemException
* : An attempt to divide by zero
*/
@@ -310,7 +295,7 @@
sb.finish();
result.set(abvs);
} catch (IOException e1) {
- throw new SystemException(ErrorCode.FOAR0001);
+ throw new SystemException(ErrorCode.FOAR0001, e1);
}
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java
similarity index 85%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java
index 0dfb54a..5f41355 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java
@@ -14,16 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import javax.xml.bind.DatatypeConverter;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -37,13 +29,22 @@
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
/**
* Utility class for writing, reading metadata file and generating checksum.
*/
public class MetaFileUtil {
private File metaFile;
- private Logger LOGGER = Logger.getLogger("MetadataFileUtil");
+ private static final Logger LOGGER = Logger.getLogger("MetadataFileUtil");
private String index;
private String collection;
private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>();
@@ -65,8 +66,11 @@
* Update the content of the metadata map.
* If the current collection data is present, replace it.
* Otherwise insert new.
- * @param metadataMap : Set of XmlMetaData objects.
- * @param index : The path to index location.
+ *
+ * @param metadataMap
+ * : Set of XmlMetaData objects.
+ * @param index
+ * : The path to index location.
*/
public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) {
this.indexMap = metadataMap;
@@ -109,22 +113,22 @@
* Write the content of the ConcurrentHashMap to the xml metadata file.
*/
public void writeMetadataToFile() {
- XmlMetadataCollection collection = new XmlMetadataCollection();
+ XmlMetadataCollection xmlMetadataCollection = new XmlMetadataCollection();
List<XmlMetadata> metadataList = new ArrayList<>();
for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) {
metadataList.add(entry.getValue());
}
- collection.setMetadataList(metadataList);
- collection.setCollection(this.collection);
- collection.setIndexLocation(this.index);
- try{
+ xmlMetadataCollection.setMetadataList(metadataList);
+ xmlMetadataCollection.setCollection(collection);
+ xmlMetadataCollection.setIndexLocation(this.index);
+ try {
FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile);
JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class);
Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
- jaxbMarshaller.marshal(collection, fileOutputStream);
+ jaxbMarshaller.marshal(xmlMetadataCollection, fileOutputStream);
if (LOGGER.isDebugEnabled()) {
LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!");
@@ -135,18 +139,18 @@
}
}
-
}
-
/**
* Generate MD5 checksum string for a given file.
*
- * @param file : File which the checksum should be generated.
+ * @param file
+ * : File which the checksum should be generated.
* @return : Checksum String
- * @throws IOException : The file is not available
+ * @throws IOException
+ * : The file is not available
*/
- public String generateMD5(File file) throws IOException {
+ public String generateMD5(File file) throws IOException {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(Files.readAllBytes(file.toPath()));
@@ -168,12 +172,12 @@
public boolean deleteMetaDataFile() {
try {
Files.delete(Paths.get(metaFile.getCanonicalPath()));
- if (LOGGER.isDebugEnabled()){
+ if (LOGGER.isDebugEnabled()) {
LOGGER.log(Level.DEBUG, "Metadata file deleted!");
}
return true;
} catch (IOException e) {
- if (LOGGER.isTraceEnabled()){
+ if (LOGGER.isTraceEnabled()) {
LOGGER.log(Level.ERROR, "Metadata file could not be deleted!");
}
return false;
@@ -182,6 +186,7 @@
/**
* Get the collection for a given index location.
+ *
* @return collection folder for a given index.
*/
public String getCollection() {
@@ -190,7 +195,9 @@
/**
* Set the entry for given index and collection.
- * @param collection : path to corresponding collection
+ *
+ * @param collection
+ * : path to corresponding collection
*/
public void setCollection(String collection) {
this.collection = collection;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java
similarity index 95%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java
index fa92b2f..fc96763 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java
similarity index 94%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java
index b6da6d9..063120c 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java
@@ -14,12 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
+
+import java.io.Serializable;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import java.io.Serializable;
/**
* Class to store metadata related to an XML file.
@@ -32,6 +33,7 @@
@XmlRootElement(name = "file")
@XmlAccessorType(XmlAccessType.FIELD)
public class XmlMetadata implements Serializable {
+ private static final long serialVersionUID = 1L;
private String path;
private String md5;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java
similarity index 96%
rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java
index 1f5c3e9..a1ca776 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.vxquery.runtime.functions.index.updateIndex;
+package org.apache.vxquery.runtime.functions.index.update;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
index 665c812..85ef4ca 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java
@@ -64,7 +64,7 @@
tvp.getValue(stringp);
try {
IParser parser = new JSONParser();
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+ FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
} catch (IOException e) {
throw new SystemException(ErrorCode.FODC0002, e);
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
index db908f6..15fd624 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java
@@ -78,7 +78,7 @@
tvp.getValue(stringp);
try {
IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+ FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
XDMConstants.setTrue(result);
} catch (Exception e) {
XDMConstants.setFalse(result);
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
index 5f08a8e..2fd1755 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java
@@ -79,7 +79,7 @@
try {
// Only one document should be parsed so its ok to have a unique parser.
IParser parser = new XMLParser(false, nodeIdProvider, nodeId);
- FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser);
+ FunctionHelper.readInDocFromPointable(stringp, abvs, parser);
} catch (Exception e) {
throw new SystemException(ErrorCode.SYSE0001, e);
}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
index 6558274..1d66c4e 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java
@@ -16,7 +16,6 @@
*/
package org.apache.vxquery.runtime.functions.util;
-import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.File;
import java.io.FileInputStream;
@@ -37,7 +36,7 @@
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.data.std.util.GrowableArray;
import org.apache.hyracks.data.std.util.UTF8StringBuilder;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.util.string.UTF8StringUtil;
import org.apache.vxquery.context.DynamicContext;
import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
import org.apache.vxquery.datamodel.accessors.TypedPointables;
@@ -485,6 +484,15 @@
return true;
}
+ public static String getStringFromBytes(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder();
+ UTF8StringUtil.toString(sb, bytes, 0);
+ return sb.toString();
+ }
+
public static boolean compareTaggedValues(AbstractValueComparisonOperation aOp, TaggedValuePointable tvp1,
TaggedValuePointable tvp2, DynamicContext dCtx, TypedPointables tp1, TypedPointables tp2)
throws SystemException {
@@ -1215,13 +1223,12 @@
System.err.println(" printUTF8String END");
}
- public static void readInDocFromPointable(UTF8StringPointable stringp, ByteBufferInputStream bbis,
- DataInputStream di, ArrayBackedValueStorage abvs, IParser parser) throws IOException {
- readInDocFromString(stringp.toString(), bbis, di, abvs, parser);
+ public static void readInDocFromPointable(UTF8StringPointable stringp, ArrayBackedValueStorage abvs,
+ IParser parser) throws IOException {
+ readInDocFromString(stringp.toString(), abvs, parser);
}
- public static void readInDocFromString(String fName, ByteBufferInputStream bbis, DataInputStream di,
- ArrayBackedValueStorage abvs, IParser parser) throws IOException {
+ public static void readInDocFromString(String fName, ArrayBackedValueStorage abvs, IParser parser) throws IOException {
Reader input;
if (!fName.contains("hdfs:/")) {
File file = new File(fName);
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
index 84c8ddf..9e21f53 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java
@@ -129,8 +129,6 @@
public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider, IFrameFieldAppender appender,
List<SequenceType> childSequenceTypes) {
this(attachTypes, nodeIdProvider, false);
-
- // Frame writing variables
this.appender = appender;
setChildPathSteps(childSequenceTypes);
}
@@ -297,10 +295,11 @@
/**
* The filter settings here are similar to one in the class linked below.
- *
+ *
+ * @throws SAXException
* @see org.apache.vxquery.runtime.functions.step.NodeTestFilter.java
*/
- private boolean startElementChildPathStep(String uri, String localName) {
+ private boolean startElementChildPathStep(String uri, String localName) throws SAXException {
if (subElement != null && depth <= subElement.length) {
// Check path step if it exists.
subElement[depth - 1] = true;
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
index 0c252b4..d3d02ae 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
@@ -222,7 +222,7 @@
}
}
- public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel)
+ public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel, List<String> collections)
throws AlgebricksException, SystemException {
moduleNode = XMLQueryParser.parse(name, query);
listener.notifyParseResult(moduleNode);
@@ -230,7 +230,7 @@
pprinter = new LogicalOperatorPrettyPrintVisitor(new AlgebricksAppendable(),
new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext()));
VXQueryMetadataProvider mdProvider = new VXQueryMetadataProvider(nodeList, ccb.getSourceFileMap(),
- module.getModuleContext(), this.hdfsConf, nodeControllerInfos);
+ module.getModuleContext(), this.hdfsConf, nodeControllerInfos, collections);
compiler = cFactory.createCompiler(module.getBody(), mdProvider, 0);
listener.notifyTranslationResult(module);
XMLQueryTypeChecker.typeCheckModule(module);
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
index 60f39f8..45d553f 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java
@@ -23,8 +23,8 @@
import javax.xml.bind.JAXBException;
import org.apache.commons.io.FileUtils;
-import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
+import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
index b79107e..1b3e0b7 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java
@@ -14,8 +14,6 @@
*/
package org.apache.vxquery.indexing;
-import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata;
-
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@@ -27,6 +25,8 @@
import java.nio.file.Paths;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.vxquery.runtime.functions.index.update.XmlMetadata;
+
/**
* TestConstants and methods which will be used in indexing test cases.
*/
diff --git a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
index 7646f97..1ee35a4 100644
--- a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
+++ b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java
@@ -142,6 +142,6 @@
XMLQueryCompiler compiler = new XMLQueryCompiler(null, nodeControllerInfos, 65536);
CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
new ResultSetId(System.nanoTime()), null);
- compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE);
+ compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE, null);
}
}
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
index e4ba6eb..fa0a900 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
@@ -14,16 +14,22 @@
*/
package org.apache.vxquery.xtest;
+import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.comm.IFrame;
@@ -53,13 +59,14 @@
public class TestRunner {
private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
-
+ private List<String> collectionList;
private XTestOptions opts;
private IHyracksClientConnection hcc;
private IHyracksDataset hds;
public TestRunner(XTestOptions opts) throws UnknownHostException {
this.opts = opts;
+ this.collectionList = new ArrayList<String>();
}
public void open() throws Exception {
@@ -67,8 +74,46 @@
hds = TestClusterUtil.getDataset();
}
+ protected static TestConfiguration getIndexConfiguration(TestCase testCase) {
+ XTestOptions opts = new XTestOptions();
+ opts.verbose = false;
+ opts.threads = 1;
+ opts.showQuery = true;
+ opts.showResult = true;
+ opts.hdfsConf = "src/test/resources/hadoop/conf";
+ opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" },
+ File.separator);
+ TestConfiguration indexConf = new TestConfiguration();
+ indexConf.options = opts;
+ String baseDir = new File(opts.catalog).getParent();
+ try {
+ String root = new File(baseDir).getCanonicalPath();
+ indexConf.testRoot = new File(root + "/./");
+ indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/");
+ indexConf.sourceFileMap = testCase.getSourceFileMap();
+ indexConf.xqueryFileExtension = ".xq";
+ indexConf.xqueryxFileExtension = "xqx";
+ indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return indexConf;
+
+ }
+
public TestCaseResult run(final TestCase testCase) {
TestCaseResult res = new TestCaseResult(testCase);
+ TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase));
+ testCaseIndex.setFolder("Indexing/Partition-1/");
+ testCaseIndex.setName("showIndexes");
+ runQuery(testCaseIndex, res);
+ String[] collections = res.result.split("\n");
+ this.collectionList = Arrays.asList(collections);
+ runQueries(testCase, res);
+ return res;
+ }
+
+ public void runQuery(TestCase testCase, TestCaseResult res) {
if (opts.verbose) {
System.err.println("Starting " + testCase.getXQueryDisplayName());
}
@@ -78,6 +123,7 @@
try {
try {
if (opts.showQuery) {
+
FileInputStream query = new FileInputStream(testCase.getXQueryFile());
System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
System.err.println(IOUtils.toString(query, "UTF-8"));
@@ -98,7 +144,7 @@
CompilerControlBlock ccb = new CompilerControlBlock(
new StaticContextImpl(RootStaticContextImpl.INSTANCE),
new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap());
- compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel);
+ compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList);
JobSpecification spec = compiler.getModule().getHyracksJobSpecification();
in.close();
@@ -172,7 +218,11 @@
System.err.println(res.result);
}
}
- return res;
+
+ }
+
+ public void runQueries(TestCase testCase, TestCaseResult res) {
+ runQuery(testCase, res);
}
public void close() throws Exception {
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt
new file mode 100644
index 0000000..baf9dca
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt
@@ -0,0 +1,2 @@
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data>
+<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
index 63fdda7..96a7671 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq
@@ -16,10 +16,10 @@
under the License. :)
(: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
let $datetime := xs:dateTime(fn:data($r/date))
where $r/station eq "GHCND:AS000000003"
and fn:year-from-dateTime($datetime) ge 2000
and fn:month-from-dateTime($datetime) eq 3
and fn:day-from-dateTime($datetime) eq 3
-return $r
\ No newline at end of file
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq
new file mode 100644
index 0000000..7a2bb2c
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
index cf41536..464c1cc 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq
@@ -19,6 +19,6 @@
(: Find all reading for hurricane force wind warning or extreme wind warning. :)
(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
index 5c99d9a..eeb019b 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq
@@ -19,7 +19,7 @@
(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
(: station (US000000002) for 2002. :)
fn:sum(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $r/station eq "GHCND:US000000002"
and $r/dataType eq "PRCP"
and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
index 39e5d17..06284ae 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find the highest recorded temperature (TMAX) in Celsius. :)
fn:max(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $r/dataType eq "TMAX"
return $r/value
) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
index 63aeca5..3a4ae05 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
index c81f271..9090edd 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station
where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
order by $s/id
return $s
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
index dd6b5f9..7c703b6 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq
@@ -18,8 +18,8 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data
where $s/id eq $r/station
and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
index fecb56d..4718240 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq
@@ -16,7 +16,7 @@
under the License. :)
(: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
let $datetime := xs:dateTime(fn:data($r/date))
where $r/station eq "GHCND:AS000000003"
and fn:year-from-dateTime($datetime) ge 2000
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq
new file mode 100644
index 0000000..0e42155
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
index 75c7a64..37e5626 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq
@@ -19,6 +19,6 @@
(: Find all reading for hurricane force wind warning or extreme wind warning. :)
(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
index 28f7473..358d3f3 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq
@@ -19,7 +19,7 @@
(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
(: station (US000000002) for 2002. :)
fn:sum(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $r/station eq "GHCND:US000000002"
and $r/dataType eq "PRCP"
and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
index 317a141..bd5ba1c 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find the highest recorded temperature (TMAX) in Celsius. :)
fn:max(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $r/dataType eq "TMAX"
return $r/value
) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
index 2deb4c3..77f6c2f 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
index a0ce1e9..c1a45f4 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station
where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
return $s
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
index b3e622c..a776ab9 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq
@@ -18,8 +18,8 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data
where $s/id eq $r/station
and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
index 0cccbc5..2bc9ce7 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq
@@ -16,7 +16,7 @@
under the License. :)
(: Search Lucene Index :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
let $datetime := xs:dateTime(fn:data($r/date))
where $r/station eq "GHCND:AS000000003"
and fn:year-from-dateTime($datetime) ge 2000
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq
new file mode 100644
index 0000000..e740365
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq
@@ -0,0 +1,25 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(: Search Lucene Index :)
+for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
+let $datetime := xs:dateTime(fn:data($r/date))
+where $r/station eq "GHCND:AS000000003"
+ and fn:year-from-dateTime($datetime) ge 2000
+ and fn:month-from-dateTime($datetime) eq 3
+ and fn:day-from-dateTime($datetime) eq 3
+return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
index c282e31..a1b86ac 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq
@@ -19,6 +19,6 @@
(: Find all reading for hurricane force wind warning or extreme wind warning. :)
(: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :)
(: meters per second). (Wind value is in tenth of a meter per second) :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744
return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
index 33ea1c9..9cc2b8e 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq
@@ -19,7 +19,7 @@
(: Find the annual precipitation (PRCP) for a Seattle using the airport :)
(: station (US000000002) for 2002. :)
fn:sum(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $r/station eq "GHCND:US000000002"
and $r/dataType eq "PRCP"
and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
index d213082..dd26e87 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq
@@ -18,7 +18,7 @@
(: Search Lucene Index :)
(: Find the highest recorded temperature (TMAX) in Celsius. :)
fn:max(
- for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+ for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $r/dataType eq "TMAX"
return $r/value
) div 10
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
index 1d98682..4a9d224 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000")
return $r
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
index abe2184..4407079 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq
@@ -18,6 +18,6 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station
where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
return $s
diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
index 7b40ca0..664b150 100644
--- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
+++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq
@@ -18,8 +18,8 @@
(: Search Lucene Index :)
(: Find all the weather readings for Washington state for a specific day :)
(: 2002-2-2. :)
-for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station
-for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data
+for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station
+for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data
where $s/id eq $r/station
and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1"))
diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
index 5ecdb94..8e28135 100644
--- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
+++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml
@@ -353,7 +353,7 @@
&LibrariesInJSONiq;
</test-group>
</test-group>
- <test-group name="SerializationQueries" featureOwner="Christina Pavlopoulou">
+ <test-group name="SerializationQueries" featureOwner="Christina Pavlopoulou">
<GroupInfo>
<title>Serialize Function Queries</title>
<description/>
@@ -365,7 +365,7 @@
</GroupInfo>
&SerializationQueries;
</test-group>
- </test-group>
+ </test-group>
<test-group name="XMLInJSONQueries" featureOwner="Riyafa Abdul Hameed">
<GroupInfo>
<title>XML in JSON</title>
diff --git a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
index 7cf6bf6..cc6b65b 100644
--- a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
+++ b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml
@@ -35,6 +35,11 @@
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
+ <test-case name="use-index-1-user" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1_user" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1_user.txt</output-file>
+ </test-case>
<test-case name="use-index-2" FilePath="Indexing/Partition-1/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
@@ -95,6 +100,11 @@
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
+ <test-case name="use-index-1-user" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1_user" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1_user.txt</output-file>
+ </test-case>
<test-case name="use-index-2" FilePath="Indexing/Partition-2/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
@@ -145,6 +155,11 @@
<query name="useIndex1" date="2016-05-26"/>
<output-file compare="Text">useIndex1.txt</output-file>
</test-case>
+ <test-case name="use-index-1-user" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
+ <description>Get Collection From Lucene Index</description>
+ <query name="useIndex1_user" date="2016-05-26"/>
+ <output-file compare="Text">useIndex1_user.txt</output-file>
+ </test-case>
<test-case name="use-index-2" FilePath="Indexing/Partition-4/" Creator="Steven Jacobs">
<description>Get Collection From Lucene Index</description>
<query name="useIndex2" date="2016-05-26"/>
