[VXQUERY-180] REST Server implementation
- Implemented REST API
- Altered CLI to use REST API
- Migrated XTests to use REST API
details:
- Implemented the REST Server to start through the cluster controller
application.
- CLI now calls the REST API (remote if given, local one else) to
execute queries.
- Migrated XTests to use the REST API to run queries related to tests
diff --git a/pom.xml b/pom.xml
index 5f03a49..7e298c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -251,6 +251,18 @@
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-http</artifactId>
+ <version>${hyracks.fullstack.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.6.Final</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>algebricks-compiler</artifactId>
<version>${hyracks.version}</version>
</dependency>
@@ -670,6 +682,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <hyracks.version>0.2.17-incubating</hyracks.version>
+ <hyracks.fullstack.version>0.3.1</hyracks.fullstack.version>
<hyracks.version>0.3.0</hyracks.version>
<apache-rat-plugin.version>0.11</apache-rat-plugin.version>
</properties>
@@ -680,5 +694,6 @@
<module>vxquery-cli</module>
<module>vxquery-xtest</module>
<module>vxquery-benchmark</module>
+ <module>vxquery-rest</module>
</modules>
</project>
diff --git a/src/site/apt/user_query.apt b/src/site/apt/user_query.apt
index c5132c3..f4d0dd5 100644
--- a/src/site/apt/user_query.apt
+++ b/src/site/apt/user_query.apt
@@ -35,21 +35,26 @@
Command line options for all systems.
----------------------------------------
--O N : Optimization Level. Default: Full Optimization
--available-processors N : Number of available processors. (default java's available processors)
--client-net-ip-address VAL : IP Address of the ClusterController
--client-net-port N : Port of the ClusterController (default 1098)
--compileonly : Compile the query and stop
--frame-size N : Frame size in bytes. (default 65536)
--local-node-controllers N : Number of local node controllers (default 1)
--repeatexec N : Number of times to repeat execution
--showast : Show abstract syntax tree
--showoet : Show optimized expression tree
--showquery : Show query string
--showrp : Show Runtime plan
--showtet : Show translated expression tree
--timing : Produce timing information
--hdfs-conf VAL : The folder containing the HDFS configuration files
+ -O N : Optimization Level. (default: Full Optimization)
+ -available-processors N : Number of available processors. (default: java's available processors)
+ -buffer-size N : Disk read buffer size in bytes.
+ -compileonly : Compile the query and stop.
+ -frame-size N : Frame size in bytes. (default: 65,536)
+ -hdfs-conf VAL : Directory path to Hadoop configuration files
+ -join-hash-size N : Join hash size in bytes. (default: 67,108,864)
+ -local-node-controllers N : Number of local node controllers. (default: 1)
+ -maximum-data-size N : Maximum possible data size in bytes. (default: 150,323,855,000)
+ -repeatexec N : Number of times to repeat execution.
+ -rest-ip-address VAL : IP Address of the REST Server.
+ -rest-port N : Port of REST Server.
+ -result-file VAL : File path to save the query result.
+ -showast : Show abstract syntax tree.
+ -showoet : Show optimized expression tree.
+ -showquery : Show query string.
+ -showrp : Show Runtime plan.
+ -showtet : Show translated expression tree.
+ -timing : Produce timing information.
+ -timing-ignore-queries N : Ignore the first X number of quereies.
----------------------------------------
* Java Options
diff --git a/vxquery-cli/pom.xml b/vxquery-cli/pom.xml
index 52c32ad..e7ad16a 100644
--- a/vxquery-cli/pom.xml
+++ b/vxquery-cli/pom.xml
@@ -14,7 +14,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -61,12 +62,6 @@
</execution>
</executions>
</plugin>
- <!--
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- </plugin>
- -->
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
@@ -82,64 +77,11 @@
<dependencies>
<dependency>
<groupId>org.apache.vxquery</groupId>
- <artifactId>apache-vxquery-core</artifactId>
+ <artifactId>apache-vxquery-rest</artifactId>
<version>0.7-SNAPSHOT</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-control-cc</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>algebricks-common</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>algebricks-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-control-common</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-dataflow-std</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
</dependency>
</dependencies>
-
-
<reporting>
<plugins>
<plugin>
diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
index 25ff9c4..23d5eaa 100644
--- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
+++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
@@ -14,73 +14,57 @@
*/
package org.apache.vxquery.cli;
-import java.io.ByteArrayOutputStream;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.io.StringReader;
-import java.net.InetAddress;
-import java.nio.file.Files;
+import java.io.PrintStream;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
+import java.util.logging.LogManager;
+
+import javax.xml.bind.JAXBException;
import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.client.HyracksConnection;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.client.NodeControllerInfo;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.job.JobFlag;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.client.dataset.HyracksDataset;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.common.controllers.CCConfig;
-import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
-import org.apache.vxquery.compiler.CompilerControlBlock;
-import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory;
-import org.apache.vxquery.context.DynamicContext;
-import org.apache.vxquery.context.DynamicContextImpl;
-import org.apache.vxquery.context.RootStaticContextImpl;
-import org.apache.vxquery.context.StaticContextImpl;
-import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.result.ResultUtils;
-import org.apache.vxquery.xmlquery.query.Module;
-import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener;
-import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.util.LocalClusterUtil;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.Metrics;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.VXQueryConfig;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
+/**
+ * CLI for VXQuery. This class is using the REST API to execute statements given by the user.
+ *
+ * @author Erandi Ganepola
+ */
public class VXQuery {
- private final CmdLineOptions opts;
- private final CmdLineOptions indexOpts;
- private ClusterControllerService cc;
- private NodeControllerService[] ncs;
- private IHyracksClientConnection hcc;
- private IHyracksDataset hds;
- private List<String> collectionList;
- private ResultSetId resultSetId;
- private static List<String> timingMessages = new ArrayList<>();
- private static long sumTiming;
- private static long sumSquaredTiming;
- private static long minTiming = Long.MAX_VALUE;
- private static long maxTiming = Long.MIN_VALUE;
+ private final CmdLineOptions opts;
+
+ private static LocalClusterUtil localClusterUtil;
+ private String restIpAddress;
+ private int restPort;
+
+ private static List<Metrics> metricsList = new ArrayList<>();
+ private int executionIteration;
/**
* Constructor to use command line options passed.
@@ -90,26 +74,17 @@
*/
public VXQuery(CmdLineOptions opts) {
this.opts = opts;
- // The index query returns only the result, without any other information.
- this.indexOpts = opts;
- indexOpts.showAST = false;
- indexOpts.showOET = false;
- indexOpts.showQuery = false;
- indexOpts.showRP = false;
- indexOpts.showTET = false;
- indexOpts.timing = false;
- indexOpts.compileOnly = false;
- this.collectionList = new ArrayList<String>();
}
/**
* Main method to get command line options and execute query process.
*
* @param args
- * @throws Exception
+ * command line arguments
*/
- public static void main(String[] args) throws Exception {
- Date start = new Date();
+ public static void main(String[] args) {
+ LogManager.getLogManager().reset();
+
final CmdLineOptions opts = new CmdLineOptions();
CmdLineParser parser = new CmdLineParser(opts);
@@ -120,243 +95,249 @@
parser.printUsage(System.err);
return;
}
- if (opts.arguments.isEmpty()) {
+
+ if (opts.xqFiles.isEmpty()) {
parser.printUsage(System.err);
return;
}
+
VXQuery vxq = new VXQuery(opts);
- vxq.execute();
- // if -timing argument passed, show the starting and ending times
- if (opts.timing) {
- Date end = new Date();
- timingMessage("Execution time: " + (end.getTime() - start.getTime()) + " ms");
- if (opts.repeatExec > opts.timingIgnoreQueries) {
- Double mean = (double) (sumTiming) / (opts.repeatExec - opts.timingIgnoreQueries);
- double sd = Math.sqrt(sumSquaredTiming / (opts.repeatExec - opts.timingIgnoreQueries) - mean * mean);
- timingMessage("Average execution time: " + mean + " ms");
- timingMessage("Standard deviation: " + String.format("%.4f", sd));
- timingMessage("Coefficient of variation: " + String.format("%.4f", sd / mean));
- timingMessage("Minimum execution time: " + minTiming + " ms");
- timingMessage("Maximum execution time: " + maxTiming + " ms");
- }
- System.out.println("Timing Summary:");
- for (String time : timingMessages) {
- System.out.println(" " + time);
- }
- }
-
+ vxq.execute(opts.xqFiles);
}
- /**
- * Creates a new Hyracks connection with: the client IP address and port provided, if IP address is provided in command line. Otherwise create a new virtual
- * cluster with Hyracks nodes. Queries passed are run either way. After running queries, if a virtual cluster has been created, it is shut down.
- *
- * @throws Exception
- */
- private void execute() throws Exception {
- System.setProperty("vxquery.buffer_size", Integer.toString(opts.bufferSize));
+ private void execute(List<String> xqFiles) {
+ if (opts.restIpAddress == null) {
+ System.out.println("No REST Ip address given. Creating a local hyracks cluster");
- if (opts.clientNetIpAddress != null) {
- hcc = new HyracksConnection(opts.clientNetIpAddress, opts.clientNetPort);
- runQueries();
- } else {
- if (!opts.compileOnly) {
- startLocalHyracks();
- }
+ VXQueryConfig vxqConfig = new VXQueryConfig();
+ vxqConfig.setAvailableProcessors(opts.availableProcessors);
+ vxqConfig.setFrameSize(opts.frameSize);
+ vxqConfig.setHdfsConf(opts.hdfsConf);
+ vxqConfig.setJoinHashSize(opts.joinHashSize);
+ vxqConfig.setMaximumDataSize(opts.maximumDataSize);
+
+ localClusterUtil = new LocalClusterUtil();
try {
- runQueries();
- } finally {
- if (!opts.compileOnly) {
- stopLocalHyracks();
- }
+ localClusterUtil.init(vxqConfig);
+ restIpAddress = localClusterUtil.getIpAddress();
+ restPort = localClusterUtil.getRestPort();
+ } catch (Exception e) {
+ System.err.println("Unable to start local hyracks cluster due to: " + e.getMessage());
+ e.printStackTrace();
+ return;
+ }
+ } else {
+ restIpAddress = opts.restIpAddress;
+ restPort = opts.restPort;
+ }
+
+ System.out.println("Running queries given in: " + Arrays.toString(xqFiles.toArray()));
+ runQueries(xqFiles);
+
+ if (localClusterUtil != null) {
+ try {
+ localClusterUtil.deinit();
+ } catch (Exception e) {
+ System.err.println("Error occurred when stopping local hyracks: " + e.getMessage());
}
}
}
- /**
- * Reads the contents of the files passed in the list of arguments to a string. If -showquery argument is passed, output the query as string. Run the query
- * for the string.
- *
- * @throws IOException
- * @throws SystemException
- * @throws Exception
- */
-
- private void runQueries() throws Exception {
- List<String> queries = opts.arguments;
- // Run the showIndexes query before executing any target query, to store the index metadata
- List<String> queriesIndex = new ArrayList<String>();
- queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq");
- OutputStream resultStream = new ByteArrayOutputStream();
- executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts);
- ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream;
- String result = new String(bos.toByteArray());
- String[] collections = result.split("\n");
- this.collectionList = Arrays.asList(collections);
- executeQueries(queries);
- }
-
- public void executeQueries(List<String> queries) throws Exception {
- for (String query : queries) {
- OutputStream resultStream = System.out;
- if (opts.resultFile != null) {
- resultStream = new FileOutputStream(new File(opts.resultFile));
+ public void runQueries(List<String> xqFiles) {
+ for (String xqFile : xqFiles) {
+ String query;
+ try {
+ query = slurp(xqFile);
+ } catch (IOException e) {
+ System.err.println(String.format("Error occurred when reading XQuery file %s with message: %s", xqFile,
+ e.getMessage()));
+ continue;
}
- executeQuery(query, opts.repeatExec, resultStream, opts);
+
+ System.out.println();
+ System.out.println("====================================================");
+ System.out.println("\tQuery - '" + xqFile + "'");
+ System.out.println("====================================================");
+
+ QueryRequest request = createQueryRequest(opts, query);
+ metricsList.clear();
+
+ for (int i = 0; i < opts.repeatExec; i++) {
+ System.out.println("**** Repetition : " + (i + 1) + " ****");
+
+ executionIteration = i;
+ sendQueryRequest(xqFile, request, this);
+ }
+
+ if (opts.repeatExec > 1) {
+ showTimingSummary();
+ }
}
}
- public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options)
- throws Exception {
- PrintWriter writer = new PrintWriter(resultStream, true);
- String qStr = slurp(query);
- if (opts.showQuery) {
- writer.println(qStr);
- }
- VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET,
- opts.showRP);
-
- Date start = opts.timing ? new Date() : null;
-
- Map<String, NodeControllerInfo> nodeControllerInfos = null;
- if (hcc != null) {
- nodeControllerInfos = hcc.getNodeControllerInfos();
- }
- XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
- opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf);
- resultSetId = createResultSetId();
- CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE),
- resultSetId, null);
- compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList);
- // if -timing argument passed, show the starting and ending times
- Date end = opts.timing ? new Date() : null;
- if (opts.timing) {
- timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms");
- }
- if (opts.compileOnly) {
+ private void onSuccess(String xqFile, QueryRequest request, SyncQueryResponse response) {
+ if (response == null) {
+ System.err.println(String.format("Unable to execute query %s", request.getStatement()));
return;
}
- Module module = compiler.getModule();
- JobSpecification js = module.getHyracksJobSpecification();
+ if (opts.showQuery) {
+ printField("Query", response.getStatement());
+ }
- DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
- js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
+ if (request.isShowMetrics()) {
+ String metrics = String.format("Compile Time:\t%d\nElapsed Time:\t%d",
+ response.getMetrics().getCompileTime(), response.getMetrics().getElapsedTime());
+ printField("Metrics", metrics);
+ }
- // Repeat execution for number of times provided in -repeatexec argument
- for (int i = 0; i < repeatedExecution; ++i) {
- start = opts.timing ? new Date() : null;
- runJob(js, writer);
- // if -timing argument passed, show the starting and ending times
- if (opts.timing) {
- end = new Date();
- long currentRun = end.getTime() - start.getTime();
- if ((i + 1) > opts.timingIgnoreQueries) {
- sumTiming += currentRun;
- sumSquaredTiming += currentRun * currentRun;
- if (currentRun < minTiming) {
- minTiming = currentRun;
- }
- if (maxTiming < currentRun) {
- maxTiming = currentRun;
- }
+ if (request.isShowAbstractSyntaxTree()) {
+ printField("Abstract Syntax Tree", response.getAbstractSyntaxTree());
+ }
+
+ if (request.isShowTranslatedExpressionTree()) {
+ printField("Translated Expression Tree", response.getTranslatedExpressionTree());
+ }
+
+ if (request.isShowOptimizedExpressionTree()) {
+ printField("Optimized Expression Tree", response.getOptimizedExpressionTree());
+ }
+
+ if (request.isShowRuntimePlan()) {
+ printField("Runtime Plan", response.getRuntimePlan());
+ }
+
+ printField("Results", response.getResults());
+
+ if (executionIteration >= opts.timingIgnoreQueries) {
+ metricsList.add(response.getMetrics());
+ }
+ }
+
+ private void onFailure(String xqFile, ErrorResponse response) {
+ if (response == null) {
+ System.err.println(String.format("Unable to execute query in %s", xqFile));
+ return;
+ }
+
+ System.err.println();
+ System.err.println("------------------------ Errors ---------------------");
+
+ Error error = response.getError();
+ String errorMsg = String.format("Code:\t %d\nMessage:\t %s", error.getCode(), error.getMessage());
+ printField(System.err, String.format("Errors for '%s'", xqFile), errorMsg);
+ }
+
+ /**
+ * Submits a query to be executed by the REST API. Will call {@link #onFailure(String, ErrorResponse)} if any error
+ * occurs when submitting the query. Else will call {@link #onSuccess(String, QueryRequest, SyncQueryResponse)} with
+ * the {@link AsyncQueryResponse}
+ *
+ * @param xqFile
+ * .xq file with the query to be executed
+ * @param request
+ * {@link QueryRequest} instance to be submitted to REST API
+ * @param cli
+ * cli class instance
+ */
+ private static void sendQueryRequest(String xqFile, QueryRequest request, VXQuery cli) {
+ URI uri = null;
+ try {
+ uri = RestUtils.buildQueryURI(request, cli.restIpAddress, cli.restPort);
+ } catch (URISyntaxException e) {
+ System.err.println(
+ String.format("Unable to build URI to call REST API for query: %s", request.getStatement()));
+ cli.onFailure(xqFile, null);
+ }
+
+ CloseableHttpClient httpClient = HttpClients.custom().build();
+ try {
+ HttpGet httpGet = new HttpGet(uri);
+ httpGet.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_JSON);
+
+ try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) {
+ HttpEntity entity = httpResponse.getEntity();
+
+ String response = RestUtils.readEntity(entity);
+ if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ cli.onSuccess(xqFile, request,
+ RestUtils.mapEntity(response, SyncQueryResponse.class, CONTENT_TYPE_JSON));
+ } else {
+ cli.onFailure(xqFile, RestUtils.mapEntity(response, ErrorResponse.class, CONTENT_TYPE_JSON));
}
- timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms");
+ } catch (IOException e) {
+ System.err.println("Error occurred when reading entity: " + e.getMessage());
+ cli.onFailure(xqFile, null);
+ } catch (JAXBException e) {
+ System.err.println("Error occurred when mapping query response: " + e.getMessage());
+ cli.onFailure(xqFile, null);
+ }
+ } finally {
+ HttpClientUtils.closeQuietly(httpClient);
+ }
+ }
+
+ /**
+ * Once the query in a given .xq file has been executed (with repeated executions as well), this method calculates
+ * mean, standard deviation, minimum and maximum execution times.
+ */
+ private void showTimingSummary() {
+ double sumTime = 0;
+ double sumSquaredTime = 0;
+ long minTime = Long.MAX_VALUE;
+ long maxTime = Long.MIN_VALUE;
+
+ for (int i = 0; i < metricsList.size(); i++) {
+ Metrics metrics = metricsList.get(i);
+ long totalTime = metrics.getCompileTime() + metrics.getElapsedTime();
+
+ sumTime += totalTime;
+ sumSquaredTime += totalTime * totalTime;
+
+ if (totalTime < minTime) {
+ minTime = totalTime;
+ }
+
+ if (totalTime > maxTime) {
+ maxTime = totalTime;
}
}
+
+ double mean = sumTime / (opts.repeatExec - opts.timingIgnoreQueries);
+ double sd = Math.sqrt(sumSquaredTime / (opts.repeatExec - opts.timingIgnoreQueries) - mean * mean);
+
+ System.out.println();
+ System.out.println("\t**** Timing Summary ****");
+ System.out.println("----------------------------------------------------");
+ System.out.println(String.format("Repetitions:\t%d, Timing Ignored Iterations:\t%d", opts.repeatExec,
+ opts.timingIgnoreQueries));
+ System.out.println("Average execution time:\t" + mean + " ms");
+ System.out.println("Standard deviation:\t" + String.format("%.4f", sd));
+ System.out.println("Coefficient of variation:\t" + String.format("%.4f", sd / mean));
+ System.out.println("Minimum execution time:\t" + minTime + " ms");
+ System.out.println("Maximum execution time:\t" + maxTime + " ms");
+ System.out.println();
+ }
+
+ private static QueryRequest createQueryRequest(CmdLineOptions opts, String query) {
+ QueryRequest request = new QueryRequest(query);
+ request.setCompileOnly(opts.compileOnly);
+ request.setOptimization(opts.optimizationLevel);
+ request.setFrameSize(opts.frameSize);
+ request.setRepeatExecutions(opts.repeatExec);
+ request.setShowMetrics(opts.timing);
+ request.setShowAbstractSyntaxTree(opts.showAST);
+ request.setShowTranslatedExpressionTree(opts.showTET);
+ request.setShowOptimizedExpressionTree(opts.showOET);
+ request.setShowRuntimePlan(opts.showRP);
+ request.setAsync(false);
+
+ return request;
}
/**
- * Creates a Hyracks dataset, if not already existing with the job frame size, and 1 reader. Allocates a new buffer of size specified in the frame of Hyracks
- * node. Creates new dataset reader with the current job ID and result set ID. Outputs the string in buffer for each frame.
- *
- * @param spec
- * JobSpecification object, containing frame size. Current specified job.
- * @param writer
- * Writer for output of job.
- * @throws Exception
- */
- private void runJob(JobSpecification spec, PrintWriter writer) throws Exception {
- int nReaders = 1;
- if (hds == null) {
- hds = new HyracksDataset(hcc, spec.getFrameSize(), nReaders);
- }
-
- JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
-
- FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize());
- IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
- IHyracksDatasetReader reader = hds.createReader(jobId, resultSetId);
- IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
-
- while (reader.read(frame) > 0) {
- writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor));
- writer.flush();
- frame.getBuffer().clear();
- }
-
- hcc.waitForCompletion(jobId);
- }
-
- /**
- * Create a unique result set id to get the correct query back from the cluster.
- *
- * @return Result Set id generated with current system time.
- */
- protected ResultSetId createResultSetId() {
- return new ResultSetId(System.nanoTime());
- }
-
- /**
- * Start local virtual cluster with cluster controller node and node controller nodes. IP address provided for node controller is localhost. Unassigned ports
- * 39000 and 39001 are used for client and cluster port respectively. Creates a new Hyracks connection with the IP address and client ports.
- *
- * @throws Exception
- */
- public void startLocalHyracks() throws Exception {
- String localAddress = InetAddress.getLocalHost().getHostAddress();
- CCConfig ccConfig = new CCConfig();
- ccConfig.clientNetIpAddress = localAddress;
- ccConfig.clientNetPort = 39000;
- ccConfig.clusterNetIpAddress = localAddress;
- ccConfig.clusterNetPort = 39001;
- ccConfig.httpPort = 39002;
- ccConfig.profileDumpPeriod = 10000;
- cc = new ClusterControllerService(ccConfig);
- cc.start();
-
- ncs = new NodeControllerService[opts.localNodeControllers];
- for (int i = 0; i < ncs.length; i++) {
- NCConfig ncConfig = new NCConfig();
- ncConfig.ccHost = "localhost";
- ncConfig.ccPort = 39001;
- ncConfig.clusterNetIPAddress = localAddress;
- ncConfig.dataIPAddress = localAddress;
- ncConfig.resultIPAddress = localAddress;
- ncConfig.nodeId = "nc" + (i + 1);
- //TODO: enable index folder as a cli option for on-the-fly indexing queries
- ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString();
- ncs[i] = new NodeControllerService(ncConfig);
- ncs[i].start();
- }
-
- hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
- }
-
- /**
- * Shuts down the virtual cluster, along with all nodes and node execution, network and queue managers.
- *
- * @throws Exception
- */
- public void stopLocalHyracks() throws Exception {
- for (int i = 0; i < ncs.length; i++) {
- ncs[i].stop();
- }
- cc.stop();
- }
-
- /**
- * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding is used.
+ * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding
+ * is used.
*
* @param query
* The query with filename to be processed
@@ -367,46 +348,49 @@
return FileUtils.readFileToString(new File(query), "UTF-8");
}
- /**
- * Save and print out the timing message.
- *
- * @param message
- */
- private static void timingMessage(String message) {
- System.out.println(message);
- timingMessages.add(message);
+ private static void printField(PrintStream out, String field, String value) {
+ out.println();
+ field = field + ":";
+ out.print(field);
+
+ String[] lines = value.split("\n");
+ for (int i = 0; i < lines.length; i++) {
+ int margin = 4;
+ if (i != 0) {
+ margin += field.length();
+ }
+ System.out.print(String.format("%1$" + margin + "s%2$s\n", "", lines[i]));
+ }
+ }
+
+ private static void printField(String field, String value) {
+ printField(System.out, field, value);
}
/**
* Helper class with fields and methods to handle all command line options
*/
private static class CmdLineOptions {
- @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)")
- private int availableProcessors = -1;
+ @Option(name = "-rest-ip-address", usage = "IP Address of the REST Server")
+ private String restIpAddress = null;
- @Option(name = "-client-net-ip-address", usage = "IP Address of the ClusterController.")
- private String clientNetIpAddress = null;
+ @Option(name = "-rest-port", usage = "Port of REST Server")
+ private int restPort = 8085;
- @Option(name = "-client-net-port", usage = "Port of the ClusterController. (default: 1098)")
- private int clientNetPort = 1098;
+ @Option(name = "-compileonly", usage = "Compile the query and stop.")
+ private boolean compileOnly;
- @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)")
- private int localNodeControllers = 1;
+ @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)")
+ private int optimizationLevel = Integer.MAX_VALUE;
@Option(name = "-frame-size", usage = "Frame size in bytes. (default: 65,536)")
private int frameSize = 65536;
- @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)")
- private long joinHashSize = -1;
+ @Option(name = "-repeatexec", usage = "Number of times to repeat execution.")
+ private int repeatExec = 1;
- @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)")
- private long maximumDataSize = -1;
-
- @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.")
- private int bufferSize = -1;
-
- @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)")
- private int optimizationLevel = Integer.MAX_VALUE;
+ @Option(name = "-timing", usage = "Produce timing information.")
+ private boolean timing;
@Option(name = "-showquery", usage = "Show query string.")
private boolean showQuery;
@@ -423,29 +407,33 @@
@Option(name = "-showrp", usage = "Show Runtime plan.")
private boolean showRP;
- @Option(name = "-compileonly", usage = "Compile the query and stop.")
- private boolean compileOnly;
+ // Optional (Not supported by REST API) parameters. Only used for creating a
+ // local hyracks cluster
+ @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)")
+ private long joinHashSize = -1;
- @Option(name = "-repeatexec", usage = "Number of times to repeat execution.")
- private int repeatExec = 1;
+ @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)")
+ private long maximumDataSize = -1;
+
+ @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.")
+ private int bufferSize = -1;
@Option(name = "-result-file", usage = "File path to save the query result.")
private String resultFile = null;
- @Option(name = "-timing", usage = "Produce timing information.")
- private boolean timing;
-
@Option(name = "-timing-ignore-queries", usage = "Ignore the first X number of quereies.")
- private int timingIgnoreQueries = 2;
-
- @Option(name = "-x", usage = "Bind an external variable")
- private Map<String, String> bindings = new HashMap<>();
+ private int timingIgnoreQueries = 0;
@Option(name = "-hdfs-conf", usage = "Directory path to Hadoop configuration files")
private String hdfsConf = null;
- @Argument
- private List<String> arguments = new ArrayList<>();
- }
+ @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)")
+ private int availableProcessors = -1;
-}
+ @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)")
+ private int localNodeControllers = 1;
+
+ @Argument
+ private List<String> xqFiles = new ArrayList<>();
+ }
+}
\ No newline at end of file
diff --git a/vxquery-cli/src/site/markdown/index.md b/vxquery-cli/src/site/markdown/index.md
new file mode 100644
index 0000000..758c947
--- /dev/null
+++ b/vxquery-cli/src/site/markdown/index.md
@@ -0,0 +1,116 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# VXQuery CLI
+
+VXQuery CLI is the command line utility which can be used to execute XQueries
+with ease. No pre-configuration needs to be done in order to execute an XQuery.
+
+---
+
+## Quick Start
+
+***
+
+### Requirements
+
+- Apache VXQuery™ source archive (apache-vxquery-X.Y-source-release.zip)
+- JDK >= 1.8
+- Apache Maven >= 3.2
+
+***
+
+### Installing
+
+VXQuery CLI comes bundled with the VXQuery source distribution
+(apache-vxquery-X.Y-source-release.zip).
+
+First, run `mvn package`.
+
+```
+$ unzip apache-vxquery-X.Y-source-release.zip
+$ cd apache-vxquery-X.Y
+$ mvn package -DskipTests
+```
+
+**vxquery-cli** binaries are located at `vxquery-cli/target/appassembler/bin`.
+There are 2 files in this directory, **vxq** which is the bash executable for unix
+based systems and **vxq.bat** for windows systems. Depending on the platform,
+suitable executable needs to be selected.
+
+***
+
+### Executing a Query
+
+#### Put the query into a file
+
+VXQuery CLI takes a file location as the argument where this file includes the
+query(statement) to be executed. Suppose the following query needs to be executed.
+
+```
+for $x in doc("books.xml")/bookstore/book
+where $x/price>30
+order by $x/title
+return $x/title
+```
+This statement is querying for the book titles in **books.xml** where price of
+the book is greater than 30. Also this query asks for the results to be ordered by
+*title* as well. Now, create a file (say **test.xq**) and put the above query as
+the content.
+
+**NOTE:** You can replace **books.xml** with any XML file that you have and want
+to run a query on.
+
+#### Execute the query
+
+We need to invoke the matching executable according to your platform (unix/windows)
+inside `vxquery-cli/target/appassembler/bin` directory. To execute the query, run:
+
+```
+sh ./apache-vxquery-X.Y/vxquery-cli/target/appassembler/bin/vxq path/to/test.xq
+```
+
+***
+
+## Command Line Options
+
+```
+ -O N : Optimization Level. (default: Full Optimization)
+ -available-processors N : Number of available processors. (default: java's available processors)
+ -buffer-size N : Disk read buffer size in bytes.
+ -compileonly : Compile the query and stop.
+ -frame-size N : Frame size in bytes. (default: 65,536)
+ -hdfs-conf VAL : Directory path to Hadoop configuration files
+ -join-hash-size N : Join hash size in bytes. (default: 67,108,864)
+ -local-node-controllers N : Number of local node controllers. (default: 1)
+ -maximum-data-size N : Maximum possible data size in bytes. (default: 150,323,855,000)
+ -repeatexec N : Number of times to repeat execution.
+ -rest-ip-address VAL : IP Address of the REST Server.
+ -rest-port N : Port of REST Server.
+ -result-file VAL : File path to save the query result.
+ -showast : Show abstract syntax tree.
+ -showoet : Show optimized expression tree.
+ -showquery : Show query string.
+ -showrp : Show Runtime plan.
+ -showtet : Show translated expression tree.
+ -timing : Produce timing information.
+ -timing-ignore-queries N : Ignore the first X number of quereies.
+```
+
+**NOTE:** Normally, CLI starts a local VXQuery Server to execute the query. But,
+if you already have a VXQuery Server running, you can send the query to the
+inbuilt *REST Server* running in that server by specifying the **port** and **ip address**
+of the REST Server through options `-rest-ip-address` and `-rest-port`.
diff --git a/vxquery-cli/src/site/site.xml b/vxquery-cli/src/site/site.xml
index 4d35a0f..8b3198d 100644
--- a/vxquery-cli/src/site/site.xml
+++ b/vxquery-cli/src/site/site.xml
@@ -15,34 +15,39 @@
limitations under the License.
-->
<project name="VXQuery">
- <bannerLeft>
- <name>VXQuery</name>
- <src>../images/VXQuery.png</src>
- <href>../index.html</href>
- </bannerLeft>
+ <bannerLeft>
+ <name>VXQuery</name>
+ <src>../images/VXQuery.png</src>
+ <href>../index.html</href>
+ </bannerLeft>
- <bannerRight>
- <name>Apache Software Foundation</name>
- <src>../images/asf_logo_wide.png</src>
- <href>http://www.apache.org/</href>
- </bannerRight>
+ <bannerRight>
+ <name>Apache Software Foundation</name>
+ <src>../images/asf_logo_wide.png</src>
+ <href>http://www.apache.org/</href>
+ </bannerRight>
- <skin>
- <groupId>org.apache.maven.skins</groupId>
- <artifactId>maven-fluido-skin</artifactId>
- <version>1.5</version>
- </skin>
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-fluido-skin</artifactId>
+ <version>1.5</version>
+ </skin>
- <body>
- <menu ref="reports"/>
- <footer><![CDATA[
- <div class="row-fluid">Apache VXQuery, VXQuery, Apache, the Apache
- feather logo, and the Apache VXQuery project logo are either
- registered trademarks or trademarks of The Apache Software
- Foundation in the United States and other countries.
- All other marks mentioned may be trademarks or registered
- trademarks of their respective owners.</div>
- ]]></footer>
- </body>
+ <body>
+ <menu name="VXQuery CLI">
+ <item name="Overview" href="index.html"/>
+ </menu>
+
+ <menu ref="reports"/>
+ <footer><![CDATA[
+ <div class="row-fluid">Apache VXQuery, VXQuery, Apache, the Apache
+ feather logo, and the Apache VXQuery project logo are either
+ registered trademarks or trademarks of The Apache Software
+ Foundation in the United States and other countries.
+ All other marks mentioned may be trademarks or registered
+ trademarks of their respective owners.
+ </div>]]>
+ </footer>
+ </body>
</project>
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java
new file mode 100644
index 0000000..7f748b7
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryRuntimeException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.exceptions;
+
+/**
+ * A runtime exception to be thrown by the VXQuery and related classes of the rest server
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryRuntimeException extends RuntimeException {
+
+ public VXQueryRuntimeException(String message) {
+ super(message);
+ }
+
+ public VXQueryRuntimeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java
new file mode 100644
index 0000000..f05cd6f
--- /dev/null
+++ b/vxquery-core/src/main/java/org/apache/vxquery/exceptions/VXQueryServletRuntimeException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.exceptions;
+
+/**
+ * A runtime exception class to be used to be thrown when runtime errors occur within servlets.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryServletRuntimeException extends VXQueryRuntimeException {
+
+ public VXQueryServletRuntimeException(String message) {
+ super(message);
+ }
+
+ public VXQueryServletRuntimeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
index 5bb9d1a..1527059 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
@@ -285,5 +285,4 @@
}
return indexExists;
}
-
}
diff --git a/vxquery-rest/pom.xml b/vxquery-rest/pom.xml
new file mode 100644
index 0000000..052132f
--- /dev/null
+++ b/vxquery-rest/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>apache-vxquery</artifactId>
+ <groupId>org.apache.vxquery</groupId>
+ <version>0.7-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <packaging>jar</packaging>
+ <name>VXQuery REST Server</name>
+ <description>Apache VXQuery REST Server</description>
+
+ <artifactId>apache-vxquery-rest</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.vxquery</groupId>
+ <artifactId>apache-vxquery-core</artifactId>
+ <version>0.7-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-http</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java
new file mode 100644
index 0000000..f5e0165
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/app/VXQueryApplication.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.app;
+
+import static org.apache.vxquery.rest.Constants.Properties.AVAILABLE_PROCESSORS;
+import static org.apache.vxquery.rest.Constants.Properties.HDFS_CONFIG;
+import static org.apache.vxquery.rest.Constants.Properties.JOIN_HASH_SIZE;
+import static org.apache.vxquery.rest.Constants.Properties.MAXIMUM_DATA_SIZE;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.api.application.ICCApplicationContext;
+import org.apache.hyracks.api.application.ICCApplicationEntryPoint;
+import org.apache.hyracks.api.client.ClusterControllerInfo;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.rest.RestServer;
+import org.apache.vxquery.rest.service.VXQueryConfig;
+import org.apache.vxquery.rest.service.VXQueryService;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+/**
+ * Main class responsible for starting the {@link RestServer} and
+ * {@link VXQueryService} classes.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryApplication implements ICCApplicationEntryPoint {
+
+ private static final Logger LOGGER = Logger.getLogger(VXQueryApplication.class.getName());
+
+ private VXQueryService vxQueryService;
+ private RestServer restServer;
+
+ @Override
+ public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception {
+ AppArgs appArgs = new AppArgs();
+ if (args != null) {
+ CmdLineParser parser = new CmdLineParser(appArgs);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ throw new VXQueryRuntimeException("Unable to parse app arguments", e);
+ }
+ }
+
+ VXQueryConfig config =
+ loadConfiguration(ccAppCtx.getCCContext().getClusterControllerInfo(), appArgs.getVxqueryConfig());
+ vxQueryService = new VXQueryService(config);
+ restServer = new RestServer(vxQueryService, appArgs.getRestPort());
+ }
+
+ public synchronized void stop() {
+ try {
+ LOGGER.log(Level.INFO, "Stopping REST server");
+ restServer.stop();
+
+ LOGGER.log(Level.INFO, "Stopping VXQueryService");
+ vxQueryService.stop();
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when stopping the application", e);
+ }
+ }
+
+ @Override
+ public void startupCompleted() throws Exception {
+ try {
+ LOGGER.log(Level.INFO, "Starting VXQueryService");
+ vxQueryService.start();
+ LOGGER.log(Level.INFO, "VXQueryService started successfully");
+
+ LOGGER.log(Level.INFO, "Starting REST server");
+ restServer.start();
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when starting application", e);
+ stop();
+ throw new VXQueryRuntimeException("Error occurred when starting application", e);
+ }
+ }
+
+ /**
+ * Loads properties from
+ *
+ * <pre>
+ * -appConfig foo/bar.properties
+ * </pre>
+ *
+ * file if specified in the app arguments.
+ *
+ * @param clusterControllerInfo
+ * cluster controller information
+ * @param propertiesFile
+ * vxquery configuration properties file, given by
+ *
+ * <pre>
+ * -appConfig
+ * </pre>
+ *
+ * option in app argument
+ * @return A new {@link VXQueryConfig} instance with either default properties
+ * or properties loaded from the properties file given.
+ */
+ private VXQueryConfig loadConfiguration(ClusterControllerInfo clusterControllerInfo, String propertiesFile) {
+ VXQueryConfig vxqConfig = new VXQueryConfig();
+ if (propertiesFile != null) {
+ try (InputStream in = new FileInputStream(propertiesFile)) {
+ System.getProperties().load(in);
+ } catch (IOException e) {
+ LOGGER.log(Level.SEVERE,
+ String.format("Error occurred when loading properties file %s", propertiesFile), e);
+ }
+ }
+
+ vxqConfig.setAvailableProcessors(Integer.getInteger(AVAILABLE_PROCESSORS, 1));
+ vxqConfig.setJoinHashSize(Long.getLong(JOIN_HASH_SIZE, -1));
+ vxqConfig.setHdfsConf(System.getProperty(HDFS_CONFIG));
+ vxqConfig.setMaximumDataSize(Long.getLong(MAXIMUM_DATA_SIZE, -1));
+
+ vxqConfig.setHyracksClientIp(clusterControllerInfo.getClientNetAddress());
+ vxqConfig.setHyracksClientPort(clusterControllerInfo.getClientNetPort());
+
+ return vxqConfig;
+ }
+
+ public VXQueryService getVxQueryService() {
+ return vxQueryService;
+ }
+
+ public RestServer getRestServer() {
+ return restServer;
+ }
+
+ /**
+ * Application Arguments bean class
+ */
+ private class AppArgs {
+ @Option(name = "-restPort", usage = "The port on which REST server starts")
+ private int restPort = 8080;
+
+ @Option(name = "-appConfig", usage = "Properties file location which includes VXQueryService Application additional configuration")
+ private String vxqueryConfig = null;
+
+ public String getVxqueryConfig() {
+ return vxqueryConfig;
+ }
+
+ public void setVxqueryConfig(String vxqueryConfig) {
+ this.vxqueryConfig = vxqueryConfig;
+ }
+
+ public int getRestPort() {
+ return restPort;
+ }
+
+ public void setRestPort(int restPort) {
+ this.restPort = restPort;
+ }
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java
new file mode 100644
index 0000000..9b18745
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/LocalClusterUtil.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.vxquery.app.util;
+
+import static org.apache.vxquery.rest.Constants.Properties.AVAILABLE_PROCESSORS;
+import static org.apache.vxquery.rest.Constants.Properties.HDFS_CONFIG;
+import static org.apache.vxquery.rest.Constants.Properties.JOIN_HASH_SIZE;
+import static org.apache.vxquery.rest.Constants.Properties.MAXIMUM_DATA_SIZE;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.client.HyracksConnection;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.control.cc.ClusterControllerService;
+import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.vxquery.app.VXQueryApplication;
+import org.apache.vxquery.rest.service.VXQueryConfig;
+import org.apache.vxquery.rest.service.VXQueryService;
+
+/**
+ * A utility class to start the a local hyracks cluster.
+ *
+ * @author Preston Carman
+ */
+public class LocalClusterUtil {
+ /*
+ * Start local virtual cluster with cluster controller node and node controller
+ * nodes. IP address provided for node controller is localhost. Unassigned ports
+ * 39000 and 39001 are used for client and cluster port respectively.
+ */
+ public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 39000;
+ public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 39001;
+ public static final int DEFAULT_HYRACKS_CC_HTTP_PORT = 39002;
+ public static final int DEFAULT_VXQUERY_REST_PORT = 39003;
+
+ // TODO review variable scope after XTest is updated to use the REST service.
+ public ClusterControllerService clusterControllerService;
+ public NodeControllerService nodeControllerSerivce;
+ public IHyracksClientConnection hcc;
+ public IHyracksDataset hds;
+ public VXQueryService vxQueryService;
+
+ public void init(VXQueryConfig config) throws Exception {
+ // Following properties are needed by the app to setup
+ System.setProperty(AVAILABLE_PROCESSORS, String.valueOf(config.getAvailableProcessors()));
+ System.setProperty(JOIN_HASH_SIZE, String.valueOf(config.getJoinHashSize()));
+ System.setProperty(MAXIMUM_DATA_SIZE, String.valueOf(config.getMaximumDataSize()));
+ if (config.getHdfsConf() != null) {
+ System.setProperty(HDFS_CONFIG, config.getHdfsConf());
+ }
+
+ // Cluster controller
+ CCConfig ccConfig = createCCConfig();
+ clusterControllerService = new ClusterControllerService(ccConfig);
+ clusterControllerService.start();
+
+ hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+ hds = new HyracksDataset(hcc, config.getFrameSize(), config.getAvailableProcessors());
+
+ // Node controller
+ NCConfig ncConfig = createNCConfig();
+ nodeControllerSerivce = new NodeControllerService(ncConfig);
+ nodeControllerSerivce.start();
+
+ hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+
+ // REST controller
+ config.setHyracksClientIp(ccConfig.clientNetIpAddress);
+ config.setHyracksClientPort(ccConfig.clientNetPort);
+ vxQueryService = new VXQueryService(config);
+ vxQueryService.start();
+ }
+
+ protected CCConfig createCCConfig() throws IOException {
+ String localAddress = getIpAddress();
+ CCConfig ccConfig = new CCConfig();
+ ccConfig.clientNetIpAddress = localAddress;
+ ccConfig.clientNetPort = DEFAULT_HYRACKS_CC_CLIENT_PORT;
+ ccConfig.clusterNetIpAddress = localAddress;
+ ccConfig.clusterNetPort = DEFAULT_HYRACKS_CC_CLUSTER_PORT;
+ ccConfig.httpPort = DEFAULT_HYRACKS_CC_HTTP_PORT;
+ ccConfig.profileDumpPeriod = 10000;
+ ccConfig.appCCMainClass = VXQueryApplication.class.getName();
+ ccConfig.appArgs = Arrays.asList("-restPort", String.valueOf(DEFAULT_VXQUERY_REST_PORT));
+
+ return ccConfig;
+ }
+
+ protected NCConfig createNCConfig() throws IOException {
+ String localAddress = getIpAddress();
+ NCConfig ncConfig = new NCConfig();
+ ncConfig.ccHost = "localhost";
+ ncConfig.ccPort = DEFAULT_HYRACKS_CC_CLUSTER_PORT;
+ ncConfig.clusterNetIPAddress = localAddress;
+ ncConfig.dataIPAddress = localAddress;
+ ncConfig.resultIPAddress = localAddress;
+ ncConfig.nodeId = "test_node";
+ ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString();
+ return ncConfig;
+ }
+
+ public IHyracksClientConnection getHyracksClientConnection() {
+ return hcc;
+ }
+
+ public VXQueryService getVxQueryService() {
+ return vxQueryService;
+ }
+
+ public void deinit() throws Exception {
+ vxQueryService.stop();
+ nodeControllerSerivce.stop();
+ clusterControllerService.stop();
+ }
+
+ public static void main(String[] args) {
+ LocalClusterUtil localClusterUtil = new LocalClusterUtil();
+ VXQueryConfig config = new VXQueryConfig();
+ run(localClusterUtil, config);
+ }
+
+ protected static void run(final LocalClusterUtil localClusterUtil, VXQueryConfig config) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ localClusterUtil.deinit();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ try {
+ localClusterUtil.init(config);
+ while (true) {
+ Thread.sleep(10000);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public String getIpAddress() throws UnknownHostException {
+ return InetAddress.getLocalHost().getHostAddress();
+ }
+
+ public int getRestPort() {
+ return DEFAULT_VXQUERY_REST_PORT;
+ }
+
+ @Deprecated
+ public IHyracksClientConnection getConnection() {
+ return hcc;
+ }
+
+ @Deprecated
+ public IHyracksDataset getDataset() {
+ return hds;
+ }
+
+}
\ No newline at end of file
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java
new file mode 100644
index 0000000..fe91836
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/app/util/RestUtils.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.app.util;
+
+import static org.apache.vxquery.rest.Constants.MODE_ASYNC;
+import static org.apache.vxquery.rest.Constants.MODE_SYNC;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+import static org.apache.vxquery.rest.Constants.Parameters.COMPILE_ONLY;
+import static org.apache.vxquery.rest.Constants.Parameters.FRAME_SIZE;
+import static org.apache.vxquery.rest.Constants.Parameters.METRICS;
+import static org.apache.vxquery.rest.Constants.Parameters.MODE;
+import static org.apache.vxquery.rest.Constants.Parameters.OPTIMIZATION;
+import static org.apache.vxquery.rest.Constants.Parameters.REPEAT_EXECUTIONS;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_AST;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_OET;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_RP;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_TET;
+import static org.apache.vxquery.rest.Constants.Parameters.STATEMENT;
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_ENDPOINT;
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_RESULT_ENDPOINT;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+
+/**
+ * A set of utility methods used by the REST related tasks
+ *
+ * @author Erandi Ganepola
+ */
+public class RestUtils {
+
+ private RestUtils() {
+ }
+
+ /**
+ * Builds the {@link URI} once the {@link QueryRequest} is given. Only the
+ * parameters given (different from the default values) are put in the
+ * {@link URI}
+ *
+ * @param request
+ * {@link QueryRequest} to be converted to a {@link URI}
+ * @param restIpAddress
+ * Ip address of the REST server
+ * @param restPort
+ * port of the REST server
+ * @return generated {@link URI}
+ * @throws URISyntaxException
+ */
+ public static URI buildQueryURI(QueryRequest request, String restIpAddress, int restPort)
+ throws URISyntaxException {
+ URIBuilder builder =
+ new URIBuilder().setScheme("http").setHost(restIpAddress).setPort(restPort).setPath(QUERY_ENDPOINT);
+
+ if (request.getStatement() != null) {
+ builder.addParameter(STATEMENT, request.getStatement());
+ }
+ if (request.isCompileOnly()) {
+ builder.addParameter(COMPILE_ONLY, String.valueOf(request.isCompileOnly()));
+ }
+ if (request.getOptimization() != QueryRequest.DEFAULT_OPTIMIZATION) {
+ builder.addParameter(OPTIMIZATION, String.valueOf(request.getOptimization()));
+ }
+ if (request.getFrameSize() != QueryRequest.DEFAULT_FRAMESIZE) {
+ builder.addParameter(FRAME_SIZE, String.valueOf(request.getFrameSize()));
+ }
+ if (request.getRepeatExecutions() != 1) {
+ builder.addParameter(REPEAT_EXECUTIONS, String.valueOf(request.getRepeatExecutions()));
+ }
+ if (request.isShowMetrics()) {
+ builder.addParameter(METRICS, String.valueOf(request.isShowMetrics()));
+ }
+ if (request.isShowAbstractSyntaxTree()) {
+ builder.addParameter(SHOW_AST, String.valueOf(request.isShowAbstractSyntaxTree()));
+ }
+ if (request.isShowTranslatedExpressionTree()) {
+ builder.addParameter(SHOW_TET, String.valueOf(request.isShowTranslatedExpressionTree()));
+ }
+ if (request.isShowOptimizedExpressionTree()) {
+ builder.addParameter(SHOW_OET, String.valueOf(request.isShowOptimizedExpressionTree()));
+ }
+ if (request.isShowRuntimePlan()) {
+ builder.addParameter(SHOW_RP, String.valueOf(request.isShowRuntimePlan()));
+ }
+ if (!request.isAsync()) {
+ builder.addParameter(MODE, request.isAsync() ? MODE_ASYNC : MODE_SYNC);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Builds the query result {@link URI} given the {@link QueryResultRequest}
+ *
+ * @param resultRequest
+ * result request
+ * @param restIpAddress
+ * rest server's ip
+ * @param restPort
+ * port of the rest server
+ * @return generated {@link URI}
+ * @throws URISyntaxException
+ */
+ public static URI buildQueryResultURI(QueryResultRequest resultRequest, String restIpAddress, int restPort)
+ throws URISyntaxException {
+ URIBuilder builder = new URIBuilder().setScheme("http").setHost(restIpAddress).setPort(restPort)
+ .setPath(QUERY_RESULT_ENDPOINT.replace("*", String.valueOf(resultRequest.getResultId())));
+
+ if (resultRequest.isShowMetrics()) {
+ builder.setParameter(METRICS, String.valueOf(resultRequest.isShowMetrics()));
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Reads the entity from an {@link HttpEntity}
+ *
+ * @param entity
+ * entity instance to be read
+ * @return entity read by this method as a string
+ * @throws IOException
+ */
+ public static String readEntity(HttpEntity entity) throws IOException {
+ StringBuilder responseBody = new StringBuilder();
+
+ try (InputStream in = entity.getContent()) {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ responseBody.append(line);
+ }
+ }
+ return responseBody.toString();
+ }
+
+ /**
+ * Maps the object in the string representation to a java object. To map json
+ * entities, this method use {@link ObjectMapper}. For XML this method use
+ * {@link Unmarshaller}.
+ *
+ * @param entity
+ * string representation of the object
+ * @param type
+ * the class to which the string needs to be mapped to
+ * @param contentType
+ * json or XML
+ * @param <T>
+ * content's class type
+ * @return mapped object
+ * @throws IOException
+ * @throws JAXBException
+ */
+ public static <T> T mapEntity(String entity, Class<T> type, String contentType) throws IOException, JAXBException {
+ if (contentType == null) {
+ contentType = CONTENT_TYPE_JSON;
+ }
+
+ switch (contentType) {
+ case CONTENT_TYPE_XML:
+ JAXBContext jaxbContext = JAXBContext.newInstance(type);
+ Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
+ return type.cast(unmarshaller.unmarshal(new StringReader(entity)));
+ case CONTENT_TYPE_JSON:
+ default:
+ ObjectMapper jsonMapper = new ObjectMapper();
+ return jsonMapper.readValue(entity, type);
+ }
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java
new file mode 100644
index 0000000..4ba79ec
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/Constants.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+public class Constants {
+
+ private Constants() {
+ }
+
+ public class Parameters {
+ public static final String STATEMENT = "statement";
+ public static final String RESULT_ID = "resultId ";
+ public static final String COMPILE_ONLY = "compileOnly";
+ public static final String OPTIMIZATION = "optimization";
+ public static final String FRAME_SIZE = "frameSize";
+ public static final String REPEAT_EXECUTIONS = "repeatExecutions";
+ public static final String METRICS = "metrics";
+ public static final String SHOW_AST = "showAbstractSyntaxTree";
+ public static final String SHOW_TET = "showTranslatedExpressionTree";
+ public static final String SHOW_OET = "showOptimizedExpressionTree";
+ public static final String SHOW_RP = "showRuntimePlan";
+ public static final String MODE = "mode";
+ }
+
+ public class URLs {
+ public static final String BASE_PATH = "/vxquery";
+
+ public static final String QUERY_ENDPOINT = BASE_PATH + "/query";
+ public static final String QUERY_RESULT_ENDPOINT = BASE_PATH + "/query/result/*";
+ }
+
+ public class Properties {
+ public static final String AVAILABLE_PROCESSORS = "org.apache.vxquery.available_processors";
+ public static final String LOCAL_NODE_CONTROLLERS = "org.apache.vxquery.local_nc";
+ public static final String JOIN_HASH_SIZE = "org.apache.vxquery.join_hash";
+ public static final String MAXIMUM_DATA_SIZE = "org.apache.vxquery.data_size";
+ public static final String HDFS_CONFIG = "org.apache.vxquery.hdfs_config";
+ }
+
+ public class HttpHeaderValues {
+ public static final String CONTENT_TYPE_JSON = "application/json";
+ public static final String CONTENT_TYPE_XML = "application/xml";
+ }
+
+ public class ErrorCodes {
+ public static final int PROBLEM_WITH_QUERY = 400;
+ public static final int UNFORSEEN_PROBLEM = 500;
+ public static final int INVALID_INPUT = 405;
+ public static final int NOT_FOUND = 404;
+ }
+
+ public static final String RESULT_URL_PREFIX = "/vxquery/query/result/";
+
+ public static final String MODE_ASYNC = "async";
+ public static final String MODE_SYNC = "sync";
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java
new file mode 100644
index 0000000..5b59c0c
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/RestServer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_ENDPOINT;
+import static org.apache.vxquery.rest.Constants.URLs.QUERY_RESULT_ENDPOINT;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.http.server.HttpServer;
+import org.apache.hyracks.http.server.WebManager;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.rest.service.VXQueryService;
+import org.apache.vxquery.rest.servlet.QueryAPIServlet;
+import org.apache.vxquery.rest.servlet.QueryResultAPIServlet;
+
+/**
+ * REST Server class responsible for starting a new server on a given port.
+ *
+ * @author Erandi Ganepola
+ */
+public class RestServer {
+
+ public static final Logger LOGGER = Logger.getLogger(RestServer.class.getName());
+
+ private WebManager webManager;
+ private int port;
+
+ public RestServer(VXQueryService vxQueryService, int port) {
+ if (port == 0) {
+ throw new IllegalArgumentException("REST Server port cannot be 0");
+ }
+
+ this.port = port;
+
+ webManager = new WebManager();
+ HttpServer restServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), this.port);
+ restServer.addServlet(new QueryAPIServlet(vxQueryService, restServer.ctx(), QUERY_ENDPOINT));
+ restServer.addServlet(new QueryResultAPIServlet(vxQueryService, restServer.ctx(), QUERY_RESULT_ENDPOINT));
+ webManager.add(restServer);
+ }
+
+ public void start() {
+ try {
+ LOGGER.log(Level.FINE, "Starting rest server on port: " + port);
+ webManager.start();
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when starting rest server", e);
+ throw new VXQueryRuntimeException("Unable to start REST server", e);
+ }
+ LOGGER.log(Level.INFO, "Rest server started on port: " + port);
+ }
+
+ public void stop() {
+ try {
+ LOGGER.log(Level.FINE, "Stopping rest server");
+ webManager.stop();
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when stopping VXQueryService", e);
+ throw new VXQueryRuntimeException("Error occurred when stopping rest server", e);
+ }
+ LOGGER.log(Level.INFO, "Rest server stopped");
+ }
+
+ public int getPort() {
+ return port;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java
new file mode 100644
index 0000000..a88ae1c
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryRequest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.request;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.vxquery.rest.RestServer;
+
+/**
+ * Request to represent a query request coming to the {@link RestServer}
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryRequest {
+
+ public static final int DEFAULT_FRAMESIZE = 65536;
+ public static final int DEFAULT_OPTIMIZATION = 0;
+
+ private String statement;
+ private boolean async = true;
+ private boolean compileOnly;
+ private int optimization = DEFAULT_OPTIMIZATION;
+ /** Frame size in bytes. (default: 65,536) */
+ private int frameSize = DEFAULT_FRAMESIZE;
+ private int repeatExecutions = 1;
+ private boolean showMetrics = false;
+ private boolean showAbstractSyntaxTree = false;
+ private boolean showTranslatedExpressionTree = false;
+ private boolean showOptimizedExpressionTree = false;
+ private boolean showRuntimePlan = false;
+ /** A unique UUID to uniquely identify a given request */
+ private String requestId;
+
+ /** An optional map of source files. Required for XTests */
+ private Map<String, File> sourceFileMap = new HashMap<>();
+
+ public QueryRequest(String statement) {
+ this(null, statement);
+ }
+
+ public QueryRequest(String requestId, String statement) {
+ if (statement == null) {
+ throw new IllegalArgumentException("Statement cannot be null");
+ }
+
+ this.statement = statement;
+ this.requestId = requestId;
+ }
+
+ public String getStatement() {
+ return statement;
+ }
+
+ public boolean isCompileOnly() {
+ return compileOnly;
+ }
+
+ public void setCompileOnly(boolean compileOnly) {
+ this.compileOnly = compileOnly;
+ }
+
+ public int getOptimization() {
+ return optimization;
+ }
+
+ public void setOptimization(int optimization) {
+ this.optimization = optimization;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public void setFrameSize(int frameSize) {
+ this.frameSize = frameSize;
+ }
+
+ public int getRepeatExecutions() {
+ return repeatExecutions;
+ }
+
+ public void setRepeatExecutions(int repeatExecutions) {
+ this.repeatExecutions = repeatExecutions;
+ }
+
+ public boolean isShowAbstractSyntaxTree() {
+ return showAbstractSyntaxTree;
+ }
+
+ public void setShowAbstractSyntaxTree(boolean showAbstractSyntaxTree) {
+ this.showAbstractSyntaxTree = showAbstractSyntaxTree;
+ }
+
+ public boolean isShowTranslatedExpressionTree() {
+ return showTranslatedExpressionTree;
+ }
+
+ public void setShowTranslatedExpressionTree(boolean showTranslatedExpressionTree) {
+ this.showTranslatedExpressionTree = showTranslatedExpressionTree;
+ }
+
+ public boolean isShowOptimizedExpressionTree() {
+ return showOptimizedExpressionTree;
+ }
+
+ public void setShowOptimizedExpressionTree(boolean showOptimizedExpressionTree) {
+ this.showOptimizedExpressionTree = showOptimizedExpressionTree;
+ }
+
+ public boolean isShowRuntimePlan() {
+ return showRuntimePlan;
+ }
+
+ public void setShowRuntimePlan(boolean showRuntimePlan) {
+ this.showRuntimePlan = showRuntimePlan;
+ }
+
+ public boolean isShowMetrics() {
+ return showMetrics;
+ }
+
+ public void setShowMetrics(boolean showMetrics) {
+ this.showMetrics = showMetrics;
+ }
+
+ public String toString() {
+ return String.format("{ statement : %s }", statement);
+ }
+
+ public String getRequestId() {
+ return requestId;
+ }
+
+ public boolean isAsync() {
+ return async;
+ }
+
+ public void setAsync(boolean async) {
+ this.async = async;
+ }
+
+ public Map<String, File> getSourceFileMap() {
+ return sourceFileMap;
+ }
+
+ public void setSourceFileMap(Map<String, File> sourceFileMap) {
+ this.sourceFileMap = sourceFileMap;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java
new file mode 100644
index 0000000..5e43181
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/request/QueryResultRequest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.request;
+
+import org.apache.vxquery.rest.RestServer;
+
+/**
+ * Request to represent a query request coming to the {@link RestServer}
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryResultRequest {
+
+ private long resultId;
+ private boolean showMetrics = false;
+ private String requestId;
+
+ public QueryResultRequest(long resultId) {
+ this(resultId, null);
+ }
+
+ public QueryResultRequest(long resultId, String requestId) {
+ this.resultId = resultId;
+ this.requestId = requestId;
+ }
+
+ public long getResultId() {
+ return resultId;
+ }
+
+ public boolean isShowMetrics() {
+ return showMetrics;
+ }
+
+ public void setShowMetrics(boolean showMetrics) {
+ this.showMetrics = showMetrics;
+ }
+
+ public String getRequestId() {
+ return requestId;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java
new file mode 100644
index 0000000..57b67b7
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/APIResponse.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import static org.apache.vxquery.rest.Constants.RESULT_URL_PREFIX;
+
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.service.Status;
+
+/**
+ * Base class for any type of response which can be sent by the REST server.
+ * These responses can be query responses, error responses or query result
+ * responses.
+ *
+ * @author Erandi Ganepola
+ */
+public class APIResponse {
+
+ private String status;
+ private String requestId;
+
+ public APIResponse() {
+ status = Status.SUCCESS.toString();
+ }
+
+ public APIResponse(String status) {
+ this.status = status;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public String getRequestId() {
+ return requestId;
+ }
+
+ public void setRequestId(String requestId) {
+ this.requestId = requestId;
+ }
+
+ public static ErrorResponse newErrorResponse(String requestId, Error error) {
+ ErrorResponse response = new ErrorResponse();
+ response.setRequestId(requestId);
+ response.setError(error);
+ return response;
+ }
+
+ public static QueryResponse newQueryResponse(QueryRequest request, ResultSetId resultSetId) {
+ QueryResponse response;
+ if (request.isAsync()) {
+ AsyncQueryResponse asyncQueryResponse = new AsyncQueryResponse();
+ if (!request.isCompileOnly()) {
+ asyncQueryResponse.setResultId(resultSetId.getId());
+ asyncQueryResponse.setResultUrl(RESULT_URL_PREFIX + resultSetId.getId());
+ }
+ response = asyncQueryResponse;
+ } else {
+ response = new SyncQueryResponse();
+ }
+
+ response.setRequestId(request.getRequestId());
+ return response;
+ }
+
+ public static QueryResultResponse newQueryResultResponse(String requestId) {
+ QueryResultResponse response = new QueryResultResponse();
+ response.setRequestId(requestId);
+ return response;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java
new file mode 100644
index 0000000..3216dce
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/AsyncQueryResponse.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Resource class to represent a response to a given user query
+ *
+ * @author Erandi Ganepola
+ */
+@XmlRootElement
+public class AsyncQueryResponse extends QueryResponse {
+
+ private long resultId;
+ private String resultUrl;
+
+ public long getResultId() {
+ return resultId;
+ }
+
+ public void setResultId(long resultId) {
+ this.resultId = resultId;
+ }
+
+ public String getResultUrl() {
+ return resultUrl;
+ }
+
+ public void setResultUrl(String resultUrl) {
+ this.resultUrl = resultUrl;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java
new file mode 100644
index 0000000..96a709d
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Error.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Represents the
+ *
+ * <pre>
+ * error
+ * </pre>
+ *
+ * part of an {@link ErrorResponse}.
+ *
+ * <pre>
+ * {@code
+ * <error>
+ * <code>405</code>
+ * <message>Invalid Input</message>
+ * </error>}
+ * </pre>
+ *
+ * @author Erandi Ganepola
+ */
+@XmlRootElement
+public class Error {
+
+ private int code;
+ private String message;
+
+ public Error() {
+ }
+
+ public Error(int code, String message) {
+ this.code = code;
+ this.message = message;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public void setCode(int code) {
+ this.code = code;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private int code = -1;
+ private String message = null;
+
+ public Builder withCode(int code) {
+ this.code = code;
+ return this;
+ }
+
+ public Builder withMessage(String message) {
+ this.message = message;
+ return this;
+ }
+
+ public Error build() {
+ if (code == -1) {
+ code = 500;
+ }
+
+ if (message == null) {
+ message = "unexpected Error";
+ }
+
+ return new Error(code, message);
+ }
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java
new file mode 100644
index 0000000..2de2b25
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/ErrorResponse.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.vxquery.rest.service.Status;
+
+/**
+ * <pre>
+ * {@code
+ * <errorResponse>
+ * <status>FATAL</status>
+ * <requestId>jabsa-jkk77j-hbah45-jknasj-jjlas</requestId>
+ * <error>
+ * <code>405</code>
+ * <message>Invalid Input</message>
+ * </error>
+ * </errorResponse>
+ * }
+ * </pre>
+ *
+ * @author Erandi Ganepola
+ */
+@XmlRootElement
+public class ErrorResponse extends APIResponse {
+
+ private Error error;
+
+ public ErrorResponse() {
+ super(Status.FATAL.toString());
+ }
+
+ public Error getError() {
+ return error;
+ }
+
+ public void setError(Error error) {
+ this.error = error;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java
new file mode 100644
index 0000000..e34e1c6
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/Metrics.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+public class Metrics {
+ private long compileTime;
+ private long elapsedTime;
+
+ public long getCompileTime() {
+ return compileTime;
+ }
+
+ public void setCompileTime(long compileTime) {
+ this.compileTime = compileTime;
+ }
+
+ public long getElapsedTime() {
+ return elapsedTime;
+ }
+
+ public void setElapsedTime(long elapsedTime) {
+ this.elapsedTime = elapsedTime;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java
new file mode 100644
index 0000000..4916ffe
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResponse.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import org.apache.vxquery.rest.service.Status;
+
+/**
+ * The base class of the query response (the response returned when a query is
+ * sent for execution)
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryResponse extends APIResponse {
+
+ private String statement;
+ private String abstractSyntaxTree;
+ private String translatedExpressionTree;
+ private String optimizedExpressionTree;
+ private String runtimePlan;
+ private Metrics metrics = new Metrics();
+
+ public QueryResponse() {
+ super(Status.SUCCESS.toString());
+ }
+
+ public String getStatement() {
+ return statement;
+ }
+
+ public void setStatement(String statement) {
+ this.statement = statement;
+ }
+
+ public String getAbstractSyntaxTree() {
+ return abstractSyntaxTree;
+ }
+
+ public void setAbstractSyntaxTree(String abstractSyntaxTree) {
+ this.abstractSyntaxTree = abstractSyntaxTree;
+ }
+
+ public String getTranslatedExpressionTree() {
+ return translatedExpressionTree;
+ }
+
+ public void setTranslatedExpressionTree(String translatedExpressionTree) {
+ this.translatedExpressionTree = translatedExpressionTree;
+ }
+
+ public String getOptimizedExpressionTree() {
+ return optimizedExpressionTree;
+ }
+
+ public void setOptimizedExpressionTree(String optimizedExpressionTree) {
+ this.optimizedExpressionTree = optimizedExpressionTree;
+ }
+
+ public String getRuntimePlan() {
+ return runtimePlan;
+ }
+
+ public void setRuntimePlan(String runtimePlan) {
+ this.runtimePlan = runtimePlan;
+ }
+
+ public Metrics getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(Metrics metrics) {
+ this.metrics = metrics;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java
new file mode 100644
index 0000000..2f74865
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/QueryResultResponse.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.vxquery.rest.service.Status;
+
+@XmlRootElement
+public class QueryResultResponse extends APIResponse {
+
+ private String results;
+ private Metrics metrics = new Metrics();
+
+ public QueryResultResponse() {
+ super(Status.SUCCESS.toString());
+ }
+
+ public String getResults() {
+ return results;
+ }
+
+ public void setResults(String results) {
+ this.results = results;
+ }
+
+ public Metrics getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(Metrics metrics) {
+ this.metrics = metrics;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java
new file mode 100644
index 0000000..b42a912
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/response/SyncQueryResponse.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.response;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+public class SyncQueryResponse extends QueryResponse {
+
+ private String results;
+
+ public String getResults() {
+ return results;
+ }
+
+ public void setResults(String results) {
+ this.results = results;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java
new file mode 100644
index 0000000..9dc967f
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/HyracksJobContext.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.job.JobId;
+
+/**
+ * A class to map {@link ResultSetId} with {@link JobId} when a job is submitted
+ * to hyracks. This mapping will later be used to determine the {@link JobId}
+ * instance of the corresponding {@link ResultSetId}
+ *
+ * @author Erandi Ganepola
+ */
+public class HyracksJobContext {
+
+ private JobId jobId;
+ private int frameSize;
+ private ResultSetId resultSetId;
+
+ public HyracksJobContext(JobId jobId, int frameSize, ResultSetId resultSetId) {
+ this.jobId = jobId;
+ this.frameSize = frameSize;
+ this.resultSetId = resultSetId;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public ResultSetId getResultSetId() {
+ return resultSetId;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java
new file mode 100644
index 0000000..d6b4b3c
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/State.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+/**
+ * An enum to represent states of {@link VXQueryService} class
+ *
+ * @author Erandi Ganepola
+ */
+public enum State {
+ STARTING,
+ STARTED,
+ STOPPING,
+ STOPPED
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java
new file mode 100644
index 0000000..5757ae7
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/Status.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+/**
+ * There can only 2 states for a response from the
+ * {@link org.apache.vxquery.rest.RestServer}. They are,
+ *
+ * <pre>
+ * SUCCESS
+ * </pre>
+ *
+ * and
+ *
+ * <pre>
+ * FATAL
+ * </pre>
+ *
+ * . This enum represents those two types.
+ *
+ * @author Erandi Ganepola
+ */
+public enum Status {
+ SUCCESS("success"),
+ FATAL("fatal");
+
+ private final String name;
+
+ Status(String name) {
+ this.name = name;
+ }
+
+ public String toString() {
+ return name;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java
new file mode 100644
index 0000000..4f20c6b
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryConfig.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.service;
+
+/**
+ * A class to store default/user specified configurations required at runtime by
+ * the {@link VXQueryService} class. These configuration will be loaded through
+ * a properties file.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryConfig {
+
+ /** Number of available processors. (default: java's available processors) */
+ private int availableProcessors = Runtime.getRuntime().availableProcessors();
+ /** Setting frame size. (default: 65,536) */
+ private int frameSize = 65536;
+ /** Join hash size in bytes. (default: 67,108,864) */
+ private long joinHashSize = -1;
+ /** Maximum possible data size in bytes. (default: 150,323,855,000) */
+ private long maximumDataSize = -1;
+ /** Directory path to Hadoop configuration files */
+ private String hdfsConf = null;
+
+ private String hyracksClientIp;
+ private int hyracksClientPort;
+
+ public int getAvailableProcessors() {
+ return availableProcessors;
+ }
+
+ public void setAvailableProcessors(int availableProcessors) {
+ if (availableProcessors > 0) {
+ this.availableProcessors = availableProcessors;
+ }
+ }
+
+ public long getJoinHashSize() {
+ return joinHashSize;
+ }
+
+ public void setJoinHashSize(long joinHashSize) {
+ this.joinHashSize = joinHashSize;
+ }
+
+ public long getMaximumDataSize() {
+ return maximumDataSize;
+ }
+
+ public void setMaximumDataSize(long maximumDataSize) {
+ this.maximumDataSize = maximumDataSize;
+ }
+
+ public String getHdfsConf() {
+ return hdfsConf;
+ }
+
+ public void setHdfsConf(String hdfsConf) {
+ this.hdfsConf = hdfsConf;
+ }
+
+ public int getHyracksClientPort() {
+ return hyracksClientPort;
+ }
+
+ public void setHyracksClientPort(int hyracksClientPort) {
+ this.hyracksClientPort = hyracksClientPort;
+ }
+
+ public String getHyracksClientIp() {
+ return hyracksClientIp;
+ }
+
+ public void setHyracksClientIp(String hyracksClientIp) {
+ this.hyracksClientIp = hyracksClientIp;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public void setFrameSize(int frameSize) {
+ this.frameSize = frameSize;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java
new file mode 100644
index 0000000..1d51b6a
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/service/VXQueryService.java
@@ -0,0 +1,482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.service;
+
+import static java.util.logging.Level.SEVERE;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.NOT_FOUND;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.PROBLEM_WITH_QUERY;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.UNFORSEEN_PROBLEM;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import org.apache.hyracks.api.client.HyracksConnection;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.client.NodeControllerInfo;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameTupleAccessor;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.dataset.DatasetJobRecord;
+import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.job.JobFlag;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.control.nc.resources.memory.FrameManager;
+import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
+import org.apache.vxquery.compiler.CompilerControlBlock;
+import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory;
+import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressionPrettyPrintVisitor;
+import org.apache.vxquery.context.DynamicContext;
+import org.apache.vxquery.context.DynamicContextImpl;
+import org.apache.vxquery.context.RootStaticContextImpl;
+import org.apache.vxquery.context.StaticContextImpl;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.response.QueryResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.result.ResultUtils;
+import org.apache.vxquery.xmlquery.ast.ModuleNode;
+import org.apache.vxquery.xmlquery.query.Module;
+import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
+import org.apache.vxquery.xmlquery.query.XQueryCompilationListener;
+
+import com.thoughtworks.xstream.XStream;
+import com.thoughtworks.xstream.io.xml.DomDriver;
+
+/**
+ * Main class responsible for handling query requests. This class will first
+ * compile, then submit query to hyracks and finally fetch results for a given
+ * query.
+ *
+ * @author Erandi Ganepola
+ */
+public class VXQueryService {
+
+ private static final Logger LOGGER = Logger.getLogger(VXQueryService.class.getName());
+
+ private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
+
+ private volatile State state = State.STOPPED;
+ private VXQueryConfig vxQueryConfig;
+ private AtomicLong atomicLong = new AtomicLong(0);
+ private Map<Long, HyracksJobContext> jobContexts = new ConcurrentHashMap<>();
+ private IHyracksClientConnection hyracksClientConnection;
+ private HyracksDataset hyracksDataset;
+
+ public VXQueryService(VXQueryConfig config) {
+ vxQueryConfig = config;
+ }
+
+ /**
+ * Starts VXQueryService class by creating a {@link IHyracksClientConnection}
+ * which will later be used to submit and retrieve queries and results to/from
+ * hyracks.
+ */
+ public synchronized void start() {
+ if (!State.STOPPED.equals(state)) {
+ throw new IllegalStateException("VXQueryService is at state : " + state);
+ }
+
+ if (vxQueryConfig.getHyracksClientIp() == null) {
+ throw new IllegalArgumentException("hyracksClientIp is required to connect to hyracks");
+ }
+
+ setState(State.STARTING);
+
+ try {
+ hyracksClientConnection =
+ new HyracksConnection(vxQueryConfig.getHyracksClientIp(), vxQueryConfig.getHyracksClientPort());
+ } catch (Exception e) {
+ LOGGER.log(SEVERE, String.format("Unable to create a hyracks client connection to %s:%d",
+ vxQueryConfig.getHyracksClientIp(), vxQueryConfig.getHyracksClientPort()));
+ throw new VXQueryRuntimeException("Unable to create a hyracks client connection", e);
+ }
+
+ LOGGER.log(Level.FINE, String.format("Using hyracks connection to %s:%d", vxQueryConfig.getHyracksClientIp(),
+ vxQueryConfig.getHyracksClientPort()));
+
+ setState(State.STARTED);
+ LOGGER.log(Level.INFO, "VXQueryService started successfully");
+ }
+
+ private synchronized void setState(State newState) {
+ state = newState;
+ }
+
+ /**
+ * Submits a query to hyracks to be run after compiling. Required intermediate
+ * results and metrics are also calculated according to the
+ * {@link QueryRequest}. Checks if this class has started before moving further.
+ *
+ * @param request
+ * {@link QueryRequest} containing information about the query to be
+ * executed and the merics required along with the results
+ * @return AsyncQueryResponse if no error occurs | ErrorResponse else
+ */
+ public APIResponse execute(final QueryRequest request) {
+ QueryRequest indexingRequest = new QueryRequest("show-indexes()");
+ indexingRequest.setAsync(false);
+ SyncQueryResponse indexingResponse = (SyncQueryResponse) execute(indexingRequest, new ArrayList<>());
+ LOGGER.log(Level.FINE, String.format("Found indexes: %s", indexingResponse.getResults()));
+
+ List<String> collections = Arrays.asList(indexingResponse.getResults().split("\n"));
+ return execute(request, collections);
+ }
+
+ private APIResponse execute(final QueryRequest request, List<String> collections) {
+ if (!State.STARTED.equals(state)) {
+ throw new IllegalStateException("VXQueryService is at state : " + state);
+ }
+
+ String query = request.getStatement();
+ final ResultSetId resultSetId = createResultSetId();
+
+ QueryResponse response = APIResponse.newQueryResponse(request, resultSetId);
+ response.setStatement(query);
+
+ // Obtaining the node controller information from hyracks client connection
+ Map<String, NodeControllerInfo> nodeControllerInfos = null;
+ try {
+ nodeControllerInfos = hyracksClientConnection.getNodeControllerInfos();
+ } catch (HyracksException e) {
+ LOGGER.log(Level.SEVERE, String.format("Error occurred when obtaining NC info: '%s'", e.getMessage()));
+ return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(UNFORSEEN_PROBLEM)
+ .withMessage("Hyracks connection problem: " + e.getMessage()).build());
+ }
+
+ // Adding a query compilation listener
+ VXQueryCompilationListener listener = new VXQueryCompilationListener(response,
+ request.isShowAbstractSyntaxTree(), request.isShowTranslatedExpressionTree(),
+ request.isShowOptimizedExpressionTree(), request.isShowRuntimePlan());
+
+ Date start = new Date();
+ // Compiling the XQuery given
+ final XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, request.getFrameSize(),
+ vxQueryConfig.getAvailableProcessors(), vxQueryConfig.getJoinHashSize(),
+ vxQueryConfig.getMaximumDataSize(), vxQueryConfig.getHdfsConf());
+ CompilerControlBlock compilerControlBlock = new CompilerControlBlock(
+ new StaticContextImpl(RootStaticContextImpl.INSTANCE), resultSetId, request.getSourceFileMap());
+ try {
+ compiler.compile(null, new StringReader(query), compilerControlBlock, request.getOptimization(),
+ collections);
+ } catch (AlgebricksException e) {
+ LOGGER.log(Level.SEVERE, String.format("Error occurred when compiling query: '%s' with message: '%s'",
+ query, e.getMessage()));
+ return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(PROBLEM_WITH_QUERY)
+ .withMessage("Query compilation failure: " + e.getMessage()).build());
+ } catch (SystemException e) {
+ LOGGER.log(Level.SEVERE, String.format("Error occurred when compiling query: '%s' with message: '%s'",
+ query, e.getMessage()));
+ return APIResponse.newErrorResponse(request.getRequestId(),
+ new Error(PROBLEM_WITH_QUERY, "Query compilation failure: " + e.getCode()));
+ }
+
+ if (request.isShowMetrics()) {
+ response.getMetrics().setCompileTime(new Date().getTime() - start.getTime());
+ }
+
+ if (request.isCompileOnly()) {
+ return response;
+ }
+
+ Module module = compiler.getModule();
+ JobSpecification js = module.getHyracksJobSpecification();
+ DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext());
+ js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
+
+ HyracksJobContext hyracksJobContext;
+ start = new Date();
+ if (!request.isAsync()) {
+ for (int i = 0; i < request.getRepeatExecutions(); i++) {
+ try {
+ hyracksJobContext = executeJob(js, resultSetId, request);
+
+ } catch (Exception e) {
+ LOGGER.log(SEVERE, "Error occurred when submitting job to hyracks for query: " + query, e);
+ return APIResponse.newErrorResponse(request.getRequestId(),
+ Error.builder().withCode(UNFORSEEN_PROBLEM)
+ .withMessage("Error occurred when starting hyracks job").build());
+ }
+ try {
+ String results = readResults(hyracksJobContext);
+ ((SyncQueryResponse) response).setResults(results);
+ } catch (HyracksException e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when reading results", e);
+ SystemException se = getSystemException(e);
+ return APIResponse.newErrorResponse(request.getRequestId(), new Error(UNFORSEEN_PROBLEM,
+ String.format("Error occurred when reading results: %s", se != null ? se.getCode() : "")));
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when reading results", e);
+ return APIResponse.newErrorResponse(request.getRequestId(),
+ new Error(UNFORSEEN_PROBLEM, "Error occurred when reading results: " + e.getMessage()));
+ }
+ }
+ } else {
+ try {
+ hyracksJobContext = executeJob(js, resultSetId, request);
+ } catch (Exception e) {
+ LOGGER.log(SEVERE, "Error occurred when submitting job to hyracks for query: " + query, e);
+ return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(UNFORSEEN_PROBLEM)
+ .withMessage("Error occurred when starting hyracks job").build());
+ }
+ jobContexts.put(resultSetId.getId(), hyracksJobContext);
+ }
+
+ if (request.isShowMetrics()) {
+ response.getMetrics().setElapsedTime(new Date().getTime() - start.getTime());
+ }
+
+ return response;
+ }
+
+ private HyracksJobContext executeJob(JobSpecification js, ResultSetId resultSetId, QueryRequest request)
+ throws Exception {
+ HyracksJobContext hyracksJobContext;
+ JobId jobId = hyracksClientConnection.startJob(js, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+ hyracksJobContext = new HyracksJobContext(jobId, js.getFrameSize(), resultSetId);
+
+ return hyracksJobContext;
+ }
+
+ private static SystemException getSystemException(HyracksException e) {
+ Throwable t = e;
+ Throwable candidate = t instanceof SystemException ? t : null;
+ while (t.getCause() != null) {
+ t = t.getCause();
+ if (t instanceof SystemException) {
+ candidate = t;
+ }
+ }
+
+ t = candidate == null ? t : candidate;
+ final String message = t.getMessage();
+ if (message != null) {
+ Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message);
+ if (m.find()) {
+ String eCode = m.group(1);
+ return new SystemException(ErrorCode.valueOf(eCode), e);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the query results for a given result set id.
+ *
+ * @param request
+ * {@link QueryResultRequest} with result ID required
+ * @return Either a {@link QueryResultResponse} if no error occurred |
+ * {@link org.apache.vxquery.rest.response.ErrorResponse} else.
+ */
+ public APIResponse getResult(QueryResultRequest request) {
+ if (jobContexts.containsKey(request.getResultId())) {
+ QueryResultResponse resultResponse = APIResponse.newQueryResultResponse(request.getRequestId());
+ Date start = new Date();
+ try {
+ String results = readResults(jobContexts.get(request.getResultId()));
+ resultResponse.setResults(results);
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when reading results for id : " + request.getResultId());
+ return APIResponse.newErrorResponse(request.getRequestId(), new Error(UNFORSEEN_PROBLEM,
+ "Error occurred when reading results for: " + request.getResultId()));
+ }
+
+ if (request.isShowMetrics()) {
+ resultResponse.getMetrics().setElapsedTime(new Date().getTime() - start.getTime());
+ }
+
+ return resultResponse;
+ } else {
+ return APIResponse.newErrorResponse(request.getRequestId(), Error.builder().withCode(NOT_FOUND)
+ .withMessage("No query found for result ID : " + request.getResultId()).build());
+ }
+ }
+
+ /**
+ * Reads results from hyracks given the {@link HyracksJobContext} containing
+ * {@link ResultSetId} and {@link JobId} mapping.
+ *
+ * @param jobContext
+ * mapoing between the {@link ResultSetId} and corresponding hyracks
+ * {@link JobId}
+ * @return Results of the given query
+ * @throws Exception
+ * IOErrors and etc
+ */
+ private String readResults(HyracksJobContext jobContext) throws Exception {
+ int nReaders = 1;
+
+ if (hyracksDataset == null) {
+ hyracksDataset = new HyracksDataset(hyracksClientConnection, jobContext.getFrameSize(), nReaders);
+ }
+
+ FrameManager resultDisplayFrameMgr = new FrameManager(jobContext.getFrameSize());
+ IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
+ IHyracksDatasetReader reader = hyracksDataset.createReader(jobContext.getJobId(), jobContext.getResultSetId());
+ OutputStream resultStream = new ByteArrayOutputStream();
+
+ // This loop is required for XTests to reliably identify the error code of
+ // SystemException.
+ while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) {
+ Thread.sleep(100);
+ }
+
+ IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
+ try (PrintWriter writer = new PrintWriter(resultStream, true)) {
+ while (reader.read(frame) > 0) {
+ writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor));
+ writer.flush();
+ frame.getBuffer().clear();
+ }
+ }
+
+ hyracksClientConnection.waitForCompletion(jobContext.getJobId());
+ LOGGER.log(Level.FINE, String.format("Result for resultId %d completed", jobContext.getResultSetId().getId()));
+ return resultStream.toString();
+ }
+
+ /**
+ * Create a unique result set id to get the correct query back from the cluster.
+ *
+ * @return Result Set id generated with current system time.
+ */
+ protected ResultSetId createResultSetId() {
+ long resultSetId = atomicLong.incrementAndGet();
+ LOGGER.log(Level.FINE, String.format("Creating result set with ID : %d", resultSetId));
+ return new ResultSetId(resultSetId);
+ }
+
+ public synchronized void stop() {
+ if (!State.STOPPED.equals(state)) {
+ setState(State.STOPPING);
+ LOGGER.log(Level.FINE, "Stooping VXQueryService");
+ setState(State.STOPPED);
+ LOGGER.log(Level.INFO, "VXQueryService stopped successfully");
+ } else {
+ LOGGER.log(Level.INFO, "VXQueryService is already in state : " + state);
+ }
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ /**
+ * A {@link XQueryCompilationListener} implementation to be used to add
+ * AbstractSyntaxTree, RuntimePlan and etc to the {@link QueryResponse} if
+ * requested by the user.
+ */
+ private class VXQueryCompilationListener implements XQueryCompilationListener {
+ private QueryResponse response;
+ private boolean showAbstractSyntaxTree;
+ private boolean showTranslatedExpressionTree;
+ private boolean showOptimizedExpressionTree;
+ private boolean showRuntimePlan;
+
+ public VXQueryCompilationListener(QueryResponse response, boolean showAbstractSyntaxTree,
+ boolean showTranslatedExpressionTree, boolean showOptimizedExpressionTree, boolean showRuntimePlan) {
+ this.response = response;
+ this.showAbstractSyntaxTree = showAbstractSyntaxTree;
+ this.showTranslatedExpressionTree = showTranslatedExpressionTree;
+ this.showOptimizedExpressionTree = showOptimizedExpressionTree;
+ this.showRuntimePlan = showRuntimePlan;
+ }
+
+ @Override
+ public void notifyParseResult(ModuleNode moduleNode) {
+ if (showAbstractSyntaxTree) {
+ response.setAbstractSyntaxTree(new XStream(new DomDriver()).toXML(moduleNode));
+ }
+ }
+
+ @Override
+ public void notifyTranslationResult(Module module) {
+ if (showTranslatedExpressionTree) {
+ response.setTranslatedExpressionTree(appendPrettyPlan(new StringBuilder(), module).toString());
+ }
+ }
+
+ @Override
+ public void notifyTypecheckResult(Module module) {
+ }
+
+ @Override
+ public void notifyCodegenResult(Module module) {
+ if (showRuntimePlan) {
+ JobSpecification jobSpec = module.getHyracksJobSpecification();
+ try {
+ response.setRuntimePlan(jobSpec.toJSON().toString());
+ } catch (IOException e) {
+ LOGGER.log(SEVERE,
+ "Error occurred when obtaining runtime plan from job specification : " + jobSpec.toString(),
+ e);
+ }
+ }
+ }
+
+ @Override
+ public void notifyOptimizedResult(Module module) {
+ if (showOptimizedExpressionTree) {
+ response.setOptimizedExpressionTree(appendPrettyPlan(new StringBuilder(), module).toString());
+ }
+ }
+
+ @SuppressWarnings("Duplicates")
+ private StringBuilder appendPrettyPlan(StringBuilder sb, Module module) {
+ try {
+ ILogicalExpressionVisitor<String, Integer> ev =
+ new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext());
+ AlgebricksAppendable buffer = new AlgebricksAppendable();
+ LogicalOperatorPrettyPrintVisitor v = new LogicalOperatorPrettyPrintVisitor(buffer, ev);
+ PlanPrettyPrinter.printPlan(module.getBody(), v, 0);
+ sb.append(buffer.toString());
+ } catch (AlgebricksException e) {
+ LOGGER.log(SEVERE, "Error occurred when pretty printing expression : " + e.getMessage());
+ }
+ return sb;
+ }
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java
new file mode 100644
index 0000000..45ef910
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryAPIServlet.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.rest.servlet;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static org.apache.vxquery.rest.Constants.MODE_ASYNC;
+import static org.apache.vxquery.rest.Constants.MODE_SYNC;
+import static org.apache.vxquery.rest.Constants.Parameters.COMPILE_ONLY;
+import static org.apache.vxquery.rest.Constants.Parameters.FRAME_SIZE;
+import static org.apache.vxquery.rest.Constants.Parameters.METRICS;
+import static org.apache.vxquery.rest.Constants.Parameters.MODE;
+import static org.apache.vxquery.rest.Constants.Parameters.OPTIMIZATION;
+import static org.apache.vxquery.rest.Constants.Parameters.REPEAT_EXECUTIONS;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_AST;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_OET;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_RP;
+import static org.apache.vxquery.rest.Constants.Parameters.SHOW_TET;
+import static org.apache.vxquery.rest.Constants.Parameters.STATEMENT;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.Constants;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.service.VXQueryService;
+
+/**
+ * Servlet to handle query requests.
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryAPIServlet extends RestAPIServlet {
+
+ private VXQueryService vxQueryService;
+
+ public QueryAPIServlet(VXQueryService vxQueryService, ConcurrentMap<String, Object> ctx, String... paths) {
+ super(ctx, paths);
+ this.vxQueryService = vxQueryService;
+ }
+
+ @Override
+ protected APIResponse doHandle(IServletRequest request) {
+ LOGGER.log(Level.INFO,
+ String.format("Received a query request with query : %s", request.getParameter("statement")));
+
+ QueryRequest queryRequest;
+ try {
+ queryRequest = getQueryRequest(request);
+ } catch (Exception e) {
+ return APIResponse.newErrorResponse(null,
+ Error.builder().withCode(Constants.ErrorCodes.INVALID_INPUT).withMessage("Invalid input").build());
+ }
+
+ try {
+ return vxQueryService.execute(queryRequest);
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when trying to execute query : " + queryRequest.getStatement(), e);
+ return APIResponse.newErrorResponse(queryRequest.getRequestId(), Error.builder()
+ .withCode(Constants.ErrorCodes.UNFORSEEN_PROBLEM).withMessage(e.getMessage()).build());
+ }
+ }
+
+ private QueryRequest getQueryRequest(IServletRequest request) throws IOException, JAXBException {
+ if (request.getParameter(STATEMENT) == null || request.getParameter(STATEMENT).trim().isEmpty()) {
+ throw new IllegalArgumentException("Parameter 'statement' is required to handle the request");
+ }
+
+ QueryRequest queryRequest = new QueryRequest(UUID.randomUUID().toString(), request.getParameter(STATEMENT));
+ queryRequest.setCompileOnly(Boolean.parseBoolean(request.getParameter(COMPILE_ONLY)));
+ queryRequest.setShowMetrics(Boolean.parseBoolean(request.getParameter(METRICS)));
+
+ queryRequest.setShowAbstractSyntaxTree(Boolean.parseBoolean(request.getParameter(SHOW_AST)));
+ queryRequest.setShowTranslatedExpressionTree(Boolean.parseBoolean(request.getParameter(SHOW_TET)));
+ queryRequest.setShowOptimizedExpressionTree(Boolean.parseBoolean(request.getParameter(SHOW_OET)));
+ queryRequest.setShowRuntimePlan(Boolean.parseBoolean(request.getParameter(SHOW_RP)));
+
+ if (request.getParameter(OPTIMIZATION) != null) {
+ queryRequest.setOptimization(Integer.parseInt(request.getParameter(OPTIMIZATION)));
+ }
+ if (request.getParameter(FRAME_SIZE) != null) {
+ queryRequest.setFrameSize(Integer.parseInt(request.getParameter(FRAME_SIZE)));
+ }
+ if (request.getParameter(REPEAT_EXECUTIONS) != null) {
+ queryRequest.setRepeatExecutions(Integer.parseInt(request.getParameter(REPEAT_EXECUTIONS)));
+ }
+
+ String sourceFileMap = request.getHttpRequest().content().toString(StandardCharsets.UTF_8);
+ if (sourceFileMap != null && !sourceFileMap.isEmpty()) {
+ Map<String, String> map = (Map<String, String>) RestUtils.mapEntity(sourceFileMap, Map.class,
+ request.getHeader(CONTENT_TYPE));
+ LOGGER.log(Level.FINE, "Found source file map");
+ Map<String, File> fileMap = map.entrySet().stream()
+ .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), new File(entry.getValue())))
+ .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
+ queryRequest.setSourceFileMap(fileMap);
+ }
+
+ if (request.getParameter(MODE) != null) {
+ switch (request.getParameter(MODE)) {
+ case MODE_SYNC:
+ queryRequest.setAsync(false);
+ break;
+ case MODE_ASYNC:
+ default:
+ queryRequest.setAsync(true);
+ break;
+ }
+ }
+
+ return queryRequest;
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java
new file mode 100644
index 0000000..84b0187
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/QueryResultAPIServlet.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.servlet;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
+
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.vxquery.rest.Constants;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.Error;
+import org.apache.vxquery.rest.service.VXQueryService;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * Servlet to handle query results requests.
+ *
+ * @author Erandi Ganepola
+ */
+public class QueryResultAPIServlet extends RestAPIServlet {
+
+ private VXQueryService vxQueryService;
+
+ public QueryResultAPIServlet(VXQueryService vxQueryService, ConcurrentMap<String, Object> ctx, String... paths) {
+ super(ctx, paths);
+ this.vxQueryService = vxQueryService;
+ }
+
+ @Override
+ protected APIResponse doHandle(IServletRequest request) {
+ String uri = request.getHttpRequest().uri();
+ long resultId;
+ try {
+ String pathParam = uri.substring(uri.lastIndexOf("/") + 1);
+ pathParam = pathParam.contains("?") ? pathParam.split("\\?")[0] : pathParam;
+ resultId = Long.parseLong(pathParam);
+ } catch (NumberFormatException e) {
+ LOGGER.log(Level.SEVERE, "Result ID could not be retrieved from URL");
+ return APIResponse.newErrorResponse(null, Error.builder().withCode(HttpResponseStatus.BAD_REQUEST.code())
+ .withMessage("Result ID couldn't be retrieved from URL").build());
+ }
+
+ QueryResultRequest resultRequest = new QueryResultRequest(resultId, UUID.randomUUID().toString());
+ resultRequest.setShowMetrics(Boolean.parseBoolean(request.getParameter(Constants.Parameters.METRICS)));
+ LOGGER.log(Level.INFO,
+ String.format("Received a result request with resultId : %d", resultRequest.getResultId()));
+ return vxQueryService.getResult(resultRequest);
+ }
+}
diff --git a/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java
new file mode 100644
index 0000000..bc93dfc
--- /dev/null
+++ b/vxquery-rest/src/main/java/org/apache/vxquery/rest/servlet/RestAPIServlet.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest.servlet;
+
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+
+import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
+import org.apache.hyracks.http.server.AbstractServlet;
+import org.apache.hyracks.http.server.utils.HttpUtil;
+import org.apache.vxquery.exceptions.VXQueryRuntimeException;
+import org.apache.vxquery.exceptions.VXQueryServletRuntimeException;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.Status;
+
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * Abstract servlet to handle REST API requests.
+ *
+ * @author Erandi Ganepola
+ */
+public abstract class RestAPIServlet extends AbstractServlet {
+
+ protected final Logger LOGGER;
+
+ private JAXBContext jaxbContext;
+
+ public RestAPIServlet(ConcurrentMap<String, Object> ctx, String... paths) {
+ super(ctx, paths);
+ LOGGER = Logger.getLogger(this.getClass().getName());
+ try {
+ jaxbContext = JAXBContext.newInstance(QueryResultResponse.class, AsyncQueryResponse.class,
+ SyncQueryResponse.class, ErrorResponse.class);
+ } catch (JAXBException e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when creating JAXB context", e);
+ throw new VXQueryRuntimeException("Unable to load JAXBContext", e);
+ }
+ }
+
+ @Override
+ protected final void post(IServletRequest request, IServletResponse response) {
+ getOrPost(request, response);
+ }
+
+ @Override
+ protected final void get(IServletRequest request, IServletResponse response) {
+ getOrPost(request, response);
+ }
+
+ private void getOrPost(IServletRequest request, IServletResponse response) {
+ try {
+ initResponse(request, response);
+ APIResponse entity = doHandle(request);
+ if (entity == null) {
+ LOGGER.log(Level.WARNING, "No entity found for request : " + request);
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ } else {
+ // Important to set Status OK before setting the entity because the response
+ // (chunked) checks it before
+ // writing the response to channel.
+ setResponseStatus(response, entity);
+ setEntity(request, response, entity);
+ }
+ } catch (IOException e) {
+ response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ LOGGER.log(Level.SEVERE, "Error occurred when setting content type", e);
+ }
+ }
+
+ private void initResponse(IServletRequest request, IServletResponse response) throws IOException {
+ // enable cross-origin resource sharing
+ response.setHeader("Access-Control-Allow-Origin", "*");
+ response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+
+ HttpUtil.setContentType(response, "text/plain");
+ }
+
+ private void setEntity(IServletRequest request, IServletResponse response, APIResponse entity) throws IOException {
+ String accept = request.getHeader(HttpHeaderNames.ACCEPT, "");
+ String entityString;
+ switch (accept) {
+ case CONTENT_TYPE_XML:
+ try {
+ HttpUtil.setContentType(response, CONTENT_TYPE_XML);
+
+ Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ StringWriter sw = new StringWriter();
+ jaxbMarshaller.marshal(entity, sw);
+ entityString = sw.toString();
+ } catch (JAXBException e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when mapping java object into xml", e);
+ throw new VXQueryServletRuntimeException("Error occurred when marshalling entity", e);
+ }
+ break;
+ case CONTENT_TYPE_JSON:
+ default:
+ try {
+ HttpUtil.setContentType(response, CONTENT_TYPE_JSON);
+ ObjectMapper jsonMapper = new ObjectMapper();
+ entityString = jsonMapper.writeValueAsString(entity);
+ } catch (JsonProcessingException e) {
+ LOGGER.log(Level.SEVERE, "Error occurred when mapping java object into JSON", e);
+ throw new VXQueryServletRuntimeException("Error occurred when mapping entity", e);
+ }
+ break;
+ }
+
+ response.writer().print(entityString);
+ }
+
+ private void setResponseStatus(IServletResponse response, APIResponse entity) {
+ if (Status.SUCCESS.toString().equals(entity.getStatus())) {
+ response.setStatus(HttpResponseStatus.OK);
+ } else if (Status.FATAL.toString().equals(entity.getStatus())) {
+ HttpResponseStatus status = HttpResponseStatus.INTERNAL_SERVER_ERROR;
+ if (entity instanceof ErrorResponse) {
+ status = HttpResponseStatus.valueOf(((ErrorResponse) entity).getError().getCode());
+ }
+ response.setStatus(status);
+ }
+ }
+
+ /**
+ * This abstract method is supposed to return an object which will be the entity
+ * of the response being sent to the client. Implementing classes doesn't have
+ * to worry about the content type of the request.
+ *
+ * @param request
+ * {@link IServletRequest} received
+ * @return Object to be set as the entity of the response
+ */
+ protected abstract APIResponse doHandle(IServletRequest request);
+}
diff --git a/vxquery-rest/src/site/markdown/index.md b/vxquery-rest/src/site/markdown/index.md
new file mode 100644
index 0000000..703df65
--- /dev/null
+++ b/vxquery-rest/src/site/markdown/index.md
@@ -0,0 +1,249 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# VXQuery REST Server
+
+VXQuery REST Server allows users to submit queries and receive results either synchronously or
+asynchronously through the exposed REST API. Along with the statement to be executed, few other parameters can be given as
+well. Complete REST API specification can be found at [REST API Specification](specification.html).
+
+## Installation
+
+No additional steps needed to be taken to get the REST Server up and running. That is, setting up a VXQuery cluster will
+automatically start the REST Server on port `8080`. Please see [VXQuery Cluster Setup](../user_cluster_installation.html)
+to understand how a VXQuery cluster is setup.
+
+## Getting Started
+
+Suppose we want to execute a very simple XQuery like:
+
+```
+for $x in (1, 2.0, 3) return $x
+```
+
+### Async (Default Mode) Example
+
+If we want to send this, following will be the plain HTTP request.
+
+```
+GET http://127.0.1.1:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x HTTP/1.1
+```
+
+Note the query parameter `statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x` in which the above mentioned statement
+has been encoded. If we send this request using **cURL**, it will look like follows.
+
+#### Accept: application/json
+
+```
+curl -i -H "Accept: application/json" -X GET "http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/json
+content-length: 320
+
+{
+ "status": "success",
+ "requestId": "b0cbe06f-3454-4422-ba23-59150e1c1400",
+ "statement": "for $x in (1, 2.0, 3) return $x",
+ "abstractSyntaxTree": null,
+ "translatedExpressionTree": null,
+ "optimizedExpressionTree": null,
+ "runtimePlan": null,
+ "metrics": {
+ "compileTime": 0,
+ "elapsedTime": 0
+ },
+ "resultId": 6,
+ "resultUrl": "/vxquery/query/result/6"
+}
+```
+
+#### Accept: application/xml
+
+```
+curl -i -H "Accept: application/xml" -X GET "http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/xml
+content-length: 403
+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<asyncQueryResponse>
+ <requestId>d0c2c0ef-2e46-4153-9d4b-1ef4593184e7</requestId>
+ <metrics>
+ <compileTime>0</compileTime>
+ <elapsedTime>0</elapsedTime>
+ </metrics>
+ <statement>for $x in (1, 2.0, 3) return $x</statement>
+ <resultId>8</resultId>
+ <resultUrl>/vxquery/query/result/8</resultUrl>
+</asyncQueryResponse>
+```
+
+#### Result Fetching
+
+Since we have used the default mode (**async**), we only got the **resultId**. Now we have to send another request asking
+for the actual results. Send a cURL request to `/vxquery/query/result/8` to fetch results for result ID 8.
+
+##### Accept: application/json
+
+```
+curl -i -H "Accept: application/json" -X GET "http://localhost:39003/vxquery/query/result/8"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/json
+content-length: 137
+
+{
+ "status": "success",
+ "requestId": "d0c2c0ef-2e46-4153-9d4b-1ef4593184e7",
+ "results": "1\n2\n3\n",
+ "metrics": {
+ "compileTime": 0,
+ "elapsedTime": 0
+ }
+}
+```
+
+Note the *results* in the JSON content in the response.
+
+##### Accept: application/xml
+
+```
+curl -i -H "Accept: application/xml" -X GET "http://localhost:39003/vxquery/query/result/8"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/xml
+content-length: 298
+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<queryResultResponse>
+ <requestId>d0c2c0ef-2e46-4153-9d4b-1ef4593184e7</requestId>
+ <metrics>
+ <compileTime>0</compileTime>
+ <elapsedTime>0</elapsedTime>
+ </metrics>
+ <results>1
+2
+3
+</results>
+</queryResultResponse>
+```
+
+Note the *<results></results>* in the XML content in the response.
+
+### Sync (Synchronous Mode) Example
+
+Similarly to what we did under async requests, we can send the query requests here as well, but with the added query parameter
+`mode=sync` which is to indicate that the response should be a synchronous one. That is, we wait for the query to be
+executed and the response to arrive.
+
+```
+curl -i -H "Accept: application/xml" -X GET \
+"http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x&mode=sync"
+```
+
+and the response now contains **results** instead of the **resultId** we received previously.
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/xml
+content-length: 353
+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<syncQueryResponse>
+ <requestId>93b67f50-4f14-4304-a9b2-f75b4a736df3</requestId>
+ <metrics>
+ <compileTime>0</compileTime>
+ <elapsedTime>0</elapsedTime>
+ </metrics>
+ <statement>for $x in (1, 2.0, 3) return $x</statement>
+ <results>1
+2
+3
+</results>
+</syncQueryResponse>
+```
+
+Similarly with `accept:application/json`,
+
+```
+curl -i -H "Accept: application/json" -X GET \
+"http://localhost:39003/vxquery/query?statement=for+%24x+in+%281%2C+2.0%2C+3%29+return+%24x&mode=sync"
+```
+
+and the response is,
+
+```
+HTTP/1.1 200 OK
+transfer-encoding: chunked
+connection: keep-alive
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept
+content-type: application/json
+content-length: 291
+
+{
+ "status": "success",
+ "requestId": "8010a699-a6f2-423c-91e1-8ac17cd5c5cd",
+ "statement": "for $x in (1, 2.0, 3) return $x",
+ "abstractSyntaxTree": null,
+ "translatedExpressionTree": null,
+ "optimizedExpressionTree": null,
+ "runtimePlan": null,
+ "metrics": {
+ "compileTime": 0,
+ "elapsedTime": 0
+ },
+ "results": "1\n2\n3\n"
+}
+```
diff --git a/vxquery-rest/src/site/markdown/specification.md b/vxquery-rest/src/site/markdown/specification.md
new file mode 100644
index 0000000..3977439
--- /dev/null
+++ b/vxquery-rest/src/site/markdown/specification.md
@@ -0,0 +1,102 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# REST API Specification
+
+Swagger configuration of the REST API can be found
+[here](https://cwiki.apache.org/confluence/display/VXQUERY/SwaggerIO+Configuration).
+
+**NOTE:** This REST API supports both **content types**, `application/json` and `application/xml`. Based on the `accept` header
+of your query request, REST API will return the results wither in *json* form or *XML* form. Returned content type
+defaults to `application/json` if no `accept` header is present.
+
+Base Path **${host}/vxquery**
+
+## Query Request
+
+Request of this type should be submitted for a given *query* to be executed. Depending on the value of the parameter
+`mode`, a **synchronous** response or an **asynchronous** response will be returned
+
+`*` required
+
+| Path | Method |Parameters | Type | Description |
+| ------ | ------ | ------ | ----- |----- |
+| /query | GET |statement* | string | Statement to be executed |
+| | |mode | string | `sync` or `async`. **async** will return an asynchronous response **(default: async)** |
+| | |compileOnly | boolean | If `true`, statement will be compiled, but won't be executed (default: false) |
+| | |optimization | integer | Optimization level (0 - 2,147,483,647). (Default: Full optimization) |
+| | |frameSize | integer | Frame size in bytes (default: 65536) |
+| | |repeatExecutions|integer | Number of times to repeat execution (default: 1) |
+| | |metrics | boolean | If `true`, returns metrics (compile and execution time) with the response (default: false) |
+| | |showAbstractSyntaxTree | boolean | Shows abstract syntax tree if `true` (default: false) |
+| | |showTranslatedExpressionTree | boolean | Shows translated expression tree if `true` (default: false) |
+| | |showOptimizedExpressionTree | boolean | Shows optimized expression tree if `true` (default: false) |
+| | |showRuntimePlan| boolean | Shows runtime plan if set to `true` (default: false) |
+
+### Synchronous Query Response
+
+Received only when `mode` is set to `sync` in the query request above.
+
+| Attribute | Type | Description |
+| ------ | ------ | ------ |
+|statement* |string | Statement submitted to be executed |
+|status* |string | `success` to indicate that the query execution was successful |
+|requestId* |string | A unique ID assigned to the request sent earlier |
+|abstractSyntaxTree |string | Abstract Syntax Tree if requested in the query request. Else `null` |
+|translatedExpressionTree |string | Translated Expression Tree if requested in the query request. Else `null` |
+|optimizedExpressionTree |string | Optimized Expression Tree if requested in the query request. Else `null` |
+|runtimePlan |string | Runtime plan if requested in the query request. Else `null` |
+|metrics |metrics | Metrics (`compileTime` and `elapsedTime`) if requested in the query request |
+|results* |string | Results of the query/statement submitted for execution |
+
+### Asynchronous Query Response
+
+Received only when `mode` is set to `async` (which is the default value) in the query request above.
+
+| Attribute | Type | Description |
+| ------ | ------ | ------ |
+|statement* |string | Statement submitted to be executed |
+|status* |string | `success` to indicate that the query execution was successful |
+|requestId* |string | A unique ID assigned to the request sent earlier |
+|abstractSyntaxTree |string | Abstract Syntax Tree if requested in the query request. Else `null` |
+|translatedExpressionTree |string | Translated Expression Tree if requested in the query request. Else `null` |
+|optimizedExpressionTree |string | Optimized Expression Tree if requested in the query request. Else `null` |
+|runtimePlan |string | Runtime plan if requested in the query request. Else `null` |
+|metrics |metrics | Metrics (`compileTime` and `elapsedTime`) if requested in the query request |
+|resultId* |string | Result ID of the query submitted for execution. This ID is required later for result fetching |
+|resultUrl* |string | URL from which the results of the submitted query can be retrieved |
+
+### Result fetching (After an Asynchronous Query Response)
+
+The **resultId** received in the asynchronous query response needs to be submitted as a
+**path parameter** (`/query/result/${resultId}`) to the REST API in order to retrieve the corresponding results.
+
+| Path | Method |Parameters | Type | Description |
+| ------ | ------ | ------ | ----- |----- |
+| /query/result/${resultId} | GET | metrics | boolean | If `true`, returns metrics (compile and execution time) with the response (default: false) |
+
+***
+
+## Error Response
+
+In any of the above scenarios, if an error occurred while processing, REST API will return an *Error Response* as
+specified below.
+
+| Attribute | Type | Description |
+| ------ | ------ | ------ |
+|status* |string | `fatal` to indicate that the query execution was failed at some point |
+|requestId* |string | A unique ID assigned to the request sent |
+|error* |Error | An error object which include an error code with an error message. {code: xxx, message: "error message"} |
diff --git a/vxquery-rest/src/site/site.xml b/vxquery-rest/src/site/site.xml
new file mode 100644
index 0000000..bd9b9ab
--- /dev/null
+++ b/vxquery-rest/src/site/site.xml
@@ -0,0 +1,53 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project name="VXQuery">
+ <bannerLeft>
+ <name>VXQuery</name>
+ <src>../images/VXQuery.png</src>
+ <href>../index.html</href>
+ </bannerLeft>
+
+ <bannerRight>
+ <name>Apache Software Foundation</name>
+ <src>../images/asf_logo_wide.png</src>
+ <href>http://www.apache.org/</href>
+ </bannerRight>
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-fluido-skin</artifactId>
+ <version>1.5</version>
+ </skin>
+
+ <body>
+ <menu name="VXQuery REST API">
+ <item name="Overview" href="index.html"/>
+ <item name="Specification" href="specification.html"/>
+ </menu>
+
+ <menu ref="reports"/>
+ <footer><![CDATA[
+ <div class="row-fluid">Apache VXQuery, VXQuery, Apache, the Apache
+ feather logo, and the Apache VXQuery project logo are either
+ registered trademarks or trademarks of The Apache Software
+ Foundation in the United States and other countries.
+ All other marks mentioned may be trademarks or registered
+ trademarks of their respective owners.
+ </div>]]>
+ </footer>
+ </body>
+</project>
\ No newline at end of file
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java
new file mode 100644
index 0000000..3d4b124
--- /dev/null
+++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/AbstractRestServerTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import java.net.URI;
+import java.util.concurrent.TimeUnit;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.VXQueryApplication;
+import org.apache.vxquery.app.util.LocalClusterUtil;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.QueryResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.VXQueryConfig;
+import org.apache.vxquery.rest.service.VXQueryService;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * Abstract test class to be used for {@link VXQueryApplication} related tests.
+ * These tests are expected to use the REST API for querying and fetching
+ * results
+ *
+ * @author Erandi Ganepola
+ */
+public class AbstractRestServerTest {
+
+ protected static LocalClusterUtil vxqueryLocalCluster = new LocalClusterUtil();
+ protected static String restIpAddress;
+ protected static int restPort;
+ protected static VXQueryService vxQueryService;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ vxqueryLocalCluster.init(new VXQueryConfig());
+ vxQueryService = vxqueryLocalCluster.getVxQueryService();
+ restIpAddress = vxqueryLocalCluster.getIpAddress();
+ restPort = vxqueryLocalCluster.getRestPort();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ vxqueryLocalCluster.deinit();
+ }
+
+ protected static String normalize(String string) {
+ if (string == null) {
+ return null;
+ }
+
+ return string.replace("\r\n", "").replace("\n", "").replace("\r", "");
+ }
+
+ protected static void checkMetrics(QueryResponse response, boolean showMetrics) {
+ if (showMetrics) {
+ Assert.assertTrue(response.getMetrics().getCompileTime() > 0);
+ Assert.assertTrue(response.getMetrics().getElapsedTime() > 0);
+ } else {
+ Assert.assertTrue(response.getMetrics().getCompileTime() == 0);
+ Assert.assertTrue(response.getMetrics().getElapsedTime() == 0);
+ }
+ }
+
+ protected static void checkResults(AsyncQueryResponse response, boolean compileOnly) {
+ if (compileOnly) {
+ Assert.assertNull(response.getResultUrl());
+ Assert.assertEquals(0, response.getResultId());
+ } else {
+ Assert.assertTrue(response.getResultUrl().startsWith(Constants.RESULT_URL_PREFIX));
+ Assert.assertNotEquals(0, response.getResultId());
+ }
+ }
+
+ protected static void checkResults(SyncQueryResponse response, boolean compileOnly) {
+ if (compileOnly) {
+ Assert.assertNull(response.getResults());
+ } else {
+ Assert.assertNotNull(response.getResults());
+ Assert.assertFalse(response.getResults().isEmpty());
+ }
+ }
+
+ /**
+ * Submit a {@link QueryRequest} and fetth the resulting
+ * {@link AsyncQueryResponse}
+ *
+ * @param uri
+ * uri of the GET request
+ * @param accepts
+ * application/json | application/xml
+ * @param method
+ * Http Method to be used to send the request
+ * @return Response received for the query request
+ * @throws Exception
+ */
+ protected static <T> T getQuerySuccessResponse(URI uri, String accepts, Class<T> type, String method)
+ throws Exception {
+ CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build();
+
+ try {
+ HttpUriRequest request = getRequest(uri, method);
+
+ if (accepts != null) {
+ request.setHeader(HttpHeaders.ACCEPT, accepts);
+ }
+
+ try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+ Assert.assertEquals(HttpResponseStatus.OK.code(), httpResponse.getStatusLine().getStatusCode());
+ if (accepts != null) {
+ Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
+ }
+
+ HttpEntity entity = httpResponse.getEntity();
+ Assert.assertNotNull(entity);
+
+ String response = RestUtils.readEntity(entity);
+ return RestUtils.mapEntity(response, type, accepts);
+ }
+ } finally {
+ HttpClientUtils.closeQuietly(httpClient);
+ }
+ }
+
+ /**
+ * Fetch the {@link QueryResultResponse} from query result endpoint once the
+ * corresponding {@link QueryResultRequest} is given.
+ *
+ * @param resultRequest
+ * {@link QueryResultRequest}
+ * @param accepts
+ * expected
+ *
+ * <pre>
+ * Accepts
+ * </pre>
+ *
+ * header in responses
+ * @param method
+ * Http Method to be used to send the request
+ * @return query result response received
+ * @throws Exception
+ */
+ protected static QueryResultResponse getQueryResultResponse(QueryResultRequest resultRequest, String accepts,
+ String method) throws Exception {
+ URI uri = RestUtils.buildQueryResultURI(resultRequest, restIpAddress, restPort);
+ CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build();
+ try {
+ HttpUriRequest request = getRequest(uri, method);
+
+ if (accepts != null) {
+ request.setHeader(HttpHeaders.ACCEPT, accepts);
+ }
+
+ try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+ if (accepts != null) {
+ Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
+ }
+ Assert.assertEquals(httpResponse.getStatusLine().getStatusCode(), HttpResponseStatus.OK.code());
+
+ HttpEntity entity = httpResponse.getEntity();
+ Assert.assertNotNull(entity);
+
+ String response = RestUtils.readEntity(entity);
+ return RestUtils.mapEntity(response, QueryResultResponse.class, accepts);
+ }
+ } finally {
+ HttpClientUtils.closeQuietly(httpClient);
+ }
+ }
+
+ /**
+ * Creates a POST or GET request accordingly from the given {@link URI}
+ *
+ * @param uri
+ * URI to which the request us to be sent
+ * @param method
+ * Http method- GET or POST
+ * @return request
+ */
+ protected static HttpUriRequest getRequest(URI uri, String method) {
+ HttpUriRequest request;
+ switch (method) {
+ case HttpMethod.POST:
+ request = new HttpPost(uri);
+ break;
+ case HttpMethod.GET:
+ default:
+ request = new HttpGet(uri);
+ }
+
+ return request;
+ }
+}
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java
new file mode 100644
index 0000000..12b61f3
--- /dev/null
+++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/ErrorResponseTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.app.util.RestUtils.buildQueryResultURI;
+import static org.apache.vxquery.app.util.RestUtils.buildQueryURI;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.INVALID_INPUT;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.NOT_FOUND;
+import static org.apache.vxquery.rest.Constants.ErrorCodes.PROBLEM_WITH_QUERY;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.net.URI;
+import java.util.concurrent.TimeUnit;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests error codes of the possible error responses that can be received for
+ * erroneous queries.
+ *
+ * @author Erandi Ganepola
+ */
+public class ErrorResponseTest extends AbstractRestServerTest {
+
+ @Test
+ public void testInvalidInput01() throws Exception {
+ QueryRequest request = new QueryRequest(" ");
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, INVALID_INPUT);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, INVALID_INPUT);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, INVALID_INPUT);
+ }
+
+ @Test
+ public void testInvalidInput02() throws Exception {
+ QueryRequest request = new QueryRequest("");
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, 405);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, 405);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, 405);
+ }
+
+ @Test
+ public void testInvalidQuery01() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1,2,3) return $y");
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testInvalidQuery02() throws Exception {
+ QueryRequest request = new QueryRequest("for x in (1,2,3) return $x");
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testInvalidQuery03() throws Exception {
+ QueryRequest request = new QueryRequest("insert nodes <book></book> into doc(\"abcd.xml\")/books");
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testInvalidQuery04() throws Exception {
+ QueryRequest request = new QueryRequest("delete nodes /a/b//node()");
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testInvalidResultId() throws Exception {
+ QueryResultRequest request = new QueryResultRequest(1000);
+ runTest(buildQueryResultURI(request, restIpAddress, restPort), null, NOT_FOUND);
+ runTest(buildQueryResultURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, NOT_FOUND);
+ runTest(buildQueryResultURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, NOT_FOUND);
+ }
+
+ @Test
+ public void testSyncInvalidInput01() throws Exception {
+ QueryRequest request = new QueryRequest(" ");
+ request.setAsync(false);
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, INVALID_INPUT);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, INVALID_INPUT);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, INVALID_INPUT);
+ }
+
+ @Test
+ public void testSyncInvalidInput02() throws Exception {
+ QueryRequest request = new QueryRequest("");
+ request.setAsync(false);
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, 405);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, 405);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, 405);
+ }
+
+ @Test
+ public void testSyncInvalidQuery01() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1,2,3) return $y");
+ request.setAsync(false);
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testSyncInvalidQuery02() throws Exception {
+ QueryRequest request = new QueryRequest("for x in (1,2,3) return $x");
+ request.setAsync(false);
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testSyncInvalidQuery03() throws Exception {
+ QueryRequest request = new QueryRequest("insert nodes <book></book> into doc(\"abcd.xml\")/books");
+ request.setAsync(false);
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ @Test
+ public void testSyncInvalidQuery04() throws Exception {
+ QueryRequest request = new QueryRequest("delete nodes /a/b//node()");
+ request.setAsync(false);
+ runTest(buildQueryURI(request, restIpAddress, restPort), null, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_JSON, PROBLEM_WITH_QUERY);
+ runTest(buildQueryURI(request, restIpAddress, restPort), CONTENT_TYPE_XML, PROBLEM_WITH_QUERY);
+ }
+
+ private void runTest(URI uri, String accepts, int expectedStatusCode) throws Exception {
+ runTest(uri, accepts, expectedStatusCode, HttpMethod.GET);
+ runTest(uri, accepts, expectedStatusCode, HttpMethod.POST);
+ }
+
+ private void runTest(URI uri, String accepts, int expectedStatusCode, String httpMethod) throws Exception {
+ CloseableHttpClient httpClient = HttpClients.custom().setConnectionTimeToLive(20, TimeUnit.SECONDS).build();
+
+ ErrorResponse errorResponse;
+ try {
+ HttpUriRequest request = getRequest(uri, httpMethod);
+ if (accepts != null) {
+ request.setHeader(HttpHeaders.ACCEPT, accepts);
+ }
+
+ try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+ Assert.assertEquals(expectedStatusCode, httpResponse.getStatusLine().getStatusCode());
+ if (accepts != null) {
+ Assert.assertEquals(accepts, httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
+ }
+
+ HttpEntity entity = httpResponse.getEntity();
+ Assert.assertNotNull(entity);
+
+ String response = RestUtils.readEntity(entity);
+ errorResponse = RestUtils.mapEntity(response, ErrorResponse.class, accepts);
+ }
+ } finally {
+ HttpClientUtils.closeQuietly(httpClient);
+ }
+
+ Assert.assertNotNull(errorResponse);
+ Assert.assertNotNull(errorResponse.getError().getMessage());
+ Assert.assertEquals(errorResponse.getError().getCode(), expectedStatusCode);
+ }
+}
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java
new file mode 100644
index 0000000..d1385c8
--- /dev/null
+++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessAsyncResponseTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.net.URI;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.request.QueryResultRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.AsyncQueryResponse;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.QueryResultResponse;
+import org.apache.vxquery.rest.service.Status;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the success responses received for XQueries submitted. i.e
+ * we are submitting correct queries which are expected to return a predictable
+ * result. All the parameters that are expected to be sent with query requests
+ * are subjected to test in this test class
+ *
+ * @author Erandi Ganepola
+ */
+public class SuccessAsyncResponseTest extends AbstractRestServerTest {
+
+ @Test
+ public void testSimpleQuery001() throws Exception {
+ QueryRequest request = new QueryRequest("1+1");
+ request.setShowAbstractSyntaxTree(true);
+ request.setShowOptimizedExpressionTree(true);
+ request.setShowRuntimePlan(true);
+ request.setShowTranslatedExpressionTree(true);
+ request.setShowMetrics(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSimpleQuery002() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowAbstractSyntaxTree(true);
+ request.setShowOptimizedExpressionTree(true);
+ request.setShowRuntimePlan(true);
+ request.setShowTranslatedExpressionTree(true);
+ request.setShowMetrics(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSimpleQuery003() throws Exception {
+ QueryRequest request = new QueryRequest("1+2+3");
+ request.setShowAbstractSyntaxTree(false);
+ request.setShowOptimizedExpressionTree(false);
+ request.setShowRuntimePlan(false);
+ request.setShowTranslatedExpressionTree(false);
+ request.setShowMetrics(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSimpleQuery004() throws Exception {
+ QueryRequest request = new QueryRequest("fn:true()");
+ request.setShowAbstractSyntaxTree(false);
+ request.setShowOptimizedExpressionTree(false);
+ request.setShowRuntimePlan(true);
+ request.setShowTranslatedExpressionTree(false);
+ request.setShowMetrics(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterNone() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterMetrics() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowMetrics(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterAST() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowAbstractSyntaxTree(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterOptimization() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setOptimization(10000);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterFrameSize() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setFrameSize((int) Math.pow(2, 12));
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterCompileOnly() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setCompileOnly(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterOET() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowOptimizedExpressionTree(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterTET() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowTranslatedExpressionTree(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterRP() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowRuntimePlan(true);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ private void runTest(String contentType, QueryRequest request) throws Exception {
+ runTest(contentType, request, HttpMethod.GET);
+ runTest(contentType, request, HttpMethod.POST);
+ }
+
+ private void runTest(String contentType, QueryRequest request, String httpMethod) throws Exception {
+ URI queryEndpointUri = RestUtils.buildQueryURI(request, restIpAddress, restPort);
+
+ /*
+ * ========== Query Response Testing ==========
+ */
+ // Testing the accuracy of VXQueryService class
+ AsyncQueryResponse expectedAsyncQueryResponse = (AsyncQueryResponse) vxQueryService.execute(request);
+
+ Assert.assertEquals(Status.SUCCESS.toString(), expectedAsyncQueryResponse.getStatus());
+ Assert.assertEquals(request.getStatement(), expectedAsyncQueryResponse.getStatement());
+ checkResults(expectedAsyncQueryResponse, request.isCompileOnly());
+ checkMetrics(expectedAsyncQueryResponse, request.isShowMetrics());
+ if (request.isShowMetrics()) {
+ Assert.assertTrue(expectedAsyncQueryResponse.getMetrics().getCompileTime() > 0);
+ } else {
+ Assert.assertTrue(expectedAsyncQueryResponse.getMetrics().getCompileTime() == 0);
+ }
+ if (request.isShowAbstractSyntaxTree()) {
+ Assert.assertNotNull(expectedAsyncQueryResponse.getAbstractSyntaxTree());
+ } else {
+ Assert.assertNull(expectedAsyncQueryResponse.getAbstractSyntaxTree());
+ }
+ if (request.isShowTranslatedExpressionTree()) {
+ Assert.assertNotNull(expectedAsyncQueryResponse.getTranslatedExpressionTree());
+ } else {
+ Assert.assertNull(expectedAsyncQueryResponse.getTranslatedExpressionTree());
+ }
+ if (request.isShowOptimizedExpressionTree()) {
+ Assert.assertNotNull(expectedAsyncQueryResponse.getOptimizedExpressionTree());
+ } else {
+ Assert.assertNull(expectedAsyncQueryResponse.getOptimizedExpressionTree());
+ }
+ if (request.isShowRuntimePlan()) {
+ Assert.assertNotNull(expectedAsyncQueryResponse.getRuntimePlan());
+ } else {
+ Assert.assertNull(expectedAsyncQueryResponse.getRuntimePlan());
+ }
+
+ // Testing the accuracy of REST server and servlets
+ AsyncQueryResponse actualAsyncQueryResponse =
+ getQuerySuccessResponse(queryEndpointUri, contentType, AsyncQueryResponse.class, httpMethod);
+
+ Assert.assertNotNull(actualAsyncQueryResponse.getRequestId());
+ Assert.assertEquals(request.getStatement(), actualAsyncQueryResponse.getStatement());
+ Assert.assertEquals(Status.SUCCESS.toString(), actualAsyncQueryResponse.getStatus());
+ checkResults(actualAsyncQueryResponse, request.isCompileOnly());
+ checkMetrics(actualAsyncQueryResponse, request.isShowMetrics());
+ // Cannot check this because Runtime plan include some object IDs which differ
+ // Assert.assertEquals(expectedAsyncQueryResponse.getRuntimePlan(),
+ // actualAsyncQueryResponse.getRuntimePlan());
+ if (request.isShowRuntimePlan()) {
+ Assert.assertNotNull(actualAsyncQueryResponse.getRuntimePlan());
+ } else {
+ Assert.assertNull(actualAsyncQueryResponse.getRuntimePlan());
+ }
+ Assert.assertEquals(normalize(expectedAsyncQueryResponse.getOptimizedExpressionTree()),
+ normalize(actualAsyncQueryResponse.getOptimizedExpressionTree()));
+ Assert.assertEquals(normalize(expectedAsyncQueryResponse.getTranslatedExpressionTree()),
+ normalize(actualAsyncQueryResponse.getTranslatedExpressionTree()));
+ Assert.assertEquals(normalize(expectedAsyncQueryResponse.getAbstractSyntaxTree()),
+ normalize(actualAsyncQueryResponse.getAbstractSyntaxTree()));
+
+ /*
+ * ========== Query Result Response Testing ========
+ */
+ QueryResultRequest resultRequest = new QueryResultRequest(actualAsyncQueryResponse.getResultId());
+ resultRequest.setShowMetrics(true);
+
+ if (request.isCompileOnly()) {
+ APIResponse resultResponse = vxQueryService.getResult(resultRequest);
+ Assert.assertTrue(resultResponse instanceof ErrorResponse);
+ } else {
+ QueryResultResponse expectedResultResponse = (QueryResultResponse) vxQueryService.getResult(resultRequest);
+ Assert.assertEquals(expectedResultResponse.getStatus(), Status.SUCCESS.toString());
+ Assert.assertNotNull(expectedResultResponse.getResults());
+
+ QueryResultResponse actualResultResponse = getQueryResultResponse(resultRequest, contentType, httpMethod);
+ Assert.assertEquals(actualResultResponse.getStatus(), Status.SUCCESS.toString());
+ Assert.assertNotNull(actualResultResponse.getResults());
+ Assert.assertNotNull(actualResultResponse.getRequestId());
+ Assert.assertEquals(normalize(expectedResultResponse.getResults()),
+ normalize(actualResultResponse.getResults()));
+ if (resultRequest.isShowMetrics()) {
+ Assert.assertTrue(actualResultResponse.getMetrics().getElapsedTime() > 0);
+ } else {
+ Assert.assertTrue(actualResultResponse.getMetrics().getElapsedTime() == 0);
+ }
+
+ }
+ }
+}
diff --git a/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java
new file mode 100644
index 0000000..31e0162
--- /dev/null
+++ b/vxquery-rest/src/test/java/org/apache/vxquery/rest/SuccessSyncResponseTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.vxquery.rest;
+
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_XML;
+
+import java.net.URI;
+
+import javax.ws.rs.HttpMethod;
+
+import org.apache.vxquery.app.util.RestUtils;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.apache.vxquery.rest.service.Status;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the success responses received for XQueries submitted. i.e
+ * we are submitting correct queries which are expected to return a predictable
+ * result. All the parameters that are expected to be sent with query requests
+ * are subjected to test in this test class
+ *
+ * @author Erandi Ganepola
+ */
+public class SuccessSyncResponseTest extends AbstractRestServerTest {
+
+ @Test
+ public void testSimpleQuery001() throws Exception {
+ QueryRequest request = new QueryRequest("1+1");
+ request.setShowAbstractSyntaxTree(true);
+ request.setShowOptimizedExpressionTree(true);
+ request.setShowRuntimePlan(true);
+ request.setShowTranslatedExpressionTree(true);
+ request.setShowMetrics(false);
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSimpleQuery002() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setShowAbstractSyntaxTree(true);
+ request.setShowOptimizedExpressionTree(true);
+ request.setShowRuntimePlan(true);
+ request.setShowTranslatedExpressionTree(true);
+ request.setShowMetrics(true);
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSimpleQuery003() throws Exception {
+ QueryRequest request = new QueryRequest("1+2+3");
+ request.setShowAbstractSyntaxTree(false);
+ request.setShowOptimizedExpressionTree(false);
+ request.setShowRuntimePlan(false);
+ request.setShowTranslatedExpressionTree(false);
+ request.setShowMetrics(false);
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSimpleQuery004() throws Exception {
+ QueryRequest request = new QueryRequest("fn:true()");
+ request.setShowAbstractSyntaxTree(false);
+ request.setShowOptimizedExpressionTree(false);
+ request.setShowRuntimePlan(true);
+ request.setShowTranslatedExpressionTree(false);
+ request.setShowMetrics(false);
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterNone() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterCompileOnly() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setCompileOnly(true);
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ @Test
+ public void testSingleParameterRepeatExecutions() throws Exception {
+ QueryRequest request = new QueryRequest("for $x in (1, 2.0, 3) return $x");
+ request.setRepeatExecutions(5);
+ request.setAsync(false);
+
+ runTest(null, request);
+ runTest(CONTENT_TYPE_JSON, request);
+ runTest(CONTENT_TYPE_XML, request);
+ }
+
+ private void runTest(String contentType, QueryRequest request) throws Exception {
+ runTest(contentType, request, HttpMethod.GET);
+ runTest(contentType, request, HttpMethod.POST);
+ }
+
+ private void runTest(String contentType, QueryRequest request, String httpMethod) throws Exception {
+ URI queryEndpointUri = RestUtils.buildQueryURI(request, restIpAddress, restPort);
+
+ /*
+ * ========== Query Response Testing ==========
+ */
+ // Testing the accuracy of VXQueryService class
+ SyncQueryResponse expectedSyncQueryResponse = (SyncQueryResponse) vxQueryService.execute(request);
+
+ Assert.assertEquals(Status.SUCCESS.toString(), expectedSyncQueryResponse.getStatus());
+ Assert.assertEquals(request.getStatement(), expectedSyncQueryResponse.getStatement());
+ checkResults(expectedSyncQueryResponse, request.isCompileOnly());
+ checkMetrics(expectedSyncQueryResponse, request.isShowMetrics());
+ if (request.isShowMetrics()) {
+ Assert.assertTrue(expectedSyncQueryResponse.getMetrics().getCompileTime() > 0);
+ } else {
+ Assert.assertTrue(expectedSyncQueryResponse.getMetrics().getCompileTime() == 0);
+ }
+ if (request.isShowAbstractSyntaxTree()) {
+ Assert.assertNotNull(expectedSyncQueryResponse.getAbstractSyntaxTree());
+ } else {
+ Assert.assertNull(expectedSyncQueryResponse.getAbstractSyntaxTree());
+ }
+ if (request.isShowTranslatedExpressionTree()) {
+ Assert.assertNotNull(expectedSyncQueryResponse.getTranslatedExpressionTree());
+ } else {
+ Assert.assertNull(expectedSyncQueryResponse.getTranslatedExpressionTree());
+ }
+ if (request.isShowOptimizedExpressionTree()) {
+ Assert.assertNotNull(expectedSyncQueryResponse.getOptimizedExpressionTree());
+ } else {
+ Assert.assertNull(expectedSyncQueryResponse.getOptimizedExpressionTree());
+ }
+ if (request.isShowRuntimePlan()) {
+ Assert.assertNotNull(expectedSyncQueryResponse.getRuntimePlan());
+ } else {
+ Assert.assertNull(expectedSyncQueryResponse.getRuntimePlan());
+ }
+
+ // Testing the accuracy of REST server and servlets
+ SyncQueryResponse actualSyncQueryResponse =
+ getQuerySuccessResponse(queryEndpointUri, contentType, SyncQueryResponse.class, httpMethod);
+
+ Assert.assertNotNull(actualSyncQueryResponse.getRequestId());
+ Assert.assertEquals(request.getStatement(), actualSyncQueryResponse.getStatement());
+ Assert.assertEquals(Status.SUCCESS.toString(), actualSyncQueryResponse.getStatus());
+ checkMetrics(actualSyncQueryResponse, request.isShowMetrics());
+ checkResults(actualSyncQueryResponse, request.isCompileOnly());
+ // Cannot check this because Runtime plan include some object IDs which differ
+ // Assert.assertEquals(expectedSyncQueryResponse.getRuntimePlan(),
+ // actualSyncQueryResponse.getRuntimePlan());
+ if (request.isShowRuntimePlan()) {
+ Assert.assertNotNull(actualSyncQueryResponse.getRuntimePlan());
+ } else {
+ Assert.assertNull(actualSyncQueryResponse.getRuntimePlan());
+ }
+ Assert.assertEquals(normalize(expectedSyncQueryResponse.getOptimizedExpressionTree()),
+ normalize(actualSyncQueryResponse.getOptimizedExpressionTree()));
+ Assert.assertEquals(normalize(expectedSyncQueryResponse.getTranslatedExpressionTree()),
+ normalize(actualSyncQueryResponse.getTranslatedExpressionTree()));
+ Assert.assertEquals(normalize(expectedSyncQueryResponse.getAbstractSyntaxTree()),
+ normalize(actualSyncQueryResponse.getAbstractSyntaxTree()));
+
+ /*
+ * ========== Query Result Response Testing ========
+ */
+ String expectedResults = expectedSyncQueryResponse.getResults();
+ String actualResults = actualSyncQueryResponse.getResults();
+ if (!request.isCompileOnly()) {
+ Assert.assertNotNull(expectedResults);
+ Assert.assertNotNull(actualResults);
+ }
+ Assert.assertEquals(normalize(expectedResults), normalize(actualResults));
+ }
+}
diff --git a/vxquery-rest/src/test/resources/vxquery.properties b/vxquery-rest/src/test/resources/vxquery.properties
new file mode 100644
index 0000000..8870f79
--- /dev/null
+++ b/vxquery-rest/src/test/resources/vxquery.properties
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Number of processors to be used for query processing
+#org.apache.vxquery.available_processors=-1
+
+# Number of local node controllers to be created when creating a local hyracks cluster
+org.apache.vxquery.local_nc=1
+
+# Join hash size
+#org.apache.vxquery.join_hash=-1
+
+# Maximum Data Size
+#org.apache.vxquery.data_size=-1
+
+# hdfs config directory
+#org.apache.vxquery.hdfs_config=foo/bar
diff --git a/vxquery-server/pom.xml b/vxquery-server/pom.xml
index e572c36..7331fde 100644
--- a/vxquery-server/pom.xml
+++ b/vxquery-server/pom.xml
@@ -41,7 +41,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>appassembler-maven-plugin</artifactId>
- <version>1.1.1</version>
+ <version>2.0.0</version>
<executions>
<execution>
<configuration>
@@ -53,6 +53,10 @@
<program>
<mainClass>org.apache.hyracks.control.cc.CCDriver</mainClass>
<name>vxquerycc</name>
+ <commandLineArguments>
+ <commandLineArgument>-app-cc-main-class</commandLineArgument>
+ <commandLineArgument>org.apache.vxquery.app.VXQueryApplication</commandLineArgument>
+ </commandLineArguments>
</program>
<program>
<mainClass>org.apache.hyracks.control.nc.NCDriver</mainClass>
@@ -139,15 +143,13 @@
<dependencies>
<dependency>
<groupId>org.apache.vxquery</groupId>
- <artifactId>apache-vxquery-core</artifactId>
+ <artifactId>apache-vxquery-rest</artifactId>
<version>0.7-SNAPSHOT</version>
</dependency>
-
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-control-cc</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml
index a00bec2..a32d913 100644
--- a/vxquery-xtest/pom.xml
+++ b/vxquery-xtest/pom.xml
@@ -144,6 +144,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.vxquery</groupId>
+ <artifactId>apache-vxquery-rest</artifactId>
+ <version>0.7-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
</dependency>
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
index 0e5b481..4d2ae8a 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java
@@ -19,81 +19,45 @@
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.client.dataset.HyracksDataset;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.common.controllers.CCConfig;
-import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.vxquery.app.util.LocalClusterUtil;
+import org.apache.vxquery.rest.service.VXQueryConfig;
-import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
public class TestClusterUtil {
- private static final int CLIENT_NET_PORT = 39000;
- private static final int CLUSTER_NET_PORT = 39001;
- private static final int PROFILE_DUMP_PERIOD = 10000;
- private static final String CC_HOST = "localhost";
- private static final String NODE_ID = "nc1";
- private static final String IO_DEVICES = "target/tmp/indexFolder";
-
private static HyracksConnection hcc;
private static HyracksDataset hds;
+ public static final LocalClusterUtil localClusterUtil = new LocalClusterUtil();
+
private TestClusterUtil() {
}
- public static CCConfig createCCConfig() throws UnknownHostException {
- String publicAddress = InetAddress.getLocalHost().getHostAddress();
- CCConfig ccConfig = new CCConfig();
- ccConfig.clientNetIpAddress = publicAddress;
- ccConfig.clientNetPort = CLIENT_NET_PORT;
- ccConfig.clusterNetIpAddress = publicAddress;
- ccConfig.clusterNetPort = CLUSTER_NET_PORT;
- ccConfig.profileDumpPeriod = PROFILE_DUMP_PERIOD;
- return ccConfig;
+ private static VXQueryConfig loadConfiguration(XTestOptions opts) {
+ VXQueryConfig vxqConfig = new VXQueryConfig();
+
+ vxqConfig.setAvailableProcessors(opts.threads);
+ vxqConfig.setFrameSize(opts.frameSize);
+ vxqConfig.setHdfsConf(opts.hdfsConf);
+
+ return vxqConfig;
}
- public static NCConfig createNCConfig() throws UnknownHostException {
- String publicAddress = InetAddress.getLocalHost().getHostAddress();
- NCConfig ncConfig1 = new NCConfig();
- ncConfig1.ccHost = CC_HOST;
- ncConfig1.ccPort = CLUSTER_NET_PORT;
- ncConfig1.clusterNetIPAddress = publicAddress;
- ncConfig1.dataIPAddress = publicAddress;
- ncConfig1.resultIPAddress = publicAddress;
- ncConfig1.nodeId = NODE_ID;
- ncConfig1.ioDevices = IO_DEVICES;
- return ncConfig1;
- }
-
- public static ClusterControllerService startCC(XTestOptions opts) throws IOException {
- CCConfig ccConfig = createCCConfig();
- File outDir = new File("target/ClusterController");
- outDir.mkdirs();
- File ccRoot = File.createTempFile(TestRunner.class.getName(), ".data", outDir);
- ccRoot.delete();
- ccRoot.mkdir();
- ccConfig.ccRoot = ccRoot.getAbsolutePath();
+ public static void startCluster(XTestOptions opts, LocalClusterUtil localClusterUtil) throws IOException {
try {
- ClusterControllerService cc = new ClusterControllerService(ccConfig);
- cc.start();
- hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
- hds = new HyracksDataset(hcc, opts.frameSize, opts.threads);
- return cc;
+ VXQueryConfig config = loadConfiguration(opts);
+ localClusterUtil.init(config);
+ hcc = (HyracksConnection) localClusterUtil.getConnection();
+ hds = (HyracksDataset) localClusterUtil.getDataset();
} catch (Exception e) {
throw new IOException(e);
}
-
}
- public static NodeControllerService startNC() throws IOException {
- NCConfig ncConfig = createNCConfig();
+ public static void stopCluster(LocalClusterUtil localClusterUtil) throws IOException {
try {
- NodeControllerService nc = new NodeControllerService(ncConfig);
- nc.start();
- return nc;
+ localClusterUtil.deinit();
} catch (Exception e) {
throw new IOException(e);
}
@@ -107,13 +71,4 @@
return hds;
}
- public static void stopCluster(ClusterControllerService cc, NodeControllerService nc) throws IOException {
- try {
- nc.stop();
- cc.stop();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
}
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
index fa0a900..8ef9426 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java
@@ -14,101 +14,54 @@
*/
package org.apache.vxquery.xtest;
+import static org.apache.vxquery.rest.Constants.HttpHeaderValues.CONTENT_TYPE_JSON;
+
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.client.NodeControllerInfo;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.dataset.DatasetJobRecord;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.job.JobFlag;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
-import org.apache.vxquery.compiler.CompilerControlBlock;
-import org.apache.vxquery.compiler.algebricks.VXQueryGlobalDataFactory;
-import org.apache.vxquery.context.DynamicContext;
-import org.apache.vxquery.context.DynamicContextImpl;
-import org.apache.vxquery.context.RootStaticContextImpl;
-import org.apache.vxquery.context.StaticContextImpl;
+import javax.xml.bind.JAXBException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.vxquery.app.util.RestUtils;
import org.apache.vxquery.exceptions.ErrorCode;
import org.apache.vxquery.exceptions.SystemException;
-import org.apache.vxquery.result.ResultUtils;
-import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener;
-import org.apache.vxquery.xmlquery.query.XMLQueryCompiler;
+import org.apache.vxquery.rest.request.QueryRequest;
+import org.apache.vxquery.rest.response.APIResponse;
+import org.apache.vxquery.rest.response.ErrorResponse;
+import org.apache.vxquery.rest.response.SyncQueryResponse;
+import org.codehaus.jackson.map.ObjectMapper;
public class TestRunner {
+
private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})");
- private List<String> collectionList;
+
private XTestOptions opts;
- private IHyracksClientConnection hcc;
- private IHyracksDataset hds;
public TestRunner(XTestOptions opts) throws UnknownHostException {
this.opts = opts;
- this.collectionList = new ArrayList<String>();
}
public void open() throws Exception {
- hcc = TestClusterUtil.getConnection();
- hds = TestClusterUtil.getDataset();
- }
-
- protected static TestConfiguration getIndexConfiguration(TestCase testCase) {
- XTestOptions opts = new XTestOptions();
- opts.verbose = false;
- opts.threads = 1;
- opts.showQuery = true;
- opts.showResult = true;
- opts.hdfsConf = "src/test/resources/hadoop/conf";
- opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" },
- File.separator);
- TestConfiguration indexConf = new TestConfiguration();
- indexConf.options = opts;
- String baseDir = new File(opts.catalog).getParent();
- try {
- String root = new File(baseDir).getCanonicalPath();
- indexConf.testRoot = new File(root + "/./");
- indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/");
- indexConf.sourceFileMap = testCase.getSourceFileMap();
- indexConf.xqueryFileExtension = ".xq";
- indexConf.xqueryxFileExtension = "xqx";
- indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/");
- } catch (IOException e) {
- e.printStackTrace();
- }
- return indexConf;
-
}
public TestCaseResult run(final TestCase testCase) {
TestCaseResult res = new TestCaseResult(testCase);
- TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase));
- testCaseIndex.setFolder("Indexing/Partition-1/");
- testCaseIndex.setName("showIndexes");
- runQuery(testCaseIndex, res);
- String[] collections = res.result.split("\n");
- this.collectionList = Arrays.asList(collections);
runQueries(testCase, res);
return res;
}
@@ -121,83 +74,32 @@
long start = System.currentTimeMillis();
try {
- try {
- if (opts.showQuery) {
+ String query = FileUtils.readFileToString(testCase.getXQueryFile(), "UTF-8");
- FileInputStream query = new FileInputStream(testCase.getXQueryFile());
- System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
- System.err.println(IOUtils.toString(query, "UTF-8"));
- query.close();
+ if (opts.showQuery) {
+ System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": ");
+ System.err.println(query);
+ }
+
+ QueryRequest request = createQueryRequest(opts, query);
+ APIResponse response = sendQueryRequest(request, testCase.getSourceFileMap());
+ if (response instanceof SyncQueryResponse) {
+ res.result = ((SyncQueryResponse) response).getResults();
+ } else {
+ System.err.println("Error response: Failure when running the query");
+ ErrorResponse errorResponse = (ErrorResponse) response;
+ Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(errorResponse.getError().getMessage());
+
+ Exception e = new RuntimeException("Failed to run the query");
+ if (m.find()) {
+ String eCode = m.group(1);
+ throw new SystemException(ErrorCode.valueOf(eCode), e);
+ } else {
+ throw e;
}
-
- VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET,
- opts.showOET, opts.showRP);
-
- Map<String, NodeControllerInfo> nodeControllerInfos = null;
- if (hcc != null) {
- nodeControllerInfos = hcc.getNodeControllerInfos();
- }
-
- XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize,
- opts.hdfsConf);
- Reader in = new InputStreamReader(new FileInputStream(testCase.getXQueryFile()), "UTF-8");
- CompilerControlBlock ccb = new CompilerControlBlock(
- new StaticContextImpl(RootStaticContextImpl.INSTANCE),
- new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap());
- compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList);
- JobSpecification spec = compiler.getModule().getHyracksJobSpecification();
- in.close();
-
- DynamicContext dCtx = new DynamicContextImpl(compiler.getModule().getModuleContext());
- spec.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
-
- spec.setMaxReattempts(0);
- JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
-
- FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize());
- IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
- IHyracksDatasetReader reader = hds.createReader(jobId, ccb.getResultSetId());
- // TODO(tillw) remove this loop once the IHyracksDatasetReader reliably returns the correct exception
- while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) {
- Thread.sleep(1);
- }
- IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
- res.result = "";
- while (reader.read(frame) > 0) {
- res.result += ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor);
- frame.getBuffer().clear();
- }
- res.result.trim();
- hcc.waitForCompletion(jobId);
- } catch (HyracksException e) {
- Throwable t = e;
- while (t.getCause() != null) {
- t = t.getCause();
- }
- final String message = t.getMessage();
- if (message != null) {
- Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message);
- if (m.find()) {
- String eCode = m.group(1);
- throw new SystemException(ErrorCode.valueOf(eCode), e);
- }
- }
- throw e;
}
} catch (Throwable e) {
- // Check for nested SystemExceptions.
- Throwable error = e;
- while (error != null) {
- if (error instanceof SystemException) {
- res.error = error;
- break;
- }
- error = error.getCause();
- }
- // Default
- if (res.error == null) {
- res.error = e;
- }
+ res.error = e;
} finally {
try {
res.compare();
@@ -208,6 +110,7 @@
long end = System.currentTimeMillis();
res.time = end - start;
}
+
if (opts.showResult) {
if (res.result == null) {
System.err.println("***Error: ");
@@ -218,7 +121,55 @@
System.err.println(res.result);
}
}
+ }
+ private static QueryRequest createQueryRequest(XTestOptions opts, String query) {
+ QueryRequest request = new QueryRequest(query);
+ request.setCompileOnly(opts.compileOnly);
+ request.setOptimization(opts.optimizationLevel);
+ request.setFrameSize(opts.frameSize);
+ request.setShowAbstractSyntaxTree(opts.showAST);
+ request.setShowTranslatedExpressionTree(opts.showTET);
+ request.setShowOptimizedExpressionTree(opts.showOET);
+ request.setShowRuntimePlan(opts.showRP);
+ request.setAsync(false);
+
+ return request;
+ }
+
+ private static APIResponse sendQueryRequest(QueryRequest request, Map<String, File> sourceFileMap)
+ throws IOException, URISyntaxException {
+
+ URI uri = RestUtils.buildQueryURI(request, TestClusterUtil.localClusterUtil.getIpAddress(),
+ TestClusterUtil.localClusterUtil.getRestPort());
+ CloseableHttpClient httpClient = HttpClients.custom().build();
+
+ try {
+ HttpPost httpRequest = new HttpPost(uri);
+ httpRequest.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_JSON);
+
+ ObjectMapper mapper = new ObjectMapper();
+ String fileMap = mapper.writeValueAsString(sourceFileMap);
+ httpRequest.setEntity(new StringEntity(fileMap, StandardCharsets.UTF_8));
+
+ try (CloseableHttpResponse httpResponse = httpClient.execute(httpRequest)) {
+ HttpEntity entity = httpResponse.getEntity();
+ String response = RestUtils.readEntity(entity);
+ if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ return RestUtils.mapEntity(response, SyncQueryResponse.class, CONTENT_TYPE_JSON);
+ } else {
+ return RestUtils.mapEntity(response, ErrorResponse.class, CONTENT_TYPE_JSON);
+ }
+ } catch (IOException e) {
+ System.err.println("Error occurred when reading entity: " + e.getMessage());
+ } catch (JAXBException e) {
+ System.err.println("Error occurred when mapping query response: " + e.getMessage());
+ }
+ } finally {
+ HttpClientUtils.closeQuietly(httpClient);
+ }
+
+ return null;
}
public void runQueries(TestCase testCase, TestCaseResult res) {
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java
index 5aae691..df7a71d 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java
@@ -16,10 +16,6 @@
*/
package org.apache.vxquery.xtest;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.mortbay.jetty.Server;
-
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,6 +24,8 @@
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.mortbay.jetty.Server;
+
public class XTest {
private XTestOptions opts;
private Server server;
@@ -36,8 +34,6 @@
private TestRunnerFactory trf;
private int count;
private int finishCount;
- private static NodeControllerService nc;
- private static ClusterControllerService cc;
XTest(XTestOptions opts) {
this.opts = opts;
@@ -81,8 +77,7 @@
}
}
});
- cc = TestClusterUtil.startCC(opts);
- nc = TestClusterUtil.startNC();
+ TestClusterUtil.startCluster(opts, TestClusterUtil.localClusterUtil);
trf = new TestRunnerFactory(opts);
trf.registerReporters(reporters);
TestCaseFactory tcf = new TestCaseFactory(trf, eSvc, opts);
@@ -104,7 +99,7 @@
r.close();
}
try {
- TestClusterUtil.stopCluster(cc, nc);
+ TestClusterUtil.stopCluster(TestClusterUtil.localClusterUtil);
} catch (IOException e) {
e.printStackTrace();
}
diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
index 1e2dcf6..8f77de4 100644
--- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
+++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java
@@ -22,8 +22,6 @@
import java.io.IOException;
import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.nc.NodeControllerService;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -38,8 +36,6 @@
private TestRunner tr;
private static MiniDFS dfs;
private final static String TMP = "target/tmp";
- private static NodeControllerService nc;
- private static ClusterControllerService cc;
protected abstract XTestOptions getTestOptions();
@@ -92,8 +88,7 @@
@BeforeClass
public static void setup() throws IOException {
- cc = TestClusterUtil.startCC(getDefaultTestOptions());
- nc = TestClusterUtil.startNC();
+ TestClusterUtil.startCluster(getDefaultTestOptions(), TestClusterUtil.localClusterUtil);
setupFS();
}
@@ -116,7 +111,7 @@
@AfterClass
public static void shutdown() throws IOException {
removeFS();
- TestClusterUtil.stopCluster(cc, nc);
+ TestClusterUtil.stopCluster(TestClusterUtil.localClusterUtil);
}
public static void removeFS() throws IOException {
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt
new file mode 100644
index 0000000..b1db973
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-1.txt
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>32</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt
new file mode 100644
index 0000000..d93567e
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-2.txt
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>31</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt
new file mode 100644
index 0000000..2ab8764
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-3.txt
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>33</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt
new file mode 100644
index 0000000..d1d6bb7
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-4.txt
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>31</value>
+<value>33</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt
new file mode 100644
index 0000000..2044b18
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-2/q03_records-5.txt
@@ -0,0 +1,3 @@
+<value>31</value>
+<value>33</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt
new file mode 100644
index 0000000..b1db973
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-1.txt
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>32</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt
new file mode 100644
index 0000000..d93567e
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-2.txt
@@ -0,0 +1,3 @@
+<value>33</value>
+<value>31</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt
new file mode 100644
index 0000000..2ab8764
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-3.txt
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>33</value>
+<value>31</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt
new file mode 100644
index 0000000..d1d6bb7
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-4.txt
@@ -0,0 +1,3 @@
+<value>32</value>
+<value>31</value>
+<value>33</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt
new file mode 100644
index 0000000..2044b18
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/GhcndRecords/Partition-4/q03_records-5.txt
@@ -0,0 +1,3 @@
+<value>31</value>
+<value>33</value>
+<value>32</value>
\ No newline at end of file
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt
new file mode 100644
index 0000000..ea5a19e
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-1.txt
@@ -0,0 +1,3 @@
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt
new file mode 100644
index 0000000..46985a9
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-2.txt
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt
new file mode 100644
index 0000000..b97b850
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-3.txt
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt
new file mode 100644
index 0000000..0bacd2b
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-4.txt
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt
new file mode 100644
index 0000000..0e79fdc
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-1/q15_parser-5.txt
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt
new file mode 100644
index 0000000..ea5a19e
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-1.txt
@@ -0,0 +1,3 @@
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt
new file mode 100644
index 0000000..46985a9
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-2.txt
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt
new file mode 100644
index 0000000..b97b850
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-3.txt
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt
new file mode 100644
index 0000000..0bacd2b
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-4.txt
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt
new file mode 100644
index 0000000..0e79fdc
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-2/q15_parser-5.txt
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt
new file mode 100644
index 0000000..ea5a19e
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-1.txt
@@ -0,0 +1,3 @@
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt
new file mode 100644
index 0000000..46985a9
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-2.txt
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt
new file mode 100644
index 0000000..b97b850
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-3.txt
@@ -0,0 +1,3 @@
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt
new file mode 100644
index 0000000..0bacd2b
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-4.txt
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt
new file mode 100644
index 0000000..0e79fdc
--- /dev/null
+++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Json/Parser/Partition-4/q15_parser-5.txt
@@ -0,0 +1,3 @@
+{"date":"2003-03-03T00:00:00.000","datatype":"TMIN","station":"GHCND:AS000000003","attributes":",,","value":13.75}
+{"date":"2002-02-02T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000002","attributes":",,","value":12.5}
+{"date":"2001-01-01T00:00:00.000","datatype":"TMIN","station":"GHCND:US000000001","attributes":",,","value":11.25}
diff --git a/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml b/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml
index 1287225..9f07e2b 100644
--- a/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml
+++ b/vxquery-xtest/src/test/resources/cat/JsonParserQueries.xml
@@ -89,16 +89,31 @@
<description>Parsing a collection of json files.</description>
<query name="q15_parser" date="2016-07-15"/>
<output-file compare="Text">q15_parser.txt</output-file>
+ <output-file compare="Text">q15_parser-1.txt</output-file>
+ <output-file compare="Text">q15_parser-2.txt</output-file>
+ <output-file compare="Text">q15_parser-3.txt</output-file>
+ <output-file compare="Text">q15_parser-4.txt</output-file>
+ <output-file compare="Text">q15_parser-5.txt</output-file>
</test-case>
<test-case name="json-parser-q15-2" FilePath="Json/Parser/Partition-2" Creator="Christina Pavlopoulou">
<description>Parsing a collection of json files.</description>
<query name="q15_parser" date="2016-07-15"/>
- <output-file compare="Text">q15_parser.txt</output-file>
+ <output-file compare="Text">q15_parser.txt</output-file>
+ <output-file compare="Text">q15_parser-1.txt</output-file>
+ <output-file compare="Text">q15_parser-2.txt</output-file>
+ <output-file compare="Text">q15_parser-3.txt</output-file>
+ <output-file compare="Text">q15_parser-4.txt</output-file>
+ <output-file compare="Text">q15_parser-5.txt</output-file>
</test-case>
<test-case name="json-parser-q15-4" FilePath="Json/Parser/Partition-4" Creator="Christina Pavlopoulou">
<description>Parsing a collection of json files.</description>
<query name="q15_parser" date="2016-07-15"/>
- <output-file compare="Text">q15_parser.txt</output-file>
+ <output-file compare="Text">q15_parser.txt</output-file>
+ <output-file compare="Text">q15_parser-1.txt</output-file>
+ <output-file compare="Text">q15_parser-2.txt</output-file>
+ <output-file compare="Text">q15_parser-3.txt</output-file>
+ <output-file compare="Text">q15_parser-4.txt</output-file>
+ <output-file compare="Text">q15_parser-5.txt</output-file>
</test-case>
<test-case name="json-parser-q16" FilePath="Json/Parser/Partition-1" Creator="Christina Pavlopoulou">
<description>Parsing a collection of json files.</description>
