DRILL-8365: HTTP Plugin Places Parameters in Wrong Place (#2715)
diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
index cc17636..579643e 100644
--- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
+++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
@@ -263,8 +263,8 @@
logger.debug("Building URL from {}", baseUrl);
HttpApiConfig apiConfig = subScan.tableSpec().connectionConfig();
- // Append table name, if available.
- if (subScan.tableSpec().tableName() != null) {
+ // Append table name, if present. When pagination is used, the paginator adds this.
+ if (subScan.tableSpec().tableName() != null && paginator == null) {
baseUrl += subScan.tableSpec().tableName();
}
diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
index 3de783f..0da06a4 100644
--- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
+++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
@@ -110,7 +110,16 @@
private Paginator getPaginator() {
HttpUrl.Builder urlBuilder;
- HttpUrl rawUrl = HttpUrl.parse(subScan.tableSpec().connectionConfig().url());
+ HttpUrl rawUrl;
+
+ // Append table name, if present.
+ if (subScan.tableSpec().tableName() != null) {
+ rawUrl = HttpUrl.parse(subScan.tableSpec().connectionConfig().url() + subScan.tableSpec().tableName());
+ } else {
+ rawUrl = HttpUrl.parse(subScan.tableSpec().connectionConfig().url());
+ }
+
+
// If the URL is not parsable or otherwise invalid
if (rawUrl == null) {
diff --git a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
index 5931e0d..ed30d8d 100644
--- a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
+++ b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
@@ -216,6 +216,15 @@
.inputType("json")
.build();
+ HttpApiConfig mockJsonConfigWithPaginatorAndTail = HttpApiConfig.builder()
+ .url("http://localhost:8092/json")
+ .method("get")
+ .headers(headers)
+ .requireTail(true)
+ .paginator(offsetPaginatorForJson)
+ .inputType("json")
+ .build();
+
HttpPaginatorConfig pagePaginatorForXML = HttpPaginatorConfig.builder()
.method("page")
.pageParam("page")
@@ -268,6 +277,7 @@
configs.put("nested_keyset", mockJsonConfigWitNestedKeyset);
configs.put("nested_keyset_and_datapath", mockJsonConfigWitNestedKeysetAndDataPath);
configs.put("json_paginator", mockJsonConfigWithPaginator);
+ configs.put("json_tail", mockJsonConfigWithPaginatorAndTail);
configs.put("xml_paginator", mockXmlConfigWithPaginator);
configs.put("xml_paginator_url_params", mockXmlConfigWithPaginatorAndUrlParams);
@@ -319,6 +329,33 @@
}
@Test
+ public void simpleJSONPaginatorQueryWithTail() throws Exception {
+ String sql = "SELECT * FROM `local`.`json_tail`.`?arg1=foo` LIMIT 4";
+ try (MockWebServer server = startServer()) {
+
+ server.enqueue(new MockResponse().setResponseCode(200).setBody(TEST_JSON_PAGE1));
+ server.enqueue(new MockResponse().setResponseCode(200).setBody(TEST_JSON_PAGE2));
+ server.enqueue(new MockResponse().setResponseCode(200).setBody(TEST_JSON_PAGE3));
+
+ List<QueryDataBatch> results = client.queryBuilder()
+ .sql(sql)
+ .results();
+
+ int count = 0;
+ for(QueryDataBatch b : results){
+ count += b.getHeader().getRowCount();
+ b.release();
+ }
+ assertEquals(2, results.size());
+ assertEquals(4, count);
+
+ // Verify that the URLs are correct
+ RecordedRequest recordedRequest = server.takeRequest();
+ assertEquals("http://localhost:8092/json?arg1=foo&offset=0&limit=2", recordedRequest.getRequestUrl().toString());
+ }
+ }
+
+ @Test
public void simpleJSONPaginatorQueryWith429() throws Exception {
// This test simulates an http request that hits a burst limit. In this situation,
// Drill will wait and retry the request.
