DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin (#2639)
* DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin
diff --git a/contrib/storage-http/README.md b/contrib/storage-http/README.md
index 0f188ba..d384bed 100644
--- a/contrib/storage-http/README.md
+++ b/contrib/storage-http/README.md
@@ -306,6 +306,9 @@
Default is `true`, but when set to false, Drill will trust all SSL certificates. Useful for debugging or on internal corporate networks using self-signed certificates or
private certificate authorities.
+#### caseSensitiveFilters
+Some APIs are case sensitive with the fields which are pushed down. If the endpoint that you are working with is in fact case sensitive, simply set this to `true`. Defaults to `false`.
+
## Usage
This plugin is different from other plugins in that it the table component of the `FROM` clause
@@ -577,9 +580,9 @@
you may encounter errors or empty responses if you are counting on the endpoint for
redirection.
-2. At this time, the plugin does not support any authentication other than basic authentication.
+~~2. At this time, the plugin does not support any authentication other than basic authentication.~~
-3. This plugin does not implement join filter pushdowns (only constant plushdowns are
+3. This plugin does not implement join filter pushdowns (only constant pushdowns are
supported). Join pushdown has the potential to improve performance if you use the HTTP service
joined to another table.
diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
index 581b61c..91af33e 100644
--- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
+++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java
@@ -104,6 +104,9 @@
@JsonProperty
private final boolean errorOn400;
+ @JsonProperty
+ private final boolean caseSensitiveFilters;
+
// Enables the user to configure JSON options at the connection level rather than globally.
@JsonProperty
private final HttpJsonOptions jsonOptions;
@@ -157,6 +160,10 @@
return this.inputType;
}
+ public boolean caseSensitiveFilters() {
+ return this.caseSensitiveFilters;
+ }
+
public int xmlDataLevel() {
return this.xmlDataLevel;
}
@@ -199,6 +206,7 @@
&& errorOn400 == that.errorOn400
&& verifySSLCert == that.verifySSLCert
&& directCredentials == that.directCredentials
+ && caseSensitiveFilters == that.caseSensitiveFilters
&& Objects.equals(url, that.url)
&& Objects.equals(method, that.method)
&& Objects.equals(postBody, that.postBody)
@@ -218,7 +226,7 @@
public int hashCode() {
return Objects.hash(url, requireTail, method, postBody, headers, params, dataPath,
authType, inputType, xmlDataLevel, limitQueryParam, errorOn400, jsonOptions, verifySSLCert,
- credentialsProvider, paginator, directCredentials, postParameterLocation);
+ credentialsProvider, paginator, directCredentials, postParameterLocation, caseSensitiveFilters);
}
@Override
@@ -232,6 +240,7 @@
.field("headers", headers)
.field("params", params)
.field("dataPath", dataPath)
+ .field("caseSensitiveFilters", caseSensitiveFilters)
.field("authType", authType)
.field("inputType", inputType)
.field("xmlDataLevel", xmlDataLevel)
@@ -328,6 +337,7 @@
this.xmlDataLevel = Math.max(1, builder.xmlDataLevel);
this.errorOn400 = builder.errorOn400;
+ this.caseSensitiveFilters = builder.caseSensitiveFilters;
this.credentialsProvider = CredentialProviderUtils.getCredentialsProvider(builder.userName, builder.password, builder.credentialsProvider);
this.directCredentials = builder.credentialsProvider == null;
@@ -422,7 +432,7 @@
private String authType;
private int xmlDataLevel;
-
+ private boolean caseSensitiveFilters;
private String limitQueryParam;
private boolean errorOn400;
@@ -489,6 +499,11 @@
return this;
}
+ public HttpApiConfigBuilder caseSensitiveFilters(boolean caseSensitiveFilters) {
+ this.caseSensitiveFilters = caseSensitiveFilters;
+ return this;
+ }
+
public HttpApiConfigBuilder method(String method) {
this.method = method;
return this;
diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
index c4f5596..30f4be5 100644
--- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
+++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java
@@ -35,6 +35,7 @@
import org.apache.drill.exec.store.http.util.SimpleHttp;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -175,7 +176,13 @@
*/
@Override
public Pair<GroupScan, List<RexNode>> transform(AndNode andNode) {
- Map<String, String> filters = CaseInsensitiveMap.newHashMap();
+ Map<String, String> filters;
+ if (groupScan.getHttpConfig().caseSensitiveFilters()) {
+ filters = new HashMap<>();
+ } else {
+ filters = CaseInsensitiveMap.newHashMap();
+ }
+
double selectivity = 1;
for (ExprNode expr : andNode.children) {
ColRelOpConstNode relOp = (ColRelOpConstNode) expr;
diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
index e5f0ce7..ee5b285 100644
--- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
+++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java
@@ -35,7 +35,6 @@
import org.apache.drill.common.exceptions.EmptyErrorContext;
import org.apache.drill.common.logical.OAuthConfig;
import org.apache.drill.common.logical.StoragePluginConfig.AuthMode;
-import org.apache.drill.common.map.CaseInsensitiveMap;
import org.apache.drill.common.exceptions.CustomErrorContext;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
@@ -712,7 +711,6 @@
.message("API Query with URL Parameters must be populated.")
.build(logger);
}
- CaseInsensitiveMap<String>caseInsensitiveFilterMap = (CaseInsensitiveMap<String>)filters;
List<String> params = SimpleHttp.getURLParameters(url);
String tempUrl = SimpleHttp.decodedURL(url);
@@ -726,7 +724,7 @@
// of providing helpful errors in strange cases, it is there.
- String value = caseInsensitiveFilterMap.get(param);
+ String value = filters.get(param);
// Check and see if there is a default for this parameter. If not throw an error.
if (StringUtils.isEmpty(value)) {
