contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.drill.exec.store.http;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import okhttp3.HttpUrl;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.drill.common.PlanStringBuilder;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.security.CredentialsProvider;
import org.apache.drill.exec.store.security.CredentialProviderUtils;
import org.apache.drill.exec.store.security.UsernamePasswordCredentials;
import org.apache.drill.exec.store.security.UsernamePasswordWithProxyCredentials;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

@JsonInclude(JsonInclude.Include.NON_DEFAULT)
@JsonDeserialize(builder = HttpApiConfig.HttpApiConfigBuilder.class)
public class HttpApiConfig {
  private static final Logger logger = LoggerFactory.getLogger(HttpApiConfig.class);

  protected static final String DEFAULT_INPUT_FORMAT = "json";
  protected static final String CSV_INPUT_FORMAT = "csv";
  protected static final String XML_INPUT_FORMAT = "xml";

  public static final String QUERY_STRING_POST_LOCATION = "query_string";
  public static final String POST_BODY_POST_LOCATION = "post_body";
  public static final String XML_BODY_POST_LOCATION = "xml_body";
  public static final String JSON_BODY_POST_LOCATION = "json_body";

  @JsonProperty
  private final String url;
  /**
   * Whether this API configuration represents a schema (with the
   * table providing additional parts of the URL), or if this
   * API represents a table (the URL is complete except for
   * parameters specified in the WHERE clause.)
   */
  @JsonInclude
  @JsonProperty
  private final boolean requireTail;

  @JsonProperty
  private final String method;

  @JsonProperty
  private final String postBody;

  @JsonProperty
  private final Map<String, String> headers;

  /**
   * List of query parameters which can be used in the SQL WHERE clause
   * to push filters to the REST request as HTTP query parameters.
   */
  @JsonProperty
  private final List<String> params;

  /**
   * Path within the message to the JSON object, or array of JSON
   * objects, which contain the actual data. Allows a request to
   * skip over "overhead" such as status codes. Must be a slash-delimited
   * set of JSON field names.
   */
  @JsonProperty
  private final String dataPath;

  @JsonProperty
  private final String authType;
  @JsonProperty
  private final String inputType;
  @JsonProperty
  @Deprecated
  private final int xmlDataLevel;
  @JsonProperty
  private final String limitQueryParam;
  @JsonProperty
  private final String postParameterLocation;

  @JsonProperty
  private final boolean errorOn400;

  @JsonProperty
  private final boolean caseSensitiveFilters;

  // Enables the user to configure JSON options at the connection level rather than globally.
  @JsonProperty
  private final HttpJsonOptions jsonOptions;

  @JsonProperty
  private final HttpXmlOptions xmlOptions;

  @JsonInclude
  @JsonProperty
  private final boolean verifySSLCert;
  private final CredentialsProvider credentialsProvider;
  @JsonProperty
  private final HttpPaginatorConfig paginator;

  protected boolean directCredentials;

  public static HttpApiConfigBuilder builder() {
    return new HttpApiConfigBuilder();
  }

  public String url() {
    return this.url;
  }

  public boolean requireTail() {
    return this.requireTail;
  }

  public String method() {
    return this.method;
  }

  public String postBody() {
    return this.postBody;
  }

  public Map<String, String> headers() {
    return this.headers;
  }

  public List<String> params() {
    return this.params;
  }

  public String dataPath() {
    return this.dataPath;
  }

  public String authType() {
    return this.authType;
  }

  public String inputType() {
    return this.inputType;
  }

  public boolean caseSensitiveFilters() {
    return this.caseSensitiveFilters;
  }

  @Deprecated
  public int xmlDataLevel() {
    return this.xmlDataLevel;
  }

  public String limitQueryParam() {
    return this.limitQueryParam;
  }

  public boolean errorOn400() {
    return this.errorOn400;
  }

  public HttpJsonOptions jsonOptions() {
    return this.jsonOptions;
  }
  public HttpXmlOptions xmlOptions() {
    return this.xmlOptions;
  }

  public boolean verifySSLCert() {
    return this.verifySSLCert;
  }

  public HttpPaginatorConfig paginator() {
    return this.paginator;
  }

  public String getPostParameterLocation() {
    return postParameterLocation;
  }

  @Override
  public boolean equals(Object o) {
    if (this == o) {
      return true;
    }
    if (o == null || getClass() != o.getClass()) {
      return false;
    }
    HttpApiConfig that = (HttpApiConfig) o;
    return requireTail == that.requireTail
      && errorOn400 == that.errorOn400
      && verifySSLCert == that.verifySSLCert
      && directCredentials == that.directCredentials
      && caseSensitiveFilters == that.caseSensitiveFilters
      && Objects.equals(url, that.url)
      && Objects.equals(method, that.method)
      && Objects.equals(postBody, that.postBody)
      && Objects.equals(headers, that.headers)
      && Objects.equals(params, that.params)
      && Objects.equals(postParameterLocation, that.postParameterLocation)
      && Objects.equals(dataPath, that.dataPath)
      && Objects.equals(authType, that.authType)
      && Objects.equals(inputType, that.inputType)
      && Objects.equals(limitQueryParam, that.limitQueryParam)
      && Objects.equals(jsonOptions, that.jsonOptions)
      && Objects.equals(xmlOptions, that.xmlOptions)
      && Objects.equals(credentialsProvider, that.credentialsProvider)
      && Objects.equals(paginator, that.paginator);
  }

  @Override
  public int hashCode() {
    return Objects.hash(url, requireTail, method, postBody, headers, params, dataPath,
      authType, inputType, limitQueryParam, errorOn400, jsonOptions, xmlOptions, verifySSLCert,
      credentialsProvider, paginator, directCredentials, postParameterLocation, caseSensitiveFilters);
  }

  @Override
  public String toString() {
    return new PlanStringBuilder(this)
      .field("url", url)
      .field("requireTail", requireTail)
      .field("method", method)
      .field("postBody", postBody)
      .field("postParameterLocation", postParameterLocation)
      .field("headers", headers)
      .field("params", params)
      .field("dataPath", dataPath)
      .field("caseSensitiveFilters", caseSensitiveFilters)
      .field("authType", authType)
      .field("inputType", inputType)
      .field("limitQueryParam", limitQueryParam)
      .field("errorOn400", errorOn400)
      .field("jsonOptions", jsonOptions)
      .field("xmlOptions", xmlOptions)
      .field("verifySSLCert", verifySSLCert)
      .field("credentialsProvider", credentialsProvider)
      .field("paginator", paginator)
      .field("directCredentials", directCredentials)
      .toString();
  }

  /**
   * Config variable to determine how POST variables are sent to the downstream API
   */
  public enum PostLocation {
    /**
     * Parameters from the query other than static parameters are pushed to
     * the query string, as in a GET request
     */
    QUERY_STRING,
    /**
     * All POST parameters, both static and from the query, are pushed to the POST body
     * as key/value pairs
     */
    POST_BODY,
    /**
     * All POST parameters, both static and from the query, are pushed to the POST body
     * as a JSON object.
     */
    JSON_BODY,
    /**
     * All POST parameters, both static and from the query, are pushed to the POST body
     * as an XML request.
     */
    XML_BODY
  }

  public enum HttpMethod {
    /**
     * Value for HTTP GET method
     */
    GET,
    /**
     * Value for HTTP POST method
     */
    POST
  }

  private HttpApiConfig(HttpApiConfig.HttpApiConfigBuilder builder) {
    this.headers = builder.headers;
    this.method = StringUtils.isEmpty(builder.method)
        ? HttpMethod.GET.toString() : builder.method.trim().toUpperCase();
    this.url = builder.url;
    this.jsonOptions = builder.jsonOptions;
    this.xmlOptions = builder.xmlOptions;

    HttpMethod httpMethod = HttpMethod.valueOf(this.method);
    // Get the request method.  Only accept GET and POST requests.  Anything else will default to GET.
    switch (httpMethod) {
      case GET:
      case POST:
        break;
      default:
        throw UserException
          .validationError()
          .message("Invalid HTTP method: %s.  Drill supports 'GET' and , 'POST'.", method)
          .build(logger);
    }
    if (StringUtils.isEmpty(url)) {
      throw UserException
        .validationError()
        .message("URL is required for the HTTP storage plugin.")
        .build(logger);
    }

    // Default to query string to avoid breaking changes
    this.postParameterLocation = StringUtils.isEmpty(builder.postParameterLocation) ?
      PostLocation.QUERY_STRING.toString() : builder.postParameterLocation.trim().toUpperCase();

    // Get the authentication method. Future functionality will include OAUTH2 authentication but for now
    // Accept either basic or none.  The default is none.
    this.authType = StringUtils.defaultIfEmpty(builder.authType, "none");
    this.postBody = builder.postBody;



    this.params = CollectionUtils.isEmpty(builder.params) ? null :
      ImmutableList.copyOf(builder.params);
    this.dataPath = StringUtils.defaultIfEmpty(builder.dataPath, null);

    // Default to true for backward compatibility with first PR.
    this.requireTail = builder.requireTail;

    // Default to true for backward compatibility, and better security practices
    this.verifySSLCert = builder.verifySSLCert;

    this.inputType = builder.inputType.trim().toLowerCase();

    this.xmlDataLevel = Math.max(1, builder.xmlDataLevel);
    this.errorOn400 = builder.errorOn400;
    this.caseSensitiveFilters = builder.caseSensitiveFilters;
    this.credentialsProvider = CredentialProviderUtils.getCredentialsProvider(builder.userName, builder.password, builder.credentialsProvider);
    this.directCredentials = builder.credentialsProvider == null;

    this.limitQueryParam = builder.limitQueryParam;
    this.paginator = builder.paginator;
  }

  @JsonProperty
  public String userName() {
    if (!directCredentials) {
      return null;
    }
    return getUsernamePasswordCredentials()
      .map(UsernamePasswordCredentials::getUsername)
      .orElse(null);
  }

  @JsonProperty
  public String password() {
    if (!directCredentials) {
      return null;
    }
    return getUsernamePasswordCredentials()
      .map(UsernamePasswordCredentials::getPassword)
      .orElse(null);
  }

  @JsonIgnore
  public HttpUrl getHttpUrl() {
    return HttpUrl.parse(this.url);
  }

  @JsonIgnore
  public HttpMethod getMethodType() {
    return HttpMethod.valueOf(this.method);
  }

  @JsonIgnore
  public PostLocation getPostLocation() {
    return PostLocation.valueOf(this.postParameterLocation);
  }

  @JsonIgnore
  public Optional<UsernamePasswordWithProxyCredentials> getUsernamePasswordCredentials() {
    return new UsernamePasswordWithProxyCredentials.Builder()
      .setCredentialsProvider(credentialsProvider)
      .build();
  }

  @JsonIgnore
  public Optional<UsernamePasswordWithProxyCredentials> getUsernamePasswordCredentials(String username) {
    return new UsernamePasswordWithProxyCredentials.Builder()
      .setCredentialsProvider(credentialsProvider)
      .setQueryUser(username)
      .build();
  }

  @JsonProperty
  public CredentialsProvider credentialsProvider() {
    if (directCredentials) {
      return null;
    }
    return credentialsProvider;
  }

  @JsonPOJOBuilder(withPrefix = "")
  public static class HttpApiConfigBuilder {
    private String userName;

    private String password;

    private boolean requireTail = true;

    private boolean verifySSLCert = true;

    private String inputType = DEFAULT_INPUT_FORMAT;

    private String url;

    private String method;

    private String postBody;

    private String postParameterLocation = QUERY_STRING_POST_LOCATION;

    private Map<String, String> headers;

    private List<String> params;

    private String dataPath;

    private String authType;

    private int xmlDataLevel;
    private boolean caseSensitiveFilters;
    private String limitQueryParam;

    private boolean errorOn400;

    private HttpJsonOptions jsonOptions;
    private HttpXmlOptions xmlOptions;

    private CredentialsProvider credentialsProvider;

    private HttpPaginatorConfig paginator;

    private boolean directCredentials;

    public HttpApiConfig build() {
      return new HttpApiConfig(this);
    }

    public String userName() {
      return this.userName;
    }

    public String password() {
      return this.password;
    }

    public boolean requireTail() {
      return this.requireTail;
    }

    public boolean verifySSLCert() {
      return this.verifySSLCert;
    }

    public String inputType() {
      return this.inputType;
    }

    public HttpApiConfigBuilder userName(String userName) {
      this.userName = userName;
      return this;
    }

    public HttpApiConfigBuilder password(String password) {
      this.password = password;
      return this;
    }

    public HttpApiConfigBuilder xmlOptions(HttpXmlOptions options) {
      this.xmlOptions = options;
      return this;
    }

    public HttpApiConfigBuilder requireTail(boolean requireTail) {
      this.requireTail = requireTail;
      return this;
    }

    public HttpApiConfigBuilder verifySSLCert(boolean verifySSLCert) {
      this.verifySSLCert = verifySSLCert;
      return this;
    }

    public HttpApiConfigBuilder inputType(String inputType) {
      this.inputType = inputType;
      return this;
    }

    public HttpApiConfigBuilder url(String url) {
      this.url = url;
      return this;
    }

    public HttpApiConfigBuilder caseSensitiveFilters(boolean caseSensitiveFilters) {
      this.caseSensitiveFilters = caseSensitiveFilters;
      return this;
    }

    public HttpApiConfigBuilder method(String method) {
      this.method = method;
      return this;
    }

    public HttpApiConfigBuilder postBody(String postBody) {
      this.postBody = postBody;
      return this;
    }

    public HttpApiConfigBuilder postParameterLocation(String postParameterLocation) {
      this.postParameterLocation = postParameterLocation;
      return this;
    }

    public HttpApiConfigBuilder headers(Map<String, String> headers) {
      this.headers = headers;
      return this;
    }

    public HttpApiConfigBuilder params(List<String> params) {
      this.params = params;
      return this;
    }

    public HttpApiConfigBuilder dataPath(String dataPath) {
      this.dataPath = dataPath;
      return this;
    }

    public HttpApiConfigBuilder authType(String authType) {
      this.authType = authType;
      return this;
    }

    /**
     * Do not use.  Use xmlOptions instead to set XML data level.
     * @param xmlDataLevel
     * @return
     */
    @Deprecated
    public HttpApiConfigBuilder xmlDataLevel(int xmlDataLevel) {
      this.xmlDataLevel = xmlDataLevel;
      return this;
    }

    public HttpApiConfigBuilder limitQueryParam(String limitQueryParam) {
      this.limitQueryParam = limitQueryParam;
      return this;
    }

    public HttpApiConfigBuilder errorOn400(boolean errorOn400) {
      this.errorOn400 = errorOn400;
      return this;
    }

    public HttpApiConfigBuilder jsonOptions(HttpJsonOptions jsonOptions) {
      this.jsonOptions = jsonOptions;
      return this;
    }

    public HttpApiConfigBuilder credentialsProvider(CredentialsProvider credentialsProvider) {
      this.credentialsProvider = credentialsProvider;
      return this;
    }

    public HttpApiConfigBuilder paginator(HttpPaginatorConfig paginator) {
      this.paginator = paginator;
      return this;
    }

    public HttpApiConfigBuilder directCredentials(boolean directCredentials) {
      this.directCredentials = directCredentials;
      return this;
    }
  }
}