NUTCH-2757 - Indexer-elastic: add authentication options
diff --git a/conf/index-writers.xml.template b/conf/index-writers.xml.template
index 4eb4386..aa9a90b 100644
--- a/conf/index-writers.xml.template
+++ b/conf/index-writers.xml.template
@@ -109,9 +109,9 @@
<param name="host" value="localhost"/>
<param name="port" value="9200"/>
<param name="index" value="nutch"/>
- <!--Configure username and password if necessary, default user is "elastic"
- <param name="user" value="user"/>
- <param name="password" value="password"/>-->
+ <param name="username" value="elastic"/>
+ <param name="password" value=""/>
+ <!--<param name="auth" value="false"/>-->
<param name="max.bulk.docs" value="250"/>
<param name="max.bulk.size" value="2500500"/>
<param name="exponential.backoff.millis" value="100"/>
diff --git a/src/plugin/indexer-elastic/README.md b/src/plugin/indexer-elastic/README.md
index 41316de..b2a875b 100644
--- a/src/plugin/indexer-elastic/README.md
+++ b/src/plugin/indexer-elastic/README.md
@@ -33,8 +33,9 @@
host | Comma-separated list of hostnames to send documents to using [TransportClient](https://static.javadoc.io/org.elasticsearch/elasticsearch/5.3.0/org/elasticsearch/client/transport/TransportClient.html). Either host and port must be defined or cluster. |
port | The port to connect to using [TransportClient](https://static.javadoc.io/org.elasticsearch/elasticsearch/5.3.0/org/elasticsearch/client/transport/TransportClient.html). | 9300
index | Default index to send documents to. | nutch
-user | Username for auth credentials | elastic
+username | Username for auth credentials | elastic
password | Password for auth credentials | ""
+auth | Whether to enable HTTP basic authentication with elastic. Use `username` and `password` properties to configure your credentials. | false
max.bulk.docs | Maximum size of the bulk in number of documents. | 250
max.bulk.size | Maximum size of the bulk in bytes. | 2500500
exponential.backoff.millis | Initial delay for the [BulkProcessor](https://static.javadoc.io/org.elasticsearch/elasticsearch/5.3.0/org/elasticsearch/action/bulk/BulkProcessor.html) exponential backoff policy. | 100
diff --git a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticConstants.java b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticConstants.java
index 7a65d0f..c0d1a61 100644
--- a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticConstants.java
+++ b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticConstants.java
@@ -20,10 +20,9 @@
String HOSTS = "host";
String PORT = "port";
- String USER = "user";
+ String USER = "username";
String PASSWORD = "password";
-
- String CLUSTER = "cluster";
+ String USE_AUTH = "auth";
String INDEX = "index";
String MAX_BULK_DOCS = "max.bulk.docs";
String MAX_BULK_LENGTH = "max.bulk.size";
diff --git a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
index 5f772ba..a9aa354 100644
--- a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
+++ b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
@@ -86,13 +86,12 @@
private static final String DEFAULT_INDEX = "nutch";
private static final String DEFAULT_USER = "elastic";
- private String cluster;
private String[] hosts;
private int port;
private Boolean https = null;
private String user = null;
private String password = null;
- private Boolean trustAllHostnames = null;
+ private Boolean auth;
private int maxBulkDocs;
private int maxBulkLength;
@@ -122,7 +121,7 @@
*/
@Override
public void open(IndexWriterParams parameters) throws IOException {
- cluster = parameters.get(ElasticConstants.CLUSTER);
+
String hosts = parameters.get(ElasticConstants.HOSTS);
if (StringUtils.isBlank(hosts)) {
@@ -170,6 +169,7 @@
hosts = parameters.getStrings(ElasticConstants.HOSTS);
port = parameters.getInt(ElasticConstants.PORT, DEFAULT_PORT);
+ auth = parameters.getBoolean(ElasticConstants.USE_AUTH, false);
user = parameters.get(ElasticConstants.USER, DEFAULT_USER);
password = parameters.get(ElasticConstants.PASSWORD, "");
@@ -186,15 +186,7 @@
hostsList[i++] = new HttpHost(host, port);
}
RestClientBuilder restClientBuilder = RestClient.builder(hostsList);
- if (StringUtils.isNotBlank(cluster)) {
- Header[] defaultHeaders = new Header[] {
- new BasicHeader("cluster.name", cluster) };
- restClientBuilder.setDefaultHeaders(defaultHeaders);
- } else {
- LOG.debug("No cluster name provided so using default");
- }
- // Only add username and password if password is configured
- if (StringUtils.isNotBlank(password)) {
+ if (auth) {
restClientBuilder
.setHttpClientConfigCallback(new HttpClientConfigCallback() {
@Override
@@ -310,17 +302,17 @@
public Map<String, Map.Entry<String, Object>> describe() {
Map<String, Map.Entry<String, Object>> properties = new LinkedHashMap<>();
- properties.put(ElasticConstants.CLUSTER, new AbstractMap.SimpleEntry<>(
- "The cluster name to discover. Either host and port must be defined or cluster.",
- this.cluster));
- properties.put(ElasticConstants.HOSTS, new AbstractMap.SimpleEntry<>(
- "Comma-separated list of hostnames to send documents to using TransportClient. "
- + "Either host and port must be defined or cluster.",
- this.hosts == null ? "" : String.join(",", hosts)));
+ properties.put(ElasticConstants.HOSTS,
+ new AbstractMap.SimpleEntry<>("Comma-separated list of hostnames",
+ this.hosts == null ? "" : String.join(",", hosts)));
properties.put(ElasticConstants.PORT, new AbstractMap.SimpleEntry<>(
- "The port to connect to using TransportClient.", this.port));
+ "The port to connect to elastic server.", this.port));
properties.put(ElasticConstants.INDEX, new AbstractMap.SimpleEntry<>(
"Default index to send documents to.", this.defaultIndex));
+ properties.put(ElasticConstants.USER, new AbstractMap.SimpleEntry<>(
+ "Username for auth credentials", this.user));
+ properties.put(ElasticConstants.PASSWORD, new AbstractMap.SimpleEntry<>(
+ "Password for auth credentials", this.password));
properties.put(ElasticConstants.MAX_BULK_DOCS,
new AbstractMap.SimpleEntry<>(
"Maximum size of the bulk in number of documents.",
