NUTCH-2780 : Upgrade index-solr to use Solr 8.5.1
diff --git a/src/plugin/indexer-solr/ivy.xml b/src/plugin/indexer-solr/ivy.xml
index c68c238..1322e91 100644
--- a/src/plugin/indexer-solr/ivy.xml
+++ b/src/plugin/indexer-solr/ivy.xml
@@ -36,9 +36,9 @@
</publications>
<dependencies>
- <dependency org="org.apache.solr" name="solr-solrj" rev="7.3.1"/>
- <dependency org="org.apache.httpcomponents" name="httpcore" rev="4.4.6" conf="*->default"/>
- <dependency org="org.apache.httpcomponents" name="httpmime" rev="4.5.3" conf="*->default"/>
+ <dependency org="org.apache.solr" name="solr-solrj" rev="8.5.1"/>
+ <dependency org="org.apache.httpcomponents" name="httpcore" rev="4.4.12" conf="*->default"/>
+ <dependency org="org.apache.httpcomponents" name="httpmime" rev="4.5.10" conf="*->default"/>
</dependencies>
</ivy-module>
diff --git a/src/plugin/indexer-solr/plugin.xml b/src/plugin/indexer-solr/plugin.xml
index 0d91eab..0d9aacd 100644
--- a/src/plugin/indexer-solr/plugin.xml
+++ b/src/plugin/indexer-solr/plugin.xml
@@ -22,15 +22,36 @@
<library name="indexer-solr.jar">
<export name="*" />
</library>
- <library name="commons-io-2.5.jar"/>
- <library name="httpclient-4.5.3.jar"/>
+ <library name="commons-io-2.6.jar"/>
+ <library name="netty-buffer-4.1.29.Final.jar"/>
+ <library name="netty-codec-4.1.29.Final.jar"/>
+ <library name="netty-common-4.1.29.Final.jar"/>
+ <library name="netty-handler-4.1.29.Final.jar"/>
+ <library name="netty-resolver-4.1.29.Final.jar"/>
+ <library name="netty-transport-4.1.29.Final.jar"/>
+ <library name="netty-transport-native-epoll-4.1.29.Final.jar"/>
+ <library name="netty-transport-native-unix-common-4.1.29.Final.jar"/>
+ <library name="commons-math3-3.6.1.jar"/>
+ <library name="httpclient-4.5.10.jar"/>
<library name="httpcore-4.4.6.jar"/>
<library name="httpmime-4.5.3.jar"/>
- <library name="noggit-0.8.jar"/>
- <library name="solr-solrj-7.3.1.jar"/>
+ <library name="zookeeper-3.5.5.jar"/>
+ <library name="zookeeper-jute-3.5.5.jar"/>
<library name="stax2-api-3.1.4.jar"/>
<library name="woodstox-core-asl-4.4.1.jar"/>
- <library name="zookeeper-3.4.11.jar"/>
+ <library name="jetty-alpn-client-9.4.24.v20191120.jar"/>
+ <library name="jetty-alpn-java-client-9.4.24.v20191120.jar"/>
+ <library name="jetty-client-9.4.24.v20191120.jar"/>
+ <library name="jetty-http-9.4.24.v20191120.jar"/>
+ <library name="jetty-io-9.4.24.v20191120.jar"/>
+ <library name="jetty-util-9.4.24.v20191120.jar"/>
+ <library name="http2-client-9.4.24.v20191120.jar"/>
+ <library name="http2-common-9.4.24.v20191120.jar"/>
+ <library name="http2-hpack-9.4.24.v20191120.jar"/>
+ <library name="http2-http-client-transport-9.4.24.v20191120.jar"/>
+ <library name="jcl-over-slf4j-1.7.24.jar"/>
+ <library name="slf4j-api-1.7.24.jar"/>
+ <library name="solr-solrj-8.5.1.jar"/>
</runtime>
<requires>
diff --git a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
index e8c1308..dbc480b 100644
--- a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
+++ b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
@@ -74,13 +74,14 @@
@Override
public void open(Configuration conf, String name) {
- //Implementation not required
+ // Implementation not required
}
/**
* Initializes the internal variables from a given index writer configuration.
*
- * @param parameters Params from the index writer configuration.
+ * @param parameters
+ * Params from the index writer configuration.
*/
@Override
public void open(IndexWriterParams parameters) {
@@ -107,10 +108,10 @@
}
break;
case "cloud":
- CloudSolrClient sc = this.auth ?
- SolrUtils.getCloudSolrClient(Arrays.asList(urls), this.username,
- this.password) :
- SolrUtils.getCloudSolrClient(Arrays.asList(urls));
+ CloudSolrClient sc = this.auth
+ ? SolrUtils.getCloudSolrClient(Arrays.asList(urls), this.username,
+ this.password)
+ : SolrUtils.getCloudSolrClient(Arrays.asList(urls));
sc.setDefaultCollection(this.collection);
solrClients.add(sc);
break;
@@ -234,9 +235,8 @@
private void push() throws IOException {
if (inputDocs.size() > 0) {
try {
- LOG.info(
- "Indexing " + Integer.toString(inputDocs.size()) + "/" + Integer
- .toString(totalAdds) + " documents");
+ LOG.info("Indexing " + Integer.toString(inputDocs.size()) + "/"
+ + Integer.toString(totalAdds) + " documents");
LOG.info("Deleting " + Integer.toString(numDeletes) + " documents");
numDeletes = 0;
UpdateRequest req = new UpdateRequest();
@@ -257,10 +257,9 @@
if (deleteIds.size() > 0) {
try {
- LOG.info(
- "SolrIndexer: deleting " + Integer.toString(deleteIds.size()) + "/"
- + Integer.toString(totalDeletes) + " documents");
-
+ LOG.info("SolrIndexer: deleting " + Integer.toString(deleteIds.size())
+ + "/" + Integer.toString(totalDeletes) + " documents");
+
UpdateRequest req = new UpdateRequest();
req.deleteById(deleteIds);
req.setAction(UpdateRequest.ACTION.OPTIMIZE, false, false);
@@ -268,11 +267,11 @@
if (this.auth) {
req.setBasicAuthCredentials(this.username, this.password);
}
-
+
for (SolrClient solrClient : solrClients) {
solrClient.request(req);
}
-
+
} catch (final SolrServerException e) {
LOG.error("Error deleting: " + deleteIds);
throw makeIOException(e);
@@ -296,9 +295,11 @@
}
/**
- * Returns {@link Map} with the specific parameters the IndexWriter instance can take.
+ * Returns {@link Map} with the specific parameters the IndexWriter instance
+ * can take.
*
- * @return The values of each row. It must have the form <KEY,<DESCRIPTION,VALUE>>.
+ * @return The values of each row. It must have the form
+ * <KEY,<DESCRIPTION,VALUE>>.
*/
@Override
public Map<String, Entry<String, Object>> describe() {
@@ -326,12 +327,10 @@
properties.put(SolrConstants.USE_AUTH, new AbstractMap.SimpleEntry<>(
"Whether to enable HTTP basic authentication for communicating with Solr. Use the username and password properties to configure your credentials.",
this.auth));
- properties.put(SolrConstants.USERNAME,
- new AbstractMap.SimpleEntry<>("The username of Solr server.",
- this.username));
- properties.put(SolrConstants.PASSWORD,
- new AbstractMap.SimpleEntry<>("The password of Solr server.",
- this.password));
+ properties.put(SolrConstants.USERNAME, new AbstractMap.SimpleEntry<>(
+ "The username of Solr server.", this.username));
+ properties.put(SolrConstants.PASSWORD, new AbstractMap.SimpleEntry<>(
+ "The password of Solr server.", this.password));
return properties;
}
diff --git a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
index 196fc5d..8f97b16 100644
--- a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
+++ b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java
@@ -37,21 +37,21 @@
return sc;
}
- static CloudSolrClient getCloudSolrClient(List<String> urls, String username, String password) {
+ static CloudSolrClient getCloudSolrClient(List<String> urls, String username,
+ String password) {
// Building http client
CredentialsProvider provider = new BasicCredentialsProvider();
- UsernamePasswordCredentials credentials
- = new UsernamePasswordCredentials(username, password);
+ UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(
+ username, password);
provider.setCredentials(AuthScope.ANY, credentials);
HttpClient client = HttpClientBuilder.create()
- .setDefaultCredentialsProvider(provider)
- .build();
+ .setDefaultCredentialsProvider(provider).build();
// Building the client
CloudSolrClient sc = new CloudSolrClient.Builder(urls)
.withParallelUpdates(true).withHttpClient(client).build();
- sc.connect();
+ sc.connect();
return sc;
}
