remove ZooKeeper 3.4 support + pass tests with Java 15 (#11073)
With this change, Druid will only support ZooKeeper 3.5.x and later.
In order to support Java 15 we need to switch to ZK 3.5.x client libraries and drop support for ZK 3.4.x
(see #10780 for the detailed reasons)
* remove ZooKeeper 3.4.x compatibility
* exclude additional ZK 3.5.x netty dependencies to ensure we use our version
* keep ZooKeeper version used for integration tests in sync with client library version
* remove the need to specify ZK version at runtime for docker
* add support to run integration tests with JDK 15
* build and run unit tests with Java 15 in travis
diff --git a/.travis.yml b/.travis.yml
index a25cc34..fe9528e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,6 @@
env:
global:
- - ZK_VERSION=3.5 # default ZK version for integration tests
- DOCKER_IP=127.0.0.1 # for integration tests
- MVN="mvn -B"
- > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -153,6 +152,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *package
+ name: "(openjdk15) packaging check"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_processing_module
name: "(openjdk8) processing module test"
stage: Tests - phase 1
@@ -219,6 +223,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_processing_module
+ name: "(openjdk15) processing module test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_processing_module_sqlcompat
<<: *test_processing_module
name: "(openjdk8) processing module test (SQL Compatibility)"
@@ -231,6 +240,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_processing_module_sqlcompat
+ name: "(openjdk15) processing module test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_indexing_module
<<: *test_processing_module
name: "(openjdk8) indexing modules test"
@@ -242,6 +256,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_indexing_module
+ name: "(openjdk15) indexing modules test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_indexing_module_sqlcompat
<<: *test_indexing_module
name: "(openjdk8) indexing modules test (SQL Compatibility)"
@@ -253,6 +272,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_indexing_module_sqlcompat
+ name: "(openjdk15) indexing modules test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_server_module
<<: *test_processing_module
name: "(openjdk8) server module test"
@@ -264,6 +288,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_server_module
+ name: "(openjdk15) server module test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_server_module_sqlcompat
<<: *test_server_module
name: "(openjdk8) server module test (SQL Compatibility)"
@@ -274,6 +303,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_server_module_sqlcompat
+ name: "(openjdk15) server module test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_other_modules
<<: *test_processing_module
name: "(openjdk8) other modules test"
@@ -285,6 +319,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_other_modules
+ name: "(openjdk15) other modules test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_other_modules_sqlcompat
<<: *test_other_modules
name: "(openjdk8) other modules test (SQL Compatibility)"
@@ -295,6 +334,11 @@
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_other_modules_sqlcompat
+ name: "(openjdk15) other modules test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- name: "web console"
install: skip
stage: Tests - phase 1
@@ -342,7 +386,7 @@
- docker
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: &run_integration_test
- - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP}
+ - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP}
after_failure: &integration_test_diags
- for v in ~/shared/logs/*.log ; do
echo $v logtail ======================== ; tail -100 $v ;
@@ -554,16 +598,6 @@
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
- # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
- - <<: *integration_tests
- name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
- jdk: openjdk8
- env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
- - <<: *integration_kafka_format_tests
- name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
- env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
# END - Integration tests for Compile with Java 8 and Run with Java 8
# START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -637,12 +671,6 @@
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
- - <<: *integration_tests
- name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
- jdk: openjdk8
- env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
# END - Integration tests for Compile with Java 8 and Run with Java 11
- &integration_batch_index_k8s
diff --git a/docs/dependencies/zookeeper.md b/docs/dependencies/zookeeper.md
index 0855a37..ec6814e 100644
--- a/docs/dependencies/zookeeper.md
+++ b/docs/dependencies/zookeeper.md
@@ -23,7 +23,17 @@
-->
-Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. The operations that happen over ZK are
+Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state.
+
+## Minimum ZooKeeper versions
+
+Apache Druid supports ZooKeeper versions 3.5.x and above.
+
+> Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed
+
+## ZooKeeper Operations
+
+The operations that happen over ZK are
1. [Coordinator](../design/coordinator.md) leader election
2. Segment "publishing" protocol from [Historical](../design/historical.md)
diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml
index a5962e2..a5f0b43 100644
--- a/extensions-core/kafka-extraction-namespace/pom.xml
+++ b/extensions-core/kafka-extraction-namespace/pom.xml
@@ -132,8 +132,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <!-- Kafka brokers require ZooKeeper 3.5.x clients for testing -->
- <version>3.5.8</version>
+ <version>${zookeeper.version}</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml
index 545b0d8..17848a3 100644
--- a/extensions-core/kafka-indexing-service/pom.xml
+++ b/extensions-core/kafka-indexing-service/pom.xml
@@ -144,8 +144,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <!-- Kafka brokers require ZooKeeper 3.5.x clients for testing -->
- <version>3.5.8</version>
+ <version>${zookeeper.version}</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml
index ef58bd6..5c1d55a 100644
--- a/indexing-service/pom.xml
+++ b/indexing-service/pom.xml
@@ -138,6 +138,10 @@
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper-jute</artifactId>
+ </dependency>
+ <dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile
index b3af96b..8076240 100644
--- a/integration-tests/docker/Dockerfile
+++ b/integration-tests/docker/Dockerfile
@@ -21,6 +21,9 @@
COPY base-setup.sh /root/base-setup.sh
ARG KAFKA_VERSION
+# ZooKeeper version to install in the base image
+# This is passed in by maven at build time to align with the client version we depend on in the pom file
+ARG ZK_VERSION
ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org
RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh
diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh
index b2aa33e..2a25866 100755
--- a/integration-tests/docker/base-setup.sh
+++ b/integration-tests/docker/base-setup.sh
@@ -40,15 +40,9 @@
rm /tmp/$ZK_TAR.tar.gz
}
-ZK_VERSION=3.4.14
-ZK_TAR=zookeeper-$ZK_VERSION
-install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
-
-ZK_VERSION=3.5.9
ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper
# Kafka
# KAFKA_VERSION is defined by docker build arguments
diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml
index 9267b1a..11d7962 100644
--- a/integration-tests/docker/docker-compose.base.yml
+++ b/integration-tests/docker/docker-compose.base.yml
@@ -53,8 +53,6 @@
- ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
env_file:
- ./environment-configs/common
- environment:
- - ZK_VERSION
druid-metadata-storage:
image: druid/cluster
diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf
index 6039e26..1ee5247 100644
--- a/integration-tests/docker/service-supervisords/zookeeper.conf
+++ b/integration-tests/docker/service-supervisords/zookeeper.conf
@@ -1,5 +1,5 @@
[program:zookeeper]
-command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground
+command=/usr/local/zookeeper/bin/zkServer.sh start-foreground
user=daemon
priority=0
stdout_logfile=/shared/logs/zookeeper.log
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index dc58ee5..bd5803d 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -462,7 +462,6 @@
<docker.build.skip>false</docker.build.skip>
<docker.build.hadoop>false</docker.build.hadoop>
<it.indexer>middleManager</it.indexer>
- <zk.version>3.5</zk.version>
<override.config.path />
<resource.file.dir.path />
@@ -495,7 +494,7 @@
<MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
<CONFLUENT_VERSION>5.5.1</CONFLUENT_VERSION>
<KAFKA_VERSION>${apache.kafka.version}</KAFKA_VERSION>
- <ZK_VERSION>${zk.version}</ZK_VERSION>
+ <ZK_VERSION>${zookeeper.version}</ZK_VERSION>
</environmentVariables>
<executable>${project.basedir}/build_run_cluster.sh</executable>
</configuration>
diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh
index 0819e3a..8e93d4f 100755
--- a/integration-tests/script/docker_build_containers.sh
+++ b/integration-tests/script/docker_build_containers.sh
@@ -22,17 +22,21 @@
if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
then
echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version"
- docker build -t druid/cluster --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker
else
echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
8)
echo "Build druid-cluster with Java 8"
- docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
;;
11)
echo "Build druid-cluster with Java 11"
- docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+ ;;
+ 15)
+ echo "Build druid-cluster with Java 15"
+ docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
;;
*)
echo "Invalid JVM Runtime given. Stopping"
diff --git a/licenses.yaml b/licenses.yaml
index 01f7bfa..2eb60d9 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -1902,9 +1902,10 @@
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 3.4.14
+version: 3.5.9
libraries:
- org.apache.zookeeper: zookeeper
+ - org.apache.zookeeper: zookeeper-jute
notices:
- zookeeper: |
Apache ZooKeeper
diff --git a/pom.xml b/pom.xml
index 251b57a..af66764 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,6 @@
<mysql.version>5.1.48</mysql.version>
<netty3.version>3.10.6.Final</netty3.version>
<resilience4j.version>1.3.1</resilience4j.version>
- <!-- Spark updated in https://github.com/apache/spark/pull/19884 -->
<netty4.version>4.1.63.Final</netty4.version>
<node.version>v10.24.0</node.version>
<npm.version>6.14.11</npm.version>
@@ -113,11 +112,8 @@
<aws.sdk.version>1.11.884</aws.sdk.version>
<caffeine.version>2.8.0</caffeine.version>
<jacoco.version>0.8.6</jacoco.version>
- <!-- Curator requires 3.4.x ZooKeeper clients to maintain compatibility with 3.4.x ZooKeeper servers,
- If we upgrade to 3.5.x clients, curator requires 3.5.x servers, which would break backwards compatibility
- see http://curator.apache.org/zk-compatibility.html -->
<!-- When upgrading ZK, edit docs and integration tests as well (integration-tests/docker-base/setup.sh) -->
- <zookeeper.version>3.4.14</zookeeper.version>
+ <zookeeper.version>3.5.9</zookeeper.version>
<checkerframework.version>2.5.7</checkerframework.version>
<com.google.apis.client.version>1.26.0</com.google.apis.client.version>
<com.google.apis.compute.version>v1-rev20190607-${com.google.apis.client.version}</com.google.apis.compute.version>
@@ -316,12 +312,21 @@
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper-jute</artifactId>
+ <version>${zookeeper.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${apache.curator.version}</version>
@@ -349,11 +354,6 @@
</exclusions>
</dependency>
<dependency>
- <!--
- ~ TODO: This dependency transitively pulls in org.codehaus.jackson:jackson-mapper.asl.1.9.13 which has
- ~ security vulnerability CVE-2017-7525. https://github.com/apache/druid/pull/8177 tracks
- ~ upgrading to a newer curator version.
- -->
<groupId>org.apache.curator</groupId>
<artifactId>curator-x-discovery</artifactId>
<version>${apache.curator.version}</version>
diff --git a/server/pom.xml b/server/pom.xml
index 2c5ad5b..cf0030d 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -75,6 +75,10 @@
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper-jute</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<!-- We use log4j2 and log4j-1.2-api as a bridge -->
diff --git a/website/.spelling b/website/.spelling
index 1219f56..5aed913 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -459,6 +459,8 @@
BasicDataSource
- ../docs/dependencies/zookeeper.md
LeadershipLatch
+3.5.x
+3.4.x
- ../docs/design/auth.md
AllowAll
AuthenticationResult
@@ -1881,4 +1883,4 @@
PiB
protobuf
Golang
-multiValueHandling
\ No newline at end of file
+multiValueHandling
