[Java] Upgrade dependencies
Upgrades the Java dependencies and Gradle versions.
Minor version upgrades:
- avro 1.8.2 -> 1.10.0
- clojure 1.9.0 -> 1.10.1
- commonsIo 2.6 -> 2.8.0
- hadoop 3.2.1 -> 3.3.0
- jmh 1.23 -> 1.25.2
- log4j 2.11.2 -> 2.13.3
- mockito 3.3.3 -> 3.5.10
- protobuf 3.11.4 -> 3.13.0
- ranger 2.0.0 -> 2.1.0
- scalatest 3.0.8 -> 3.2.2
Maintenance version upgrades:
- jetty 9.4.28.v20200408 -> 9.4.31.v20200723
- micrometer 1.5.1 -> 1.5.4
- netty 4.1.49.Final -> 4.1.52.Final
- parquet 1.11.0 -> 1.11.1
- spark 2.4.5 -> 2.4.7
Gradle upgrades:
- gradle 6.2.2 -> 6.6.1
- checkstyle 8.29 -> 8.36.1
- clojureToolsCli 0.4.1 -> 1.0.194
- spotBugs 4.0.0-beta1 -> 4.1.1
- gradle-versions-plugin 0.28.0 -> 0.33.0
- shadow 5.2.0 -> 6.0.0
- protobuf-gradle-plugin 0.8.12 -> 0.8.13
- nebula-clojure-plugin 9.4.0 -> 9.4.1
- spotbugs-gradle-plugin 4.0.2 -> 4.5.0
- gradle-errorprone-plugin 1.1.1 -> 1.2.1
- gradle-animalsniffer-plugin 1.5.0 -> 1.5.1
- gradle-scalafmt 1.7.2 -> 1.14.0
- jmh-gradle-plugin 0.5.0 -> 0.5.1
Change-Id: Icc61547b6844893085de8c0f7edee50d84a248cf
Reviewed-on: http://gerrit.cloudera.org:8080/16455
Reviewed-by: Andrew Wong <awong@cloudera.com>
Tested-by: Kudu Jenkins
diff --git a/java/buildSrc/build.gradle b/java/buildSrc/build.gradle
index 5fb85a9..81a1ac1 100644
--- a/java/buildSrc/build.gradle
+++ b/java/buildSrc/build.gradle
@@ -28,19 +28,19 @@
// Manage plugin dependencies since the plugin block can't be used in included build scripts yet.
// For more details see: https://docs.gradle.org/current/userguide/plugins.html#plugins_dsl_limitations
dependencies {
- compile "com.github.ben-manes:gradle-versions-plugin:0.28.0"
- compile "com.github.jengelman.gradle.plugins:shadow:5.2.0"
+ compile "com.github.ben-manes:gradle-versions-plugin:0.33.0"
+ compile "com.github.jengelman.gradle.plugins:shadow:6.0.0"
compile "gradle.plugin.com.google.gradle:osdetector-gradle-plugin:1.6.2"
- compile "com.google.protobuf:protobuf-gradle-plugin:0.8.12"
- compile "com.netflix.nebula:nebula-clojure-plugin:9.4.0"
- compile "gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.0.2"
+ compile "com.google.protobuf:protobuf-gradle-plugin:0.8.13"
+ compile "com.netflix.nebula:nebula-clojure-plugin:9.4.1"
+ compile "gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:4.5.0"
compile "io.spring.gradle:propdeps-plugin:0.0.9.RELEASE"
- compile "net.ltgt.gradle:gradle-errorprone-plugin:1.1.1"
- compile "ru.vyarus:gradle-animalsniffer-plugin:1.5.0"
+ compile "net.ltgt.gradle:gradle-errorprone-plugin:1.2.1"
+ compile "ru.vyarus:gradle-animalsniffer-plugin:1.5.1"
compile "com.google.code.gson:gson:2.8.6"
- compile "cz.alenkacz:gradle-scalafmt:1.7.2"
+ compile "cz.alenkacz:gradle-scalafmt:1.14.0"
compile "com.google.guava:guava:29.0-jre"
- compile "me.champeau.gradle:jmh-gradle-plugin:0.5.0"
+ compile "me.champeau.gradle:jmh-gradle-plugin:0.5.1"
}
// Compiler configuration
diff --git a/java/config/spotbugs/excludeFilter.xml b/java/config/spotbugs/excludeFilter.xml
index bcc7ad6..0a7c55f 100644
--- a/java/config/spotbugs/excludeFilter.xml
+++ b/java/config/spotbugs/excludeFilter.xml
@@ -256,6 +256,12 @@
<Bug pattern="SF_SWITCH_NO_DEFAULT" />
</Or>
</Match>
+ <Match>
+ <!-- Null pointer dereference isn't possible given we check for existence. -->
+ <Class name="org.apache.kudu.client.RpcTraceFrame"/>
+ <Method name="getHumanReadableSummaryStringForTraces" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
<!-- kudu-client-tools exclusions -->
<Match>
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index e66d7f5..37fb170 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -26,40 +26,40 @@
versions += [
async : "1.4.1",
- avro : "1.8.2",
- checkstyle : "8.29",
- clojure : "1.9.0",
- clojureToolsCli: "0.4.1",
- commonsIo : "2.6",
+ avro : "1.10.0",
+ checkstyle : "8.36.1",
+ clojure : "1.10.1",
+ clojureToolsCli: "1.0.194",
+ commonsIo : "2.8.0",
errorProne : "2.3.3",
errorProneJavac: "9+181-r4173-1",
- gradle : "6.2.2",
+ gradle : "6.6.1",
guava : "29.0-jre",
- hadoop : "3.2.1",
+ hadoop : "3.3.0",
hamcrest : "2.2",
hdrhistogram : "2.1.12",
hive : "3.1.2",
httpClient : "4.5.12",
jepsen : "0.1.5",
- jetty : "9.4.28.v20200408",
- jmh : "1.23",
+ jetty : "9.4.31.v20200723",
+ jmh : "1.25.2",
jsr305 : "3.0.2",
junit : "4.13",
- log4j : "2.11.2",
- micrometer : "1.5.1",
- mockito : "3.3.3",
+ log4j : "2.13.3",
+ micrometer : "1.5.4",
+ mockito : "3.5.10",
murmur : "1.0.0",
- netty : "4.1.49.Final",
+ netty : "4.1.52.Final",
osdetector : "1.6.2",
- parquet : "1.11.0",
- protobuf : "3.11.4",
- ranger : "2.0.0",
+ parquet : "1.11.1",
+ protobuf : "3.13.0",
+ ranger : "2.1.0",
scala : "2.11.12",
- scalatest : "3.0.8",
+ scalatest : "3.2.2",
scopt : "3.7.1",
slf4j : "1.7.30",
- spark : "2.4.5",
- spotBugs : "4.0.0-beta1",
+ spark : "2.4.7",
+ spotBugs : "4.1.1",
yetus : "0.12.0"
]
@@ -101,7 +101,9 @@
jmhGenerator : "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh",
jsr305 : "com.google.code.findbugs:jsr305:$versions.jsr305",
junit : "junit:junit:$versions.junit",
- log4j : "org.apache.logging.log4j:log4j-1.2-api:$versions.log4j",
+ log4jApi : "org.apache.logging.log4j:log4j-api:$versions.log4j",
+ log4jCompat : "org.apache.logging.log4j:log4j-1.2-api:$versions.log4j",
+ log4jCore : "org.apache.logging.log4j:log4j-core:$versions.log4j",
log4jSlf4jImpl : "org.apache.logging.log4j:log4j-slf4j-impl:$versions.log4j",
micrometerCore : "io.micrometer:micrometer-core:$versions.micrometer",
mockitoCore : "org.mockito:mockito-core:$versions.mockito",
diff --git a/java/gradle/quality.gradle b/java/gradle/quality.gradle
index e113c33..69d3759 100644
--- a/java/gradle/quality.gradle
+++ b/java/gradle/quality.gradle
@@ -117,7 +117,7 @@
dependencyUpdates {
gradleReleaseChannel = "current"
revision = "release"
- resolutionStrategy = {
+ resolutionStrategy {
componentSelection { rules ->
rules.all { ComponentSelection selection ->
boolean rejected = ["snap", "alpha", "beta", "rc", "cr", "m"].any { qualifier ->
diff --git a/java/kudu-backup-common/build.gradle b/java/kudu-backup-common/build.gradle
index f5d76dd..56d8ad8 100644
--- a/java/kudu-backup-common/build.gradle
+++ b/java/kudu-backup-common/build.gradle
@@ -34,7 +34,8 @@
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
testCompile libs.scalatest
}
diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle
index 7ee49f5..0a30de9 100644
--- a/java/kudu-backup/build.gradle
+++ b/java/kudu-backup/build.gradle
@@ -44,7 +44,8 @@
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile project(path: ":kudu-spark", configuration: "test")
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
testCompile libs.scalatest
}
diff --git a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index fec45e8..9e8e6f2 100644
--- a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++ b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -27,7 +27,6 @@
import org.apache.kudu.ColumnSchema
import org.apache.kudu.Schema
import org.apache.kudu.Type
-import org.apache.kudu.client
import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
import org.apache.kudu.spark.kudu.SparkListenerUtil.withJobDescriptionCollector
import org.apache.kudu.spark.kudu.SparkListenerUtil.withJobTaskCounter
@@ -40,7 +39,11 @@
import org.apache.kudu.util.SchemaGenerator.SchemaGeneratorBuilder
import org.apache.spark.scheduler.SparkListener
import org.apache.spark.scheduler.SparkListenerJobEnd
-import org.junit.Assert._
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotEquals
+import org.junit.Assert.assertTrue
+import org.junit.Assert.fail
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -301,10 +304,10 @@
assertTrue(runRestore(createRestoreOptions(Seq(table1Name, table2Name))))
val rdd1 = kuduContext.kuduRDD(ss.sparkContext, s"$table1Name-restore", List("key"))
- assertResult(numRows)(rdd1.count())
+ assertEquals(numRows, rdd1.count())
val rdd2 = kuduContext.kuduRDD(ss.sparkContext, s"$table2Name-restore", List("key"))
- assertResult(numRows)(rdd2.count())
+ assertEquals(numRows, rdd2.count())
}
@Test
@@ -320,9 +323,9 @@
assertTrue(runBackup(createBackupOptions(tableNames).copy(numParallelBackups = 3)))
assertTrue(runRestore(createRestoreOptions(tableNames).copy(numParallelRestores = 4)))
- tableNames.map { tableName =>
+ tableNames.foreach { tableName =>
val rdd = kuduContext.kuduRDD(ss.sparkContext, s"$tableName-restore", List("key"))
- assertResult(numRows)(rdd.count())
+ assertEquals(numRows, rdd.count())
}
}
diff --git a/java/kudu-client-tools/build.gradle b/java/kudu-client-tools/build.gradle
index 4a6c59b..333e269 100644
--- a/java/kudu-client-tools/build.gradle
+++ b/java/kudu-client-tools/build.gradle
@@ -32,6 +32,7 @@
testCompile project(path: ":kudu-mapreduce", configuration: "test")
testCompile libs.commonsIo
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
}
diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle
index 759dee0..8e655ff 100644
--- a/java/kudu-client/build.gradle
+++ b/java/kudu-client/build.gradle
@@ -39,7 +39,8 @@
testCompile project(":kudu-test-utils")
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
testCompile libs.mockitoCore
}
diff --git a/java/kudu-hive/build.gradle b/java/kudu-hive/build.gradle
index dcf6d14..e18eac6 100644
--- a/java/kudu-hive/build.gradle
+++ b/java/kudu-hive/build.gradle
@@ -29,7 +29,8 @@
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.hiveTestUtils
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
}
diff --git a/java/kudu-mapreduce/build.gradle b/java/kudu-mapreduce/build.gradle
index f53f156..b6d9b8d 100644
--- a/java/kudu-mapreduce/build.gradle
+++ b/java/kudu-mapreduce/build.gradle
@@ -33,6 +33,7 @@
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.commonsIo
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
}
\ No newline at end of file
diff --git a/java/kudu-spark-tools/build.gradle b/java/kudu-spark-tools/build.gradle
index 57090a0..b585627 100644
--- a/java/kudu-spark-tools/build.gradle
+++ b/java/kudu-spark-tools/build.gradle
@@ -38,7 +38,8 @@
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile project(path: ":kudu-spark", configuration: "test")
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
testCompile libs.scalatest
}
diff --git a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
index b6beb1b..8629a14 100644
--- a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
@@ -28,7 +28,7 @@
import org.apache.kudu.client.CreateTableOptions
import org.apache.kudu.client.KuduTable
import org.apache.kudu.spark.kudu._
-import org.junit.Assert._
+import org.junit.Assert.assertEquals
import org.junit.Before
import org.junit.Test
@@ -145,6 +145,6 @@
// TODO(KUDU-2454): Verify every value to ensure all values round trip.
val rdd1 = kuduContext.kuduRDD(ss.sparkContext, fromTable, List("key"))
val rdd2 = kuduContext.kuduRDD(ss.sparkContext, toTable, List("key"))
- assertResult(rdd1.count())(rdd2.count())
+ assertEquals(rdd1.count(), rdd2.count())
}
}
diff --git a/java/kudu-spark/build.gradle b/java/kudu-spark/build.gradle
index c866a7c..abd7a4b 100644
--- a/java/kudu-spark/build.gradle
+++ b/java/kudu-spark/build.gradle
@@ -29,11 +29,11 @@
provided libs.sparkSql
provided libs.slf4jApi
-
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.junit
testCompile libs.scalatest
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
}
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index b504116..e82af8b 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -31,13 +31,13 @@
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.types.StructType
import org.junit.Assert._
-import org.scalatest.Matchers
import org.apache.kudu.client.CreateTableOptions
import org.apache.kudu.test.RandomUtils
import org.apache.kudu.spark.kudu.SparkListenerUtil.withJobTaskCounter
import org.apache.kudu.test.KuduTestHarness.MasterServerConfig
import org.junit.Before
import org.junit.Test
+import org.scalatest.matchers.should.Matchers
import scala.util.Random
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
index 61eb3c0..d701016 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
@@ -30,7 +30,7 @@
import org.apache.kudu.util.TimestampUtil
import org.apache.spark.sql.functions.decode
import org.junit.Test
-import org.scalatest.Matchers
+import org.scalatest.matchers.should.Matchers
class KuduContextTest extends KuduTestSuite with Matchers {
val rowCount = 10
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
index 08b1a04..c17edfc 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
@@ -27,6 +27,7 @@
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.junit.Assert.assertEquals
+import org.junit.Assert.fail
import org.junit.Test
class KuduRDDTest extends KuduTestSuite {
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduTestSuite.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduTestSuite.scala
index 804efb7..88d440d 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduTestSuite.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduTestSuite.scala
@@ -42,11 +42,10 @@
import org.junit.After
import org.junit.Before
import org.junit.Rule
-import org.scalatestplus.junit.JUnitSuite
import scala.annotation.meta.getter
-trait KuduTestSuite extends JUnitSuite {
+trait KuduTestSuite {
var ss: SparkSession = _
var kuduClient: KuduClient = _
var table: KuduTable = _
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/SparkSQLTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/SparkSQLTest.scala
index 92ad869..654f167 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/SparkSQLTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/SparkSQLTest.scala
@@ -22,7 +22,6 @@
import scala.util.control.NonFatal
import org.apache.spark.sql.SQLContext
import org.junit.Assert._
-import org.scalatest.Matchers
import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
import org.apache.kudu.client.CreateTableOptions
import org.apache.kudu.Schema
@@ -33,6 +32,7 @@
import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
import org.junit.Before
import org.junit.Test
+import org.scalatest.matchers.should.Matchers
class SparkSQLTest extends KuduTestSuite with Matchers {
val rowCount = 10
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/StreamingTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/StreamingTest.scala
index e6b5c55..1e2eaab 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/StreamingTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/StreamingTest.scala
@@ -20,6 +20,7 @@
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.streaming.OutputMode
+import org.junit.Assert.assertEquals
import org.junit.Before
import org.junit.Test
@@ -61,7 +62,7 @@
}
.collect()
.toSet
- assert(actual === expectedData.toSet)
+ assertEquals(actual, expectedData.toSet)
}
input.addData(1, 2, 3)
query.processAllAvailable()
diff --git a/java/kudu-subprocess/build.gradle b/java/kudu-subprocess/build.gradle
index 4463cc2..d262659 100644
--- a/java/kudu-subprocess/build.gradle
+++ b/java/kudu-subprocess/build.gradle
@@ -35,18 +35,18 @@
exclude group: "org.apache.kafka"
}
compile libs.slf4jApi
- compile libs.log4j
+ compile libs.log4jApi
+ compile libs.log4jCore
compile libs.log4jSlf4jImpl
- // Workaround for RANGER-2749. Remove once resolved.
- compile "commons-lang:commons-lang:2.6"
-
optional libs.jsr305
optional libs.yetusAnnotations
testCompile project(path: ":kudu-test-utils", configuration: "shadow")
testCompile libs.junit
- testCompile libs.log4j
+ testCompile libs.log4jApi
+ testCompile libs.log4jCompat
+ testCompile libs.log4jCore
testCompile libs.log4jSlf4jImpl
testCompile libs.mockitoCore
}
diff --git a/java/kudu-test-utils/build.gradle b/java/kudu-test-utils/build.gradle
index 72db57c..a4a23e4 100644
--- a/java/kudu-test-utils/build.gradle
+++ b/java/kudu-test-utils/build.gradle
@@ -40,7 +40,8 @@
}
// Needed for CapturingLogAppender. Optional otherwise.
- optional libs.log4j
+ optional libs.log4jApi
+ optional libs.log4jCore
optional libs.log4jSlf4jImpl
optional libs.jsr305