[LIVY-518][BUILD] Support Spark 2.4
## What changes were proposed in this pull request?
This PR proposes to support Spark 2.4 in Livy.
## How was this patch tested?
Unit tests and manual tests.
Author: hyukjinkwon <gurwls223@apache.org>
Author: Hyukjin Kwon <gurwls223@apache.org>
Closes #121 from HyukjinKwon/spark24-support.
diff --git a/.travis.yml b/.travis.yml
index a72a875..d05e6dc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,6 +29,10 @@
env: MVN_FLAG='-Pspark-2.3 -Pthriftserver -DskipITs'
- name: "Spark 2.3 ITs"
env: MVN_FLAG='-Pspark-2.3 -Pthriftserver -DskipTests'
+ - name: "Spark 2.4 Unit Tests"
+ env: MVN_FLAG='-Pspark-2.4 -Pthriftserver -DskipITs'
+ - name: "Spark 2.4 ITs"
+ env: MVN_FLAG='-Pspark-2.4 -Pthriftserver -DskipTests'
jdk:
- oraclejdk8
diff --git a/README.md b/README.md
index 735afb3..a1f0854 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@
To run Livy, you will also need a Spark installation. You can get Spark releases at
https://spark.apache.org/downloads.html.
-Livy requires Spark 2.2 or 2.3. You can switch to a different version of Spark by setting the
+Livy requires Spark 2.2+. You can switch to a different version of Spark by setting the
``SPARK_HOME`` environment variable in the Livy server process, without needing to rebuild Livy.
diff --git a/pom.xml b/pom.xml
index 16b1bc0..e24827c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1042,6 +1042,38 @@
</profile>
<profile>
+ <id>spark-2.4</id>
+ <activation>
+ <property>
+ <name>spark-2.4</name>
+ </property>
+ </activation>
+ <properties>
+ <spark.scala-2.11.version>2.4.0</spark.scala-2.11.version>
+ <spark.version>${spark.scala-2.11.version}</spark.version>
+ <netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version>
+ <java.version>1.8</java.version>
+ <py4j.version>0.10.7</py4j.version>
+ <json4s.version>3.2.11</json4s.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>spark-2.4-it</id>
+ <activation>
+ <property>
+ <name>spark-2.4-it</name>
+ </property>
+ </activation>
+ <properties>
+ <spark.bin.download.url>
+ http://mirrors.advancedhosters.com/apache/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz
+ </spark.bin.download.url>
+ <spark.bin.name>spark-2.4.0-bin-hadoop2.7</spark.bin.name>
+ </properties>
+ </profile>
+
+ <profile>
<id>skip-parent-modules</id>
<activation>
<file>
diff --git a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java
index aa4d319..bf0592b 100644
--- a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java
+++ b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java
@@ -57,6 +57,10 @@
private static final long TIMEOUT = 100;
private Properties createConf(boolean local) {
+ return createConf(local, true);
+ }
+
+ private Properties createConf(boolean local, boolean hiveSupport) {
Properties conf = new Properties();
if (local) {
conf.put(CLIENT_IN_PROCESS.key(), "true");
@@ -72,8 +76,8 @@
conf.put(CLIENT_SHUTDOWN_TIMEOUT.key(), "30s");
conf.put(LIVY_JARS.key(), "");
- conf.put("spark.repl.enableHiveContext", "true");
- conf.put("spark.sql.catalogImplementation", "hive");
+ conf.put("spark.repl.enableHiveContext", hiveSupport);
+ conf.put("spark.sql.catalogImplementation", hiveSupport ? "hive" : "in-memory");
conf.put(RETAINED_SHARE_VARIABLES.key(), "2");
return conf;
}
@@ -271,7 +275,7 @@
@Test
public void testSparkSQLJob() throws Exception {
- runTest(true, new TestFunction() {
+ runTest(true, false, new TestFunction() {
@Override
void call(LivyClient client) throws Exception {
JobHandle<List<String>> handle = client.submit(new SQLGetTweets(false));
@@ -518,7 +522,11 @@
}
private void runTest(boolean local, TestFunction test) throws Exception {
- Properties conf = createConf(local);
+ runTest(local, true, test);
+ }
+
+ private void runTest(boolean local, boolean hiveSupport, TestFunction test) throws Exception {
+ Properties conf = createConf(local, hiveSupport);
LivyClient client = null;
try {
test.config(conf);
diff --git a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
index 6097d32..dc98b0d 100644
--- a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
+++ b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala
@@ -30,6 +30,8 @@
// For each Spark version we supported, we need to add this mapping relation in case Scala
// version cannot be detected from "spark-submit --version".
private val _defaultSparkScalaVersion = SortedMap(
+ // Spark 2.4 + Scala 2.11
+ (2, 4) -> "2.11",
// Spark 2.3 + Scala 2.11
(2, 3) -> "2.11",
// Spark 2.2 + Scala 2.11
@@ -38,7 +40,7 @@
// Supported Spark version
private val MIN_VERSION = (2, 2)
- private val MAX_VERSION = (2, 4)
+ private val MAX_VERSION = (3, 0)
private val sparkVersionRegex = """version (.*)""".r.unanchored
private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored
