[LIVY-975][SERVER] Fix compatibility issue related to a change on the Spark side (#403)
## What changes were proposed in this pull request?
Recent change in spark required us to fix compatibility issue related to a change on the Spark side.
## How was this patch tested?
Verified manually by creating interactive / batch sessions via REST API call in a local Yarn cluster. Also, we have updated the unit tests.
diff --git a/conf/livy.conf.template b/conf/livy.conf.template
index edec95a..aedc632 100644
--- a/conf/livy.conf.template
+++ b/conf/livy.conf.template
@@ -188,3 +188,7 @@
# livy.server.auth.<custom>.class = <class of custom auth filter>
# livy.server.auth.<custom>.param.<foo1> = <bar1>
# livy.server.auth.<custom>.param.<foo2> = <bar2>
+
+# Enable to allow custom classpath by proxy user in cluster mode
+# The below configuration parameter is disabled by default.
+# livy.server.session.allow-custom-classpath = true
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala
index 5808ba4..51179e1 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -264,6 +264,8 @@
// Max creating session in livyServer
val SESSION_MAX_CREATION = Entry("livy.server.session.max-creation", 100)
+ val SESSION_ALLOW_CUSTOM_CLASSPATH = Entry("livy.server.session.allow-custom-classpath", false)
+
val SPARK_MASTER = "spark.master"
val SPARK_DEPLOY_MODE = "spark.submit.deployMode"
val SPARK_JARS = "spark.jars"
diff --git a/server/src/main/scala/org/apache/livy/sessions/Session.scala b/server/src/main/scala/org/apache/livy/sessions/Session.scala
index 197e22f..6b01c4f 100644
--- a/server/src/main/scala/org/apache/livy/sessions/Session.scala
+++ b/server/src/main/scala/org/apache/livy/sessions/Session.scala
@@ -34,6 +34,9 @@
object Session {
trait RecoveryMetadata { val id: Int }
+ val BLACKLIST_CUSTOM_CLASSPATH: Set[String] = Set("spark.submit.deployMode",
+ "spark.submit.proxyUser.allowCustomClasspathInClusterMode")
+
lazy val configBlackList: Set[String] = {
val url = getClass.getResource("/spark-blacklist.conf")
if (url != null) Utils.loadProperties(url).keySet else Set()
@@ -58,7 +61,11 @@
return Map()
}
- val errors = conf.keySet.filter(configBlackList.contains)
+ val errors = if (livyConf.getBoolean(LivyConf.SESSION_ALLOW_CUSTOM_CLASSPATH)) {
+ conf.keySet.filter(configBlackList.contains)
+ } else {
+ conf.keySet.filter((configBlackList ++ BLACKLIST_CUSTOM_CLASSPATH).contains)
+ }
if (errors.nonEmpty) {
throw new IllegalArgumentException(
"Blacklisted configuration values in session config: " + errors.mkString(", "))
diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
index 05b41bb..aa5d456 100644
--- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
+++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
@@ -65,6 +65,18 @@
intercept[IllegalArgumentException] {
Session.prepareConf(Map("spark.do_not_set" -> "1"), Nil, Nil, Nil, Nil, conf)
}
+
+ // Test for "spark.submit.deployMode".
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map("spark.submit.deployMode" -> "standalone"), Nil, Nil, Nil, Nil, conf)
+ }
+
+ // Test for "spark.submit.proxyUser.allowCustomClasspathInClusterMode".
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map("spark.submit.proxyUser.allowCustomClasspathInClusterMode"
+ -> "false"), Nil, Nil, Nil, Nil, conf)
+ }
+
conf.sparkFileLists.foreach { key =>
intercept[IllegalArgumentException] {
Session.prepareConf(Map(key -> "file:/not_allowed"), Nil, Nil, Nil, Nil, conf)
@@ -95,4 +107,20 @@
userLists.foreach { key => assert(result.get(key) === expected) }
}
+
+ test("conf validation to allow custom classpath") {
+ val conf = new LivyConf(false)
+ conf.set(LivyConf.SESSION_ALLOW_CUSTOM_CLASSPATH, true)
+
+ // Test for "spark.submit.deployMode".
+ assert(Session.prepareConf(Map("spark.submit.deployMode" -> "standalone"), Nil, Nil, Nil,
+ Nil, conf) === Map("spark.submit.deployMode" -> "standalone", "spark.master" -> "local"))
+
+ // Test for "spark.submit.proxyUser.allowCustomClasspathInClusterMode".
+ assert(Session.prepareConf(Map("spark.submit.proxyUser.allowCustomClasspathInClusterMode" ->
+ "false"), Nil, Nil, Nil, Nil, conf) === Map("spark.master" -> "local",
+ "spark.submit.proxyUser.allowCustomClasspathInClusterMode" -> "false"))
+
+ }
+
}
