[TOREE-480] Add support for spark-context-initialization-mode none
Deprecate nosparkcontext in favor of spark-context-initialization-mode none.
Note that old usage of nosparkcontext is automatically converted to use
spark-context-initialization-mode none.
diff --git a/etc/pip_install/toree/toreeapp.py b/etc/pip_install/toree/toreeapp.py
index 84dad4c..c4dc11d 100644
--- a/etc/pip_install/toree/toreeapp.py
+++ b/etc/pip_install/toree/toreeapp.py
@@ -49,7 +49,7 @@
jupyter toree install --spark_home=/spark/home/dir
jupyter toree install --spark_opts='--master=local[4]'
jupyter toree install --kernel_name=toree_special
- jupyter toree install --toree_opts='--nosparkcontext'
+ jupyter toree install --toree_opts='--spark-context-initialization-mode none'
jupyter toree install --interpreters=PySpark,SQL
jupyter toree install --python=python
'''
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index ae7f220..39376ae 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -93,14 +93,15 @@
.withRequiredArg().ofType(classOf[String])
private val _nosparkcontext =
- parser.accepts("nosparkcontext", "kernel should not create a spark context")
+ parser.accepts("nosparkcontext", "Deprecated (see spark-context-initialization-mode). \n Kernel should not create a spark context")
private val _spark_context_initialization_mode = parser.accepts(
"spark-context-initialization-mode",
"Identify how the Spark context initialization occurs. " +
+ "NONE will not initialize a Spark Context," +
"EAGER initialization will happen during runtime initialization, " +
- "LAZY initialization will happen when the context is used for the first time ."
- ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(lazy)|(eager)")).defaultsTo("lazy")
+ "LAZY initialization will happen when the context is used for the first time."
+ ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(none)|(lazy)|(eager)")).defaultsTo("lazy")
private val _spark_context_initialization_timeout = parser.accepts(
"spark-context-initialization-timeout",
@@ -171,9 +172,10 @@
"alternate_sigint" -> get(_alternate_sigint),
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
- "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
- "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode))
- get(_spark_context_initialization_mode) else Some("lazy")),
+ // deprecated in favor of spark-context-initialization-mode none
+ // "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
+ "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode)) get(_spark_context_initialization_mode)
+ else ( if (has(_nosparkcontext)) Some("none") else Some("lazy"))),
"spark_context_initialization_timeout" -> get(_spark_context_initialization_timeout),
"interpreter_plugins" -> interpreterPlugins,
"default_repositories" -> getAll(_default_repositories).map(_.asJava)
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index 029f46b..2ec7020 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -387,7 +387,20 @@
config.getString(key) should be("eager")
}
- it("when an invalid value is specified, an exception must be thrown") {
+ it("when the option nosparkcontext is specified, it should properly set spark-context-initialization-mode to none") {
+ val options = new CommandLineOptions(List(
+ "--stdin-port", "99999",
+ "--shell-port", "88888",
+ "--iopub-port", "77777",
+ "--control-port", "55555",
+ "--heartbeat-port", "44444",
+ "--nosparkcontext", "foo"
+ ))
+ val config: Config = options.toConfig
+ config.getString(key) should be("none")
+ }
+
+ it("when an invalid value is specified, an exception must be thrown") {
intercept [OptionException] {
val options = new CommandLineOptions(List(
"--stdin-port", "99999",
