| commit | fbbf22291caa703ff32e4986869969735e5d8d65 | [log] [tgz] |
|---|---|---|
| author | Yaniv Rodenski <yaniv@shinto.io> | Sun Apr 21 12:39:46 2019 +1000 |
| committer | Yaniv Rodenski <yaniv@shinto.io> | Sun Apr 21 12:39:46 2019 +1000 |
| tree | 7352312e4274e95ab54f16210a30d876f1319b87 | |
| parent | f9398c9ce3cf7fb3719c530269f502e9363d4e4f [diff] | |
| parent | 72668dc605839e92aa5873c9c0c3bb94937e09e3 [diff] |
Merge branch 'nadav-har-tzvi-AMATERASU-45-Rebase' of github.com:nadav-har-tzvi/incubator-amaterasu into nadav-har-tzvi-AMATERASU-45-Rebase
diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala index 0b7ed21..2c718d9 100644 --- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
@@ -10,7 +10,7 @@ val command = super.getCommand(jobId: String, actionData: ActionData, env: String, executorId: String, callbackAddress: String) log.info(s"===> Cluster manager: ${conf.mode}") command + - //s" $$SPARK_HOME/conf/spark-env.sh &&" + + //s" $$SPARK_HOME/conf/spark-env.sh" + s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + s" && $$SPARK_HOME/bin/spark-submit ${actionData.getSrc}" }