| commit | 88309c352808c58302d6c0f93d621f82320115a4 | [log] [tgz] |
|---|---|---|
| author | Nadav Har Tzvi <nadavha@apache.org> | Sat Mar 16 14:56:28 2019 +0200 |
| committer | Nadav Har Tzvi <nadavha@apache.org> | Sat Mar 16 14:56:28 2019 +0200 |
| tree | bdfb3bdc4364d83e529eefad95256b384f15d8b5 | |
| parent | 79e06ea63b68e8ba5adf2c3ef06cfd8210c1ad9e [diff] |
set pyspark_python
diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala index 5945d8f..1f9ce38 100644 --- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
@@ -12,7 +12,7 @@ val pyhtonBinPath = Seq("python3", "-c", "import sys; print(sys.executable)").!!.trim() conf.mode match { case "mesos" => - command + s"env PYSPARK_PYTHON=$pyhtonBinPath && env AMA_NODE=${sys.env("AMA_NODE")} && env MESOS_NATIVE_JAVA_LIBRARY=${conf.mesos.libPath}" + + command + s" && env PYSPARK_PYTHON=$pyhtonBinPath && env AMA_NODE=${sys.env("AMA_NODE")} && env MESOS_NATIVE_JAVA_LIBRARY=${conf.mesos.libPath}" + s" && python3 ${actionData.getSrc}" case "yarn" => command + s" && /bin/bash spark/bin/load-spark-env.sh" +