commit | 8043d0eaafe6cf13ffe2bac0b2733a4e74fd67e9 | [log] [tgz] |
---|---|---|
author | Nadav Har Tzvi <nadavha@apache.org> | Sat Apr 27 12:30:58 2019 +0300 |
committer | Nadav Har Tzvi <nadavha@apache.org> | Sat Apr 27 12:30:58 2019 +0300 |
tree | dcb42e6b2c693ad331c1ae62b7e5216f51f005d1 | |
parent | 59c4c1ca96379725cc08916dad28ac4d0f1879b7 [diff] |
virtualenv isn't supported by spark, provided alternatives
diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala index 7cac602..61fa555 100644 --- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
@@ -14,10 +14,8 @@ // s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + //s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + d s" && $$SPARK_HOME/bin/spark-submit --master yarn-client " + - s"--conf spark.pyspark.virtualenv.enabled=true " + - s"--conf spark.pyspark.virtualenv.type=native " + - s"--conf spark.pyspark.virtualenv.bin.path=$getVirtualPythonBin " + - s"--conf spark.pyspark.python=$getVirtualPythonPath " + + s"--conf spark.pyspark.python=${conf.pythonPath}" + + s"--conf spark.pyspark.driver.python=$getVirtualPythonPath " + s"--files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}" }