commit | 667bb98319cce4bce4b8a4a769f72fb246f12d91 | [log] [tgz] |
---|---|---|
author | Yaniv Rodenski <yaniv@shinto.io> | Fri Apr 26 14:59:00 2019 +1000 |
committer | Yaniv Rodenski <yaniv@shinto.io> | Fri Apr 26 14:59:00 2019 +1000 |
tree | a60d5d658025ff259a1aab8bfaced7b7941ddd4e | |
parent | 8943ba1930afc2e50c02c4db54a8f8efcc880762 [diff] |
fixing virtual env config
diff --git a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt index 0f91d3a..59f3896 100644 --- a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt +++ b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt
@@ -27,6 +27,7 @@ private val requirementsFileName: String = "ama-requirements.txt" private val mandatoryPYPIPackages: Array<String> = arrayOf("requests") protected val virtualPythonPath = "amaterasu_env/bin/python" + protected val virtualPythonBin = "amaterasu_env/bin" override val runnerResources: Array<String> get() = arrayOf("amaterasu-sdk-${conf.version()}.zip")
diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala index 60e0700..f4ddb6e 100644 --- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala
@@ -11,9 +11,9 @@ log.info(s"===> Cluster manager: ${conf.mode}") command + //s" $$SPARK_HOME/conf/spark-env.sh" + - s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + - s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + - s" $$SPARK_HOME/bin/spark-submit --master yarn-cluster --files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}" + // s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + + //s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + d + s" $$SPARK_HOME/bin/spark-submit --master yarn-cluster --conf spark.pyspark.virtualenv.enabled=true --conf spark.pyspark.virtualenv.type=native--conf spark.pyspark.virtualenv.bin.path=$getVirtualPythonBin --conf spark.pyspark.python=$getVirtualPythonPath --files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}" } override def getRunnerResources: Array[String] = {