| commit | d39a4c460a5aa141e193b6e40b737aaa7948794c | [log] [tgz] |
|---|---|---|
| author | Yaniv Rodenski <yaniv@shinto.io> | Sun May 05 14:21:57 2019 +1000 |
| committer | Yaniv Rodenski <yaniv@shinto.io> | Sun May 05 14:21:57 2019 +1000 |
| tree | 08bb36e750f44f1c96625ebf9f510c7992cebbb6 | |
| parent | a28eea9b69dd3085b5a87aaafdc597806ec4dcb0 [diff] |
fixing ama-requierments location in YARN
diff --git a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt index daea452..01ef770 100644 --- a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt +++ b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt
@@ -22,6 +22,8 @@ import org.apache.amaterasu.leader.common.utilities.DataLoader import org.apache.amaterasu.sdk.frameworks.RunnerSetupProvider import java.io.File +import java.nio.file.Files +import java.nio.file.Paths abstract class PythonRunnerProviderBase(val env: String, val conf: ClusterConfig) : RunnerSetupProvider() { @@ -47,7 +49,11 @@ override fun getActionDependencies(jobId: String, actionData: ActionData): Array<String> { val reqFile = File("dist/$requirementsFileName") + val dist = Paths.get("dist/") + if (reqFile.exists()) reqFile.delete() + if (Files.notExists(dist)) Files.createDirectories(dist) + val dependencies = runnerResources + mandatoryPYPIPackages dependencies.forEach { resource ->
diff --git a/leader-yarn/build.gradle b/leader-yarn/build.gradle index 2c617b9..65c33dc 100644 --- a/leader-yarn/build.gradle +++ b/leader-yarn/build.gradle
@@ -34,7 +34,6 @@ plugins { id "com.github.johnrengelman.shadow" version "2.0.4" - id 'scala' } apply plugin: 'kotlin'