when working through spark-submit, unless something changed, can use the default sparkcontext and session
diff --git a/frameworks/spark/pyspark_runtime/amaterasu_pyspark/runtime.py b/frameworks/spark/pyspark_runtime/amaterasu_pyspark/runtime.py
index 72587b7..8878103 100644
--- a/frameworks/spark/pyspark_runtime/amaterasu_pyspark/runtime.py
+++ b/frameworks/spark/pyspark_runtime/amaterasu_pyspark/runtime.py
@@ -29,7 +29,10 @@
def __init__(self):
super().__init__()
- self.spark_conf = SparkConf()
+ try:
+ self.spark_conf = sc.getConf() # When running through spark-submit
+ except UnboundLocalError:
+ self.spark_conf = SparkConf()
def setMaster(self, master_uri) -> "AmaContextBuilder":
self.spark_conf.setMaster(master_uri)
@@ -41,7 +44,7 @@
def build(self) -> "AmaContext":
spark = SparkSession.builder.config(conf=self.spark_conf).getOrCreate()
- sc = spark.sparkContext
+ sc: SparkContext = spark.sparkContext
return AmaContext(self.env, sc, spark)
