sdk_python/amaterasu/pyspark/runtime.pyi

# Stubs for amaterasu_pyspark.runtime (Python 3)
#
# NOTE: This dynamically typed stub was automatically generated by stubgen.

from .datasets import DatasetManager
from amaterasu import BaseAmaContext
from amaterasu.datasets import BaseDatasetManager
from amaterasu.runtime import AmaContextBuilder, Environment
from pyspark import SparkContext
from pyspark.sql import DataFrame, SparkSession
from typing import Any

class SparkAmaContextBuilder(AmaContextBuilder):
    spark_conf: Any = ...
    def __init__(self) -> None: ...
    def setMaster(self, master_uri: Any) -> AmaContextBuilder: ...
    def set(self, key: Any, value: Any) -> AmaContextBuilder: ...
    def build(self) -> AmaContext: ...

class AmaContext(BaseAmaContext):
    @classmethod
    def builder(cls: Any) -> AmaContextBuilder: ...
    @property
    def dataset_manager(self) -> BaseDatasetManager: ...
    @property
    def sc(self) -> SparkContext: ...
    @property
    def spark(self) -> SparkSession: ...
    def __init__(self, env: Environment, sc: SparkContext=..., spark: SparkSession=...) -> Any: ...
    def get_dataset(self, dataset_name: str) -> DataFrame: ...
    def persist(self, dataset_name: str, dataset: DataFrame, overwrite: bool=...) -> Any: ...