sdk_python/amaterasu/pyspark/datasets.pyi

# Stubs for amaterasu_pyspark.datasets (Python 3)
#
# NOTE: This dynamically typed stub was automatically generated by stubgen.

from abc import ABC
from amaterasu.datasets import BaseDatasetLoader, BaseDatasetManager
from pyspark.sql import DataFrame, SparkSession
from typing import Any, Dict, Type

class BaseSparkDatasetLoader(BaseDatasetLoader, ABC):
    spark: Any = ...
    def __init__(self, dataset_conf: Dict, spark: SparkSession) -> Any: ...

class HiveDatasetLoader(BaseSparkDatasetLoader):
    def load_dataset(self) -> DataFrame: ...
    def persist_dataset(self, dataset: DataFrame, overwrite: bool=...) -> Any: ...

class FileDatasetLoader(BaseSparkDatasetLoader):
    def load_dataset(self) -> DataFrame: ...
    def persist_dataset(self, dataset: DataFrame, overwrite: bool) -> Any: ...

class DatasetManager(BaseDatasetManager):
    def get_datastore(self, datastore_cls: Type[BaseSparkDatasetLoader], dataset_conf: Dict) -> Any: ...
    spark: Any = ...
    def __init__(self, dataset_conf: Any, spark: Any) -> None: ...