Search.setIndex({"docnames": ["_api/airflow/providers/apache/spark/hooks/index", "_api/airflow/providers/apache/spark/hooks/spark_jdbc/index", "_api/airflow/providers/apache/spark/hooks/spark_jdbc_script/index", "_api/airflow/providers/apache/spark/hooks/spark_sql/index", "_api/airflow/providers/apache/spark/hooks/spark_submit/index", "_api/airflow/providers/apache/spark/index", "_api/airflow/providers/apache/spark/operators/index", "_api/airflow/providers/apache/spark/operators/spark_jdbc/index", "_api/airflow/providers/apache/spark/operators/spark_sql/index", "_api/airflow/providers/apache/spark/operators/spark_submit/index", "_api/tests/system/providers/apache/spark/example_spark_dag/index", "_api/tests/system/providers/apache/spark/index", "commits", "connections/spark", "index", "installing-providers-from-sources", "operators"], "filenames": ["_api/airflow/providers/apache/spark/hooks/index.rst", "_api/airflow/providers/apache/spark/hooks/spark_jdbc/index.rst", "_api/airflow/providers/apache/spark/hooks/spark_jdbc_script/index.rst", "_api/airflow/providers/apache/spark/hooks/spark_sql/index.rst", "_api/airflow/providers/apache/spark/hooks/spark_submit/index.rst", "_api/airflow/providers/apache/spark/index.rst", "_api/airflow/providers/apache/spark/operators/index.rst", "_api/airflow/providers/apache/spark/operators/spark_jdbc/index.rst", "_api/airflow/providers/apache/spark/operators/spark_sql/index.rst", "_api/airflow/providers/apache/spark/operators/spark_submit/index.rst", "_api/tests/system/providers/apache/spark/example_spark_dag/index.rst", "_api/tests/system/providers/apache/spark/index.rst", "commits.rst", "connections/spark.rst", "index.rst", "installing-providers-from-sources.rst", "operators.rst"], "titles": ["<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_jdbc</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_jdbc_script</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_sql</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_submit</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_jdbc</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_sql</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_submit</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">tests.system.providers.apache.spark.example_spark_dag</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">tests.system.providers.apache.spark</span></code>", "Package apache-airflow-providers-apache-spark", "Apache Spark Connection", "<code class=\"docutils literal notranslate\"><span class=\"pre\">apache-airflow-providers-apache-spark</span></code>", "Installing from sources", "Apache Spark Operators"], "terms": {"spark_jdbc": [0, 5, 6], "spark_jdbc_script": [0, 5, 12], "spark_sql": [0, 5, 6], "spark_submit": [0, 1, 5, 6, 7], "3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16], "0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16], "2": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16], "5": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], "dev0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], "i": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], "an": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], "experiment": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], "featur": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16], "sparkjdbchook": 1, "spark_app_nam": [1, 7], "jdbc": [1, 2, 7, 12, 13, 16], "spark_conn_id": [1, 4, 7, 9], "default_conn_nam": [1, 3, 4], "spark_conf": [1, 7], "none": [1, 3, 4, 7, 8, 9, 12, 15], "spark_py_fil": [1, 7], "spark_fil": [1, 7], "spark_jar": [1, 7, 16], "num_executor": [1, 3, 4, 7, 8, 9], "executor_cor": [1, 3, 4, 7, 8, 9], "executor_memori": [1, 3, 4, 7, 8, 9], "driver_memori": [1, 4, 7, 9], "verbos": [1, 3, 4, 7, 8, 9, 12], "fals": [1, 4, 7, 9], "princip": [1, 3, 4, 7, 8, 9], "keytab": [1, 3, 4, 7, 8, 9], "cmd_type": [1, 7, 16], "spark_to_jdbc": [1, 2, 7, 16], "jdbc_tabl": [1, 2, 7, 16], "jdbc_conn_id": [1, 7], "default": [1, 2, 3, 4, 7, 8, 9, 12], "jdbc_driver": [1, 7, 16], "metastore_t": [1, 2, 7, 16], "jdbc_truncat": [1, 7], "save_mod": [1, 2, 7, 16], "save_format": [1, 2, 7, 16], "batch_siz": [1, 2, 7], "fetch_siz": [1, 2, 7], "num_partit": [1, 2, 7], "partition_column": [1, 2, 7], "lower_bound": [1, 2, 7], "upper_bound": [1, 2, 7], "create_table_column_typ": [1, 2, 7], "arg": [1, 12], "kwarg": [1, 3, 4, 7, 8, 9], "sourc": [1, 2, 3, 4, 7, 8, 9, 10, 14, 16], "base": [1, 3, 4, 7, 8, 9, 16], "sparksubmithook": [1, 4, 7, 9, 12, 14], "extend": [1, 7], "specif": [1, 4, 7, 9, 12], "perform": [1, 7, 16], "data": [1, 2, 7, 12, 16], "transfer": [1, 2, 7, 12, 16], "from": [1, 2, 7, 12, 14, 16], "databas": [1, 7, 16], "paramet": [1, 2, 3, 4, 7, 8, 9, 12, 13, 16], "str": [1, 2, 3, 4, 7, 8, 9], "name": [1, 2, 3, 4, 7, 8, 9, 12, 13], "job": [1, 3, 4, 7, 8, 9], "The": [1, 3, 4, 7, 8, 9, 13, 15, 16], "connect": [1, 2, 3, 4, 7, 8, 9, 12, 14, 16], "id": [1, 4, 7, 9, 15], "configur": [1, 3, 4, 7, 8, 9, 12, 14, 16], "administr": [1, 4, 7, 9], "dict": [1, 4, 7, 9, 12], "ani": [1, 2, 3, 4, 7, 8, 9, 15], "addit": [1, 4, 7, 9], "properti": [1, 3, 4, 7, 8, 9], "python": [1, 4, 7, 9, 12, 13, 14, 15, 16], "file": [1, 3, 4, 7, 8, 9, 12, 14, 15, 16], "us": [1, 4, 7, 8, 9, 10, 12, 13, 14, 15], "zip": [1, 4, 7, 9], "egg": [1, 4, 7, 9], "py": [1, 4, 7, 9, 12, 14, 16], "upload": [1, 4, 7, 9], "contain": [1, 3, 4, 7, 8, 9, 15], "run": [1, 4, 7, 8, 9, 12, 14, 15, 16], "jar": [1, 4, 7, 9, 16], "add": [1, 7, 12, 14], "driver": [1, 2, 4, 7, 9, 12, 13, 14, 16], "executor": [1, 3, 4, 7, 8, 9], "classpath": [1, 4, 7, 9, 16], "int": [1, 3, 4, 7, 8, 9], "number": [1, 3, 4, 7, 8, 9, 12], "should": [1, 4, 7, 13, 15], "set": [1, 3, 4, 7, 8, 9, 12, 14, 16], "so": [1, 7], "manag": [1, 7, 12, 15, 16], "made": [1, 7, 15], "core": [1, 3, 4, 7, 8, 9], "per": [1, 3, 4, 7, 8, 9, 12], "memori": [1, 3, 4, 7, 8, 9], "e": [1, 3, 4, 7, 8, 9], "g": [1, 3, 4, 7, 8, 9], "1000m": [1, 3, 4, 7, 8, 9], "2g": [1, 3, 4, 7, 8, 9], "alloc": [1, 4, 7, 9], "bool": [1, 3, 4, 7, 8, 9], "whether": [1, 3, 4, 7, 8, 9, 13], "pass": [1, 3, 4, 7, 8, 9, 12, 13], "flag": [1, 3, 4, 7, 8, 9], "submit": [1, 3, 4, 7, 8, 9, 10, 12, 13, 16], "debug": [1, 4, 7, 9], "full": [1, 3, 4, 7, 8, 9], "path": [1, 3, 4, 7, 8, 9, 13, 16], "kerbero": [1, 4, 7, 9, 12], "which": [1, 7, 13, 16], "wai": [1, 7], "flow": [1, 7], "possibl": [1, 7, 16], "valu": [1, 3, 7, 8], "written": [1, 7], "metastor": [1, 7, 16], "jdbc_to_spark": [1, 2, 7, 16], "tabl": [1, 2, 7, 16], "usual": [1, 7], "onli": [1, 3, 4, 7, 8, 9, 12, 14], "truncat": [1, 2, 7], "drop": [1, 7, 15], "recreat": [1, 7], "take": [1, 7, 8, 9, 16], "effect": [1, 7], "overwrit": [1, 7, 16], "also": [1, 7, 15, 16], "schema": [1, 7], "differ": [1, 7, 15, 16], "cannot": [1, 7], "save": [1, 7], "mode": [1, 4, 7, 9, 13, 16], "append": [1, 3, 7, 16], "etc": [1, 7], "format": [1, 7], "parquet": [1, 7], "size": [1, 7], "batch": [1, 7, 12], "insert": [1, 7], "round": [1, 7], "trip": [1, 7], "1000": [1, 7], "fetch": [1, 7], "depend": [1, 4, 7, 9, 12, 16], "maximum": [1, 7], "partit": [1, 7], "can": [1, 4, 7, 9, 13, 14, 15, 16], "simultan": [1, 7], "both": [1, 7], "oper": [1, 5, 12, 13, 14], "cap": [1, 7], "open": [1, 7], "A": [1, 7], "numer": [1, 7], "column": [1, 7], "If": [1, 7, 13, 14, 15], "specifi": [1, 7, 13], "you": [1, 7, 13, 14, 15, 16], "must": [1, 7, 16], "lower": [1, 7], "bound": [1, 7], "rang": [1, 7], "upper": [1, 7], "type": [1, 7, 12, 13, 14], "instead": [1, 7], "when": [1, 4, 7, 8, 9, 13], "creat": [1, 7, 15], "inform": [1, 7, 8, 9, 12, 16], "same": [1, 7, 13], "syntax": [1, 7, 13], "char": [1, 7], "64": [1, 7], "comment": [1, 7, 12], "varchar": [1, 7], "1024": [1, 7], "valid": [1, 7, 12, 14, 15], "sql": [1, 3, 7, 8, 12, 13, 14, 16], "conn_name_attr": [1, 3, 4], "spark_default": [1, 4, 9, 13], "conn_typ": [1, 3, 4], "hook_nam": [1, 3, 4], "submit_jdbc_job": 1, "get_conn": [1, 3, 4], "return": [1, 3, 4], "spark_write_to_jdbc": 2, "spark_read_from_jdbc": 2, "set_common_opt": 2, "spark_sourc": 2, "url": [2, 13], "localhost": 2, "5432": 2, "user": [2, 4, 9, 13, 15], "root": 2, "password": [2, 12], "get": [2, 7, 8, 9, 12, 15, 16], "here": 2, "reader": 2, "writer": 2, "resourc": [2, 13], "spark_sess": 2, "sparksqlhook": [3, 8, 12, 14], "conf": [3, 4, 8, 9, 12], "conn_id": [3, 4, 8, 9], "total_executor_cor": [3, 4, 8, 9], "master": [3, 8, 16], "true": [3, 8], "yarn_queu": [3, 8], "basehook": [3, 4], "wrapper": [3, 4, 9], "around": [3, 4, 9], "binari": [3, 4, 7, 9, 13, 15], "It": [3, 4, 9, 15], "requir": [3, 4, 9, 13, 16], "queri": [3, 8, 16], "execut": [3, 4, 7, 8, 9, 13], "arbitrari": [3, 4, 8, 9], "connection_id": [3, 4, 8, 9], "string": [3, 8, 12, 13], "standalon": [3, 4, 8, 9], "meso": [3, 4, 8, 9], "total": [3, 4, 8, 9], "all": [3, 4, 8, 9, 13, 14, 16], "avail": [3, 4, 7, 8, 9, 14, 15], "worker": [3, 4, 8, 9, 13], "yarn": [3, 4, 8, 9, 12, 13], "1g": [3, 4, 8, 9], "host": [3, 8, 13], "port": [3, 8, 13], "local": [3, 8, 13, 15, 16], "launch": [3, 4, 8, 9, 16], "queue": [3, 8, 13], "spark_sql_default": [3, 8, 13], "run_queri": [3, 12], "cmd": 3, "remot": [3, 4, 9], "popen": [3, 4], "actual": 3, "command": [3, 4, 9, 12, 13, 16], "extra": [3, 4, 9, 13], "argument": [3, 4, 9, 12], "see": [3, 4, 12, 14], "subprocess": [3, 4, 7, 8, 9], "kill": [3, 4, 7, 8, 9, 12], "py_fil": [4, 9], "archiv": [4, 9], "driver_class_path": [4, 9], "java_class": [4, 9], "packag": [4, 9], "exclude_packag": [4, 9], "repositori": [4, 9, 14], "proxy_us": [4, 9], "status_poll_interv": [4, 9], "1": [4, 9, 16], "application_arg": [4, 9], "env_var": [4, 9], "spark_binari": [4, 9, 13], "util": 4, "log": [4, 12, 14], "logging_mixin": 4, "loggingmixin": 4, "kick": [4, 9], "off": [4, 9], "spark_hom": [4, 16], "suppli": [4, 9], "invalid": [4, 9], "separ": [4, 9, 12], "comma": [4, 9], "place": [4, 9], "work": [4, 9], "directori": [4, 9, 15], "each": [4, 9], "For": [4, 7, 8, 9, 12, 13, 15, 16], "exampl": [4, 9, 10, 12, 13, 14, 15, 16], "serial": [4, 9], "object": [4, 9], "unzip": 4, "possibli": 4, "tag": 4, "alia": 4, "applic": [4, 9, 10, 13, 16], "them": [4, 9], "main": [4, 9, 14, 15, 16], "java": [4, 9], "list": [4, 9, 12, 14], "maven": [4, 9], "coordin": [4, 9], "includ": [4, 9, 12], "exclud": [4, 9], "while": [4, 9], "resolv": [4, 9], "search": [4, 9], "given": [4, 9], "kubernet": [4, 12, 13, 14], "imperson": [4, 9], "second": [4, 9], "wait": [4, 9], "between": [4, 9, 13], "poll": [4, 9], "statu": [4, 9, 12, 14], "cluster": [4, 9, 13, 16], "being": [4, 9], "environ": [4, 9, 12, 13], "variabl": [4, 9, 12, 13], "support": [4, 9, 12, 13, 14, 16], "k8": [4, 9, 12], "too": [4, 9], "process": [4, 7, 8, 9, 12], "some": [4, 9, 13], "distro": [4, 9, 13], "mai": [4, 9, 12, 13], "spark2": [4, 9, 13], "static": 4, "get_ui_field_behaviour": 4, "custom": 4, "field": [4, 12, 14], "behaviour": 4, "on_kil": [4, 7, 8, 9], "hook": [5, 9, 12, 13], "sparkjdbcoper": [7, 10], "sparksubmitoper": [7, 9, 10, 12], "As": 7, "assum": [7, 13], "more": [7, 8, 9, 12, 14], "how": [7, 8, 9, 15], "look": [7, 8, 9, 16], "guid": [7, 8, 9, 12, 15], "context": [7, 8, 9], "call": [7, 8, 9, 12], "overrid": [7, 8, 9], "method": [7, 8, 9], "cleanup": [7, 8, 9, 12], "task": [7, 8, 9], "instanc": [7, 8, 9], "thread": [7, 8, 9], "multiprocess": [7, 8, 9], "within": [7, 8, 9], "need": [7, 8, 9], "clean": [7, 8, 9, 12, 14], "up": [7, 8, 9, 12, 16], "leav": [7, 8, 9], "ghost": [7, 8, 9], "behind": [7, 8, 9], "sparksqloper": [8, 10, 12], "model": [8, 9], "baseoper": [8, 9], "templat": [8, 9, 12, 14, 16], "template_field": [8, 9, 12], "sequenc": [8, 9, 12], "_sql": 8, "template_ext": 8, "hql": [8, 16], "template_fields_render": 8, "arrow": 9, "home": [9, 13], "either": 9, "_applic": 9, "_conf": 9, "_file": 9, "_py_fil": 9, "_jar": 9, "_driver_class_path": 9, "_packag": 9, "ui_color": 9, "airflow": [10, 15], "dag": [10, 12, 14], "env_id": 10, "dag_id": 10, "example_spark_oper": 10, "submit_job": [10, 16], "test_run": 10, "example_spark_dag": [11, 16], "detail": [12, 14, 15], "commit": 12, "chang": 12, "version": [12, 14, 15], "high": 12, "level": 12, "changelog": 12, "latest": 12, "2022": 12, "06": 12, "07": 12, "subject": 12, "717a7588bc": 12, "updat": [12, 14], "descript": 12, "remov": [12, 14, 15], "doubl": 12, "min": 12, "24292": 12, "aeabe994b3": 12, "prepar": 12, "doc": [12, 14], "": [12, 15], "releas": [12, 14], "24231": 12, "b4a5783a2a": 12, "chore": [12, 14], "refactor": [12, 14], "24219": [12, 14], "9dc2851671": 12, "05": 12, "fix": 12, "backward": [12, 14], "compat": [12, 14], "introduc": [12, 14], "mypi": [12, 14], "problem": [12, 14], "24230": [12, 14], "027b707d21": 12, "explanatori": 12, "note": [12, 13], "contributor": 12, "about": [12, 15], "24229": 12, "a2bfc0e62d": 12, "aip": [12, 14], "47": [12, 14], "migrat": [12, 14], "new": [12, 14], "design": [12, 14], "22439": [12, 14], "24210": [12, 14], "71e4deb1b0": 12, "16": 12, "23716": [12, 14], "03": 12, "22": 12, "d7dbfb7e26": 12, "document": [12, 14, 16], "bugfix": 12, "22383": 12, "14": 12, "16adc035b1": 12, "classifi": [12, 14], "march": 12, "22226": 12, "f5b96315fe": 12, "feb": 12, "22056": 12, "6322dad2ca": 12, "02": 12, "25": 12, "param": [12, 14], "render": [12, 14], "21788": [12, 14], "08": 12, "d94fa37830": 12, "januari": 12, "delai": 12, "21439": 12, "6c3a67d4fc": 12, "2021": [12, 15], "21257": 12, "39e395f981": 12, "04": 12, "21237": [12, 14], "cb73053211": 12, "01": 12, "27": 12, "option": [12, 13, 14], "21074": [12, 14], "602abe8394": 12, "20": 12, "line": 12, "now": 12, "sphinx": 12, "autoapi": 12, "typehint": 12, "20951": 12, "12": [12, 15, 16], "31": 12, "f77417eb0d": 12, "pypi": [12, 14], "20614": 12, "97496ba2b4": 12, "decemb": 12, "20523": 12, "83f8e178ba": 12, "even": 12, "ext": 12, "20608": 12, "d56e7b56bb": 12, "30": 12, "have": [12, 14], "friendli": 12, "20571": 12, "485ff6cc64": 12, "29": 12, "error": 12, "20422": 12, "dad2f8103b": 12, "15": 12, "20190": 12, "1a2a2498d6": 12, "20290": 12, "a50d2ac872": 12, "ensur": [12, 14], "respons": [12, 14], "befor": [12, 14], "unknown": [12, 14, 15], "19978": [12, 14], "11": [12, 15], "853576d901": 12, "novemb": 12, "19882": 12, "79b30ff59c": 12, "24": 12, "bug": 12, "sparksql": [12, 14], "go": [12, 14], "infinit": [12, 14], "loop": [12, 14], "19449": [12, 14], "ae044884d1": 12, "start_dat": 12, "18657": 12, "d9567eb106": 12, "10": [12, 14], "octob": 12, "19321": 12, "86a2a19ad2": 12, "17": 12, "f": 12, "18855": 12, "42dc0767b8": 12, "unnecessari": 12, "concaten": 12, "airflowexcept": 12, "messag": 12, "18817": 12, "0a68588479": 12, "august": 12, "17890": 12, "be75dcd39c": 12, "23": 12, "meta": 12, "76ed2a49c6": 12, "19": 12, "import": [12, 14, 15], "lazili": 12, "individu": 12, "17682": 12, "87f408b1e7": 12, "26": 12, "rc2": 12, "juli": 12, "17116": 12, "91f4d80ff0": 12, "xcomarg": 12, "16869": 12, "b916b75079": 12, "17015": 12, "866a601b76": 12, "28": 12, "pylint": 12, "our": 12, "toolchain": 12, "16682": 12, "18": 12, "bbc627a3da": 12, "16501": 12, "cbf8001d76": 12, "synchron": 12, "after": 12, "buggfix": 12, "16464": 12, "1fba5402bb": 12, "june": 12, "16405": 12, "5c86e3d509": 12, "make": [12, 14], "15794": [12, 14], "9c94b72d44": 12, "16294": 12, "37681bca00": 12, "auto": [12, 14], "appli": [12, 14], "apply_default": [12, 14], "decor": [12, 14], "15667": [12, 14], "807ad32ce5": 12, "pip": [12, 14, 15], "21": 12, "15576": 12, "5b2fe0e740": 12, "popular": 12, "15393": 12, "4b031d39e1": 12, "code": [12, 15], "8": 12, "15534": 12, "657384615f": 12, "except": [12, 14], "redund": [12, 14], "14823": [12, 14], "9015beb316": 12, "15304": 12, "68e4c4dcb0": 12, "backport": 12, "14886": 12, "589d6dec92": 12, "next": 12, "wave": 12, "14487": 12, "f9c9e9c38f": 12, "without": [12, 14], "14187": [12, 14], "88bdcfa0df": 12, "14013": 12, "ac2f72c98d": 12, "implement": 12, "tool": 12, "13767": 12, "a9ac2b040b": 12, "switch": 12, "flynt": 12, "13732": 12, "295d66f914": 12, "2020": 12, "grammar": 12, "warn": [12, 15], "13380": 12, "6cf76d7ac0": 12, "typo": 12, "upgrad": [12, 14], "13148": 12, "5090fb0c89": 12, "script": [12, 15, 16], "gener": 12, "integr": 12, "json": [12, 13, 16], "13073": 12, "09": 12, "32971a1a2d": 12, "12955": 12, "b40dffa085": 12, "renam": 12, "rema": 12, "modul": 12, "match": [12, 15], "12917": 12, "9b39f24780": 12, "dynam": 12, "form": 12, "12558": 12, "bd90136aaf": 12, "move": 12, "12681": 12, "c34ef853c8": 12, "out": [12, 13], "build": [12, 13, 15], "12444": 12, "0080354502": 12, "readm": [12, 14], "0b2": 12, "12449": 12, "ae7cb4a1e2": 12, "wrong": 12, "hash": 12, "12390": 12, "6889a333cf": 12, "improv": 12, "ref": 12, "12366": 12, "7825e8f590": 12, "13": 12, "instal": 12, "12304": 12, "85a18e13d9": 12, "point": [12, 13], "project": 12, "page": [12, 15], "cross": 12, "12212": 12, "59eb5de78c": 12, "come": 12, "0beta1": 12, "12206": 12, "b2a28d1590": 12, "dev": 12, "12082": 12, "4e8f9cc8d0": 12, "enabl": [12, 13], "black": 12, "formmatt": 12, "9550": 12, "8c42cf1b00": 12, "pyupgrad": 12, "6": 12, "11447": 12, "5a439e84eb": 12, "2a1": 12, "11855": 12, "872b1566a1": 12, "setup": 12, "11826": 12, "349b0811c3": 12, "d200": 12, "pydocstyl": 12, "check": [12, 15], "11688": 12, "16e7129719": 12, "ad": [12, 14], "11487": 12, "d305876bee": 12, "11448": 12, "0a0e1af800": 12, "broken": 12, "markdown": 12, "link": [12, 15], "toc": 12, "11249": 12, "ca4238eb4d": 12, "month": 12, "11242": 12, "5220e4c384": 12, "11238": 12, "f3e87c5030": 12, "d202": 12, "11032": 12, "fdd9b6f65b": 12, "10543": 12, "d760265452": 12, "No": 12, "whitespac": 12, "allow": 12, "surround": 12, "docstr": 12, "text": 12, "10533": 12, "d1bce91bb2": 12, "d403": 12, "capit": 12, "first": [12, 14], "word": 12, "10530": 12, "3696c34c28": 12, "10528": 12, "ee7ca128a1": 12, "refernc": 12, "10483": 12, "7c206a82a6": 12, "replac": 12, "assig": 12, "augment": 12, "assign": 12, "10468": 12, "3b3287d7ac": 12, "enforc": 12, "keyword": 12, "10170": 12, "7d24b088cd": 12, "stop": 12, "default_arg": 12, "example_dag": 12, "9985": 12, "33f0cd2657": 12, "keep": 12, "function": 12, "signatur": [12, 15], "9784": 12, "1427e4acb4": 12, "8730": 12, "4d74ac2111": 12, "increas": 12, "http": [12, 14, 15], "9729": 12, "0873070e08": 12, "mask": 12, "other": 12, "9615": 12, "13a827d80f": 12, "token": 12, "9044": 12, "067806d598": 12, "test": [12, 16], "9491": 12, "d0e7db4024": 12, "fresh": 12, "9408": 12, "12af6a0800": 12, "final": 12, "23rc1": 12, "9404": 12, "c7e5bce57f": 12, "candid": 12, "9370": 12, "40bf8f28f9": 12, "detect": 12, "automat": [12, 14], "lack": 12, "refer": 12, "9290": 12, "f6bd817a3a": 12, "9320": 12, "0b0e4f7a4c": 12, "rc3": 12, "relas": 12, "9026": 12, "00642a46d0": 12, "remain": 12, "wrongli": 12, "8994": 12, "375d1ca229": 12, "8898": 12, "12c5e5d8a": 12, "8891": 12, "f3521fb0e3": 12, "regener": 12, "8886": 12, "92585ca4cb": 12, "autom": 12, "8807": 12, "7506c73f17": 12, "8787": 12, "487b5cc50c": 12, "8305": 12, "87969a350d": 12, "6515": 12, "info": 12, "8170": 12, "be1451b0e1": 12, "7026": 12, "7749": 12, "4bde99f132": 12, "7802": 12, "7e6372a681": 12, "super": 12, "7820": 12, "2327aa5a26": 12, "7025": 12, "handl": 12, "its": [12, 16], "properli": 12, "7677": 12, "024b4bf962": 12, "7024": 12, "7676": 12, "b59042b5ab": 12, "6949": 12, "respect": 12, "explicit": 12, "namespac": [12, 13], "7575": 12, "97a429f9d0": 12, "6714": 12, "magic": 12, "utf": 12, "7338": 12, "0481b9a957": 12, "6539": 12, "class": [12, 14], "7142": 12, "case": 13, "dictionari": 13, "follow": [13, 15], "standard": 13, "ar": [13, 14, 15], "deploi": [13, 16], "your": [13, 14], "node": 13, "extern": 13, "client": 13, "bin": [13, 15], "otherwis": [13, 14], "present": 13, "divid": 13, "multipl": 13, "via": [13, 14, 15], "quota": 13, "uri": 13, "compon": 13, "encod": 13, "mongo": 13, "export": 13, "airflow_conn_spark_default": 13, "mysparkclust": 13, "com": [13, 14, 15], "80": 13, "kube": 13, "api": 14, "top": [14, 15], "exist": 14, "below": [14, 15], "minimum": 14, "pyspark": 14, "explain": 14, "polici": 14, "github": 14, "blob": 14, "md": 14, "mistakenli": 14, "install_requir": 14, "22382": 14, "trove": 14, "framework": 14, "optimis": 14, "due": 14, "want": [14, 15], "least": 14, "manual": 14, "db": 14, "complet": 14, "initi": 14, "describ": 15, "download": 15, "apach": 15, "provid": [15, 16], "spark": 15, "offici": 15, "most": 15, "choos": 15, "select": [15, 16], "down": 15, "left": 15, "sdist": 15, "whl": 15, "origin": 15, "checksum": 15, "softwar": 15, "foundat": 15, "asc": 15, "sha512": 15, "those": 15, "abov": 15, "pgp": 15, "kei": 15, "essenti": 15, "sha": 15, "gpg": 15, "pleas": 15, "well": 15, "relev": 15, "distribut": 15, "recommend": 15, "mirror": 15, "pgpk": 15, "ka": 15, "To": [15, 16], "pgpv": 15, "tar": 15, "gz": 15, "sat": 15, "sep": 15, "49": 15, "54": 15, "bst": 15, "rsa": 15, "cde15c6e4d3a8ec4ecf4ba4b6674e08ad7de406f": 15, "issuer": 15, "kaxilnaik": 15, "org": [15, 16], "good": 15, "kaxil": 15, "naik": 15, "aka": 15, "gmail": 15, "certifi": 15, "trust": 15, "There": 15, "indic": 15, "belong": 15, "owner": 15, "primari": 15, "fingerprint": 15, "cde1": 15, "5c6e": 15, "4d3a": 15, "8ec4": 15, "ecf4": 15, "ba4b": 15, "6674": 15, "e08a": 15, "d7de": 15, "406f": 15, "correct": 15, "do": 15, "worri": 15, "certif": 15, "self": 15, "sign": 15, "why": 15, "By": 15, "server": [15, 16], "previou": 15, "step": 15, "know": 15, "alreadi": 15, "sum": 15, "shasum": 15, "512": 15, "diff": 15, "one": 15, "bash": 15, "package_vers": 15, "package_nam": 15, "provider_download_dir": 15, "mktemp": 15, "d": 15, "dep": 15, "dest": 15, "curl": 15, "apache_airflow_providers_apache_spark": 15, "py3": 15, "l": 15, "o": 15, "echo": 15, "la": 15, "onc": 15, "instruct": 15, "chapter": 15, "temporari": 15, "folder": 15, "definit": 16, "write": 16, "saveast": 16, "system": 16, "jdbc_to_spark_job": 16, "foo": 16, "postgresql": 16, "42": 16, "bar": 16, "task_id": 16, "spark_to_jdbc_job": 16, "further": 16, "dataframewrit": 16, "hive": 16, "servic": 16, "spark_sql_job": 16, "count": 16, "cnt": 16, "temp_tabl": 16, "cli": 16, "care": 16, "src": 16, "pi": 16}, "objects": {"airflow.providers.apache": [[5, 0, 0, "-", "spark"]], "airflow.providers.apache.spark": [[0, 0, 0, "-", "hooks"], [6, 0, 0, "-", "operators"]], "airflow.providers.apache.spark.hooks": [[1, 0, 0, "-", "spark_jdbc"], [2, 0, 0, "-", "spark_jdbc_script"], [3, 0, 0, "-", "spark_sql"], [4, 0, 0, "-", "spark_submit"]], "airflow.providers.apache.spark.hooks.spark_jdbc": [[1, 1, 1, "", "SparkJDBCHook"]], "airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook": [[1, 2, 1, "", "conn_name_attr"], [1, 2, 1, "", "conn_type"], [1, 2, 1, "", "default_conn_name"], [1, 3, 1, "", "get_conn"], [1, 2, 1, "", "hook_name"], [1, 3, 1, "", "submit_jdbc_job"]], "airflow.providers.apache.spark.hooks.spark_jdbc_script": [[2, 4, 1, "", "SPARK_READ_FROM_JDBC"], [2, 4, 1, "", "SPARK_WRITE_TO_JDBC"], [2, 5, 1, "", "set_common_options"], [2, 5, 1, "", "spark_read_from_jdbc"], [2, 5, 1, "", "spark_write_to_jdbc"]], "airflow.providers.apache.spark.hooks.spark_sql": [[3, 1, 1, "", "SparkSqlHook"]], "airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook": [[3, 2, 1, "", "conn_name_attr"], [3, 2, 1, "", "conn_type"], [3, 2, 1, "", "default_conn_name"], [3, 3, 1, "", "get_conn"], [3, 2, 1, "", "hook_name"], [3, 3, 1, "", "kill"], [3, 3, 1, "", "run_query"]], "airflow.providers.apache.spark.hooks.spark_submit": [[4, 1, 1, "", "SparkSubmitHook"]], "airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook": [[4, 2, 1, "", "conn_name_attr"], [4, 2, 1, "", "conn_type"], [4, 2, 1, "", "default_conn_name"], [4, 3, 1, "", "get_conn"], [4, 3, 1, "", "get_ui_field_behaviour"], [4, 2, 1, "", "hook_name"], [4, 3, 1, "", "on_kill"], [4, 3, 1, "", "submit"]], "airflow.providers.apache.spark.operators": [[7, 0, 0, "-", "spark_jdbc"], [8, 0, 0, "-", "spark_sql"], [9, 0, 0, "-", "spark_submit"]], "airflow.providers.apache.spark.operators.spark_jdbc": [[7, 1, 1, "", "SparkJDBCOperator"]], "airflow.providers.apache.spark.operators.spark_jdbc.SparkJDBCOperator": [[7, 3, 1, "", "execute"], [7, 3, 1, "", "on_kill"]], "airflow.providers.apache.spark.operators.spark_sql": [[8, 1, 1, "", "SparkSqlOperator"]], "airflow.providers.apache.spark.operators.spark_sql.SparkSqlOperator": [[8, 3, 1, "", "execute"], [8, 3, 1, "", "on_kill"], [8, 2, 1, "", "template_ext"], [8, 2, 1, "", "template_fields"], [8, 2, 1, "", "template_fields_renderers"]], "airflow.providers.apache.spark.operators.spark_submit": [[9, 1, 1, "", "SparkSubmitOperator"]], "airflow.providers.apache.spark.operators.spark_submit.SparkSubmitOperator": [[9, 3, 1, "", "execute"], [9, 3, 1, "", "on_kill"], [9, 2, 1, "", "template_fields"], [9, 2, 1, "", "ui_color"]], "tests.system.providers.apache": [[11, 0, 0, "-", "spark"]], "tests.system.providers.apache.spark": [[10, 0, 0, "-", "example_spark_dag"]], "tests.system.providers.apache.spark.example_spark_dag": [[10, 4, 1, "", "DAG_ID"], [10, 4, 1, "", "ENV_ID"], [10, 4, 1, "", "submit_job"], [10, 4, 1, "", "test_run"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:attribute", "3": "py:method", "4": "py:data", "5": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "method", "Python method"], "4": ["py", "data", "Python data"], "5": ["py", "function", "Python function"]}, "titleterms": {"airflow": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 14], "provid": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14], "apach": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16], "spark": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16], "hook": [0, 1, 2, 3, 4], "submodul": [0, 6, 11], "spark_jdbc": [1, 7], "modul": [1, 2, 3, 4, 7, 8, 9, 10], "content": [1, 2, 3, 4, 7, 8, 9, 10, 14], "class": [1, 3, 4, 7, 8, 9], "spark_jdbc_script": 2, "function": 2, "attribut": 2, "spark_sql": [3, 8], "spark_submit": [4, 9], "subpackag": 5, "oper": [6, 7, 8, 9, 16], "test": [10, 11], "system": [10, 11], "example_spark_dag": 10, "packag": [12, 14, 15], "3": [12, 14], "0": [12, 14], "2": [12, 14], "1": [12, 14], "connect": 13, "default": 13, "id": 13, "configur": 13, "guid": 14, "refer": [14, 16], "resourc": 14, "commit": 14, "instal": [14, 15], "requir": 14, "changelog": 14, "break": 14, "chang": 14, "bug": 14, "fix": 14, "misc": 14, "featur": 14, "from": 15, "sourc": 15, "releas": 15, "integr": 15, "verifi": 15, "pypi": 15, "prerequisit": 16, "sparkjdbcoper": 16, "us": 16, "sparksqloper": 16, "sparksubmitoper": 16}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx": 56}})