Search.setIndex({"docnames": ["_api/airflow/providers/databricks/hooks/databricks/index", "_api/airflow/providers/databricks/hooks/databricks_base/index", "_api/airflow/providers/databricks/hooks/databricks_sql/index", "_api/airflow/providers/databricks/hooks/index", "_api/airflow/providers/databricks/index", "_api/airflow/providers/databricks/operators/databricks/index", "_api/airflow/providers/databricks/operators/databricks_repos/index", "_api/airflow/providers/databricks/operators/databricks_sql/index", "_api/airflow/providers/databricks/operators/index", "_api/airflow/providers/databricks/triggers/databricks/index", "_api/airflow/providers/databricks/triggers/index", "_api/airflow/providers/databricks/utils/databricks/index", "_api/airflow/providers/databricks/utils/index", "_api/tests/system/providers/databricks/example_databricks/index", "_api/tests/system/providers/databricks/example_databricks_repos/index", "_api/tests/system/providers/databricks/example_databricks_sql/index", "_api/tests/system/providers/databricks/index", "commits", "connections/databricks", "index", "installing-providers-from-sources", "operators/copy_into", "operators/index", "operators/repos_create", "operators/repos_delete", "operators/repos_update", "operators/run_now", "operators/sql", "operators/submit_run"], "filenames": ["_api/airflow/providers/databricks/hooks/databricks/index.rst", "_api/airflow/providers/databricks/hooks/databricks_base/index.rst", "_api/airflow/providers/databricks/hooks/databricks_sql/index.rst", "_api/airflow/providers/databricks/hooks/index.rst", "_api/airflow/providers/databricks/index.rst", "_api/airflow/providers/databricks/operators/databricks/index.rst", "_api/airflow/providers/databricks/operators/databricks_repos/index.rst", "_api/airflow/providers/databricks/operators/databricks_sql/index.rst", "_api/airflow/providers/databricks/operators/index.rst", "_api/airflow/providers/databricks/triggers/databricks/index.rst", "_api/airflow/providers/databricks/triggers/index.rst", "_api/airflow/providers/databricks/utils/databricks/index.rst", "_api/airflow/providers/databricks/utils/index.rst", "_api/tests/system/providers/databricks/example_databricks/index.rst", "_api/tests/system/providers/databricks/example_databricks_repos/index.rst", "_api/tests/system/providers/databricks/example_databricks_sql/index.rst", "_api/tests/system/providers/databricks/index.rst", "commits.rst", "connections/databricks.rst", "index.rst", "installing-providers-from-sources.rst", "operators/copy_into.rst", "operators/index.rst", "operators/repos_create.rst", "operators/repos_delete.rst", "operators/repos_update.rst", "operators/run_now.rst", "operators/sql.rst", "operators/submit_run.rst"], "titles": ["<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.hooks.databricks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.hooks.databricks_base</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.hooks.databricks_sql</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.hooks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.operators.databricks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.operators.databricks_repos</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.operators.databricks_sql</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.operators</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.triggers.databricks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.triggers</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.utils.databricks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.databricks.utils</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">tests.system.providers.databricks.example_databricks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">tests.system.providers.databricks.example_databricks_repos</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">tests.system.providers.databricks.example_databricks_sql</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">tests.system.providers.databricks</span></code>", "Package apache-airflow-providers-databricks", "Databricks Connection", "<code class=\"docutils literal notranslate\"><span class=\"pre\">apache-airflow-providers-databricks</span></code>", "Installing from sources", "DatabricksCopyIntoOperator", "Databricks Operators", "DatabricksReposCreateOperator", "DatabricksReposDeleteOperator", "DatabricksReposUpdateOperator", "DatabricksRunNowOperator", "DatabricksSqlOperator", "DatabricksSubmitRunOperator"], "terms": {"thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "enabl": [0, 1, 17, 18], "submit": [0, 1, 5, 17, 28], "run": [0, 1, 2, 5, 9, 13, 15, 17, 18, 19, 20, 26, 28], "job": [0, 1, 5, 13, 15, 17, 19, 25, 26, 28], "platform": [0, 1], "intern": [0, 1, 2, 7, 18], "oper": [0, 1, 4, 17, 18, 19], "talk": [0, 1], "api": [0, 1, 2, 5, 6, 9, 13, 15, 17, 18, 19, 23, 24, 25, 26, 28], "2": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28], "1": [0, 1, 5, 6, 26, 27, 28], "now": [0, 5, 7, 17, 19, 26], "endpoint": [0, 1, 2, 5, 6, 7, 18, 21, 23, 24, 25, 26, 27, 28], "http": [0, 1, 2, 5, 6, 7, 13, 15, 17, 18, 19, 20, 21, 23, 27], "doc": [0, 5, 13, 15, 17, 19], "com": [0, 1, 5, 13, 14, 15, 19, 20, 23, 24, 25, 28], "dev": [0, 5, 17], "tool": [0, 5, 17], "latest": [0, 5, 13, 15, 17, 19, 25], "html": [0, 5, 13, 15], "jobsrunnow": [0, 5], "_": 0, "restart_cluster_endpoint": 0, "post": [0, 6], "0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28], "cluster": [0, 2, 5, 7, 13, 15, 18, 21, 27, 28], "restart": 0, "sourc": [0, 1, 2, 5, 6, 7, 9, 11, 13, 14, 15, 19, 21, 23, 24, 25, 27, 28], "start_cluster_endpoint": 0, "start": 0, "terminate_cluster_endpoint": 0, "delet": [0, 6], "run_now_endpoint": 0, "submit_run_endpoint": 0, "get_run_endpoint": 0, "get": [0, 2, 5, 17, 20, 28], "cancel_run_endpoint": 0, "cancel": 0, "output_runs_job_endpoint": 0, "output": [0, 7, 17, 19], "install_libs_endpoint": 0, "librari": [0, 1, 5, 28], "instal": [0, 17], "uninstall_libs_endpoint": 0, "uninstal": [0, 17], "list_jobs_endpoint": 0, "list": [0, 2, 5, 7, 17, 19, 23, 27], "workspace_get_status_endpoint": 0, "workspac": [0, 13, 15, 18], "statu": [0, 17, 19], "run_life_cycle_st": 0, "pend": 0, "termin": [0, 5], "skip": [0, 17, 19], "internal_error": 0, "spark_versions_endpoint": 0, "spark": [0, 2, 5, 7, 13, 15, 17, 18, 28], "version": [0, 2, 5, 7, 17, 19, 20, 26, 28], "runstat": [0, 13, 15], "life_cycle_st": 0, "result_st": [0, 13, 15], "state_messag": 0, "arg": [0, 17], "kwarg": [0, 2, 5, 6, 7], "util": [0, 4, 6, 26, 28], "state": [0, 13, 15, 17, 19], "concept": [0, 17], "properti": 0, "is_termin": 0, "true": [0, 2, 5, 7, 11, 21], "current": [0, 5, 28], "i": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "is_success": 0, "result": [0, 2, 5, 7, 9], "success": [0, 13, 15], "__eq__": 0, "other": [0, 5, 17, 19, 21, 26, 27], "return": [0, 1, 2, 5, 6, 9], "self": [0, 5, 11, 20], "valu": [0, 5, 6, 7, 11, 17, 18, 19, 27, 28], "__repr__": 0, "repr": 0, "to_json": 0, "classmethod": 0, "from_json": 0, "data": [0, 7, 17], "databrickshook": [0, 17, 19], "databricks_conn_id": [0, 1, 2, 5, 6, 7, 9, 17, 19, 21, 23, 24, 25, 27], "basedatabrickshook": [0, 1, 2], "default_conn_nam": [0, 1, 2, 7], "timeout_second": [0, 1, 5, 28], "180": [0, 1], "retry_limit": [0, 1], "3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28], "retry_delai": [0, 1], "retry_arg": [0, 1], "none": [0, 1, 2, 5, 6, 7, 9, 20], "caller": [0, 1, 2], "base": [0, 1, 2, 5, 6, 7, 9], "databricks_bas": [0, 2, 3, 4], "interact": [0, 1, 2], "paramet": [0, 1, 2, 5, 6, 7, 9, 18, 21, 23, 24, 25, 26, 27], "str": [0, 1, 2, 5, 6, 7, 9, 17, 23, 24, 25], "refer": [0, 1, 2, 5, 6, 7, 9, 13, 15], "connect": [0, 1, 2, 5, 6, 7, 9, 17, 19, 23, 24, 25], "int": [0, 1, 5, 6, 7, 9, 27], "The": [0, 1, 2, 5, 9, 11, 13, 15, 18, 19, 20, 21, 26, 27, 28], "amount": [0, 1, 5, 6, 23, 24, 25], "time": [0, 1, 5, 6, 17, 19, 23, 24, 25, 28], "second": [0, 1, 5, 6, 9, 13, 15, 23, 24, 25, 28], "request": [0, 1, 2, 5, 7, 17, 18, 19], "wait": [0, 1, 5, 6, 23, 24, 25], "befor": [0, 1, 2, 25], "out": [0, 1, 6, 17, 19], "number": [0, 1, 5, 6, 7, 17, 23, 24, 25], "retri": [0, 1, 5, 6, 17, 19, 23, 24, 25], "case": [0, 1, 5, 6, 9, 28], "servic": [0, 1, 17, 18], "outag": [0, 1], "float": [0, 1, 5, 6], "between": [0, 1, 5, 6, 23, 24, 25], "might": [0, 1, 5, 6, 19], "point": [0, 1, 5, 6, 17], "option": [0, 1, 2, 5, 6, 7, 17, 18, 19, 21, 23, 26, 27], "dict": [0, 1, 2, 5, 7, 9, 11], "ani": [0, 1, 2, 5, 7, 9, 20], "an": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "dictionari": [0, 1, 2, 5, 6, 7, 18], "argument": [0, 1, 5, 9, 17, 18, 19], "pass": [0, 1, 2, 5, 7, 17, 26, 28], "tenac": [0, 1, 5], "hook_nam": [0, 2], "run_now": 0, "json": [0, 5, 7, 11, 17, 18, 19, 21, 26], "function": [0, 5, 17, 19, 26, 28], "call": [0, 2, 5, 9, 17, 19, 26, 28], "us": [0, 2, 5, 6, 7, 9, 13, 15, 17, 18, 19, 20], "bodi": [0, 17], "run_id": [0, 5, 9], "type": [0, 2, 5, 9, 11, 17, 18, 19, 26, 28], "submit_run": 0, "list_job": 0, "limit": 0, "25": [0, 17], "offset": 0, "expand_task": 0, "fals": [0, 2, 6, 7, 11], "batch": [0, 17], "size": 0, "retriev": [0, 5], "first": [0, 5, 7, 13, 15, 19, 26, 28], "rel": 0, "most": [0, 5, 20], "recent": 0, "creat": [0, 5, 6, 7, 13, 15, 17, 19, 20, 27], "bool": [0, 2, 5, 6, 7, 23], "whether": [0, 2, 5], "includ": [0, 17, 28], "task": [0, 5, 9, 13, 15, 17, 19, 26, 28], "detail": [0, 5, 17, 19, 20], "respons": 0, "A": [0, 5], "find_job_id_by_nam": 0, "job_nam": [0, 5, 26], "find": 0, "id": [0, 5, 6, 7, 9, 20, 24, 25, 26, 27, 28], "its": [0, 6, 23, 25], "name": [0, 2, 5, 6, 7, 17, 18, 19, 21, 23, 24, 25, 26, 27], "If": [0, 2, 5, 6, 7, 9, 18, 19, 20, 23, 28], "ar": [0, 5, 7, 9, 17, 18, 19, 20, 21, 26, 27, 28], "multipl": [0, 26, 28], "same": [0, 5, 6, 7, 26, 28], "rais": 0, "airflowexcept": 0, "look": [0, 5, 7], "up": [0, 5, 17], "job_id": [0, 5, 17, 26], "wa": [0, 2, 5, 9], "found": [0, 21, 26, 27], "get_run_page_url": 0, "run_page_url": [0, 5], "url": [0, 6, 18, 23], "page": [0, 17, 20], "async": [0, 1, 9], "a_get_run_page_url": 0, "param": [0, 28], "get_job_id": 0, "from": [0, 5, 6, 7, 11, 17, 18, 19, 21, 23], "given": [0, 5, 6, 7, 23, 25, 27], "get_run_st": 0, "pleas": [0, 20], "note": [0, 5, 17, 18, 26, 28], "method": [0, 5, 6, 7, 17, 18, 19], "failur": 0, "unless": 0, "you": [0, 5, 18, 19, 20, 23, 24, 25, 26, 28], "have": [0, 5, 11, 13, 15, 17, 18, 19, 27], "xcom": [0, 5, 7], "pickl": 0, "can": [0, 5, 6, 7, 11, 18, 19, 20, 23, 26, 28], "done": 0, "follow": [0, 5, 18, 20, 21, 23, 24, 25, 26, 27, 28], "environ": [0, 18], "variabl": [0, 18], "airflow__core__enable_xcom_pickl": 0, "do": [0, 17, 19, 20, 28], "want": [0, 9, 19, 20], "get_run_state_str": 0, "string": [0, 2, 7, 11, 17, 19, 21, 23, 24, 25, 27], "describ": [0, 2, 7, 20, 26], "get_run_state_lifecycl": 0, "get_run_state_result": 0, "get_run_state_messag": 0, "individu": [0, 7, 17], "compon": [0, 18], "a_get_run_st": 0, "get_run": 0, "inform": [0, 5, 7, 9, 13, 15, 17, 19], "a_get_run": 0, "represent": [0, 5], "lifecycl": 0, "messag": 0, "get_run_output": 0, "cancel_run": 0, "restart_clust": 0, "contain": [0, 5, 6, 7, 11, 18, 20, 21], "specif": [0, 5, 6, 7, 17, 18, 28], "start_clust": 0, "terminate_clust": 0, "cluster_id": 0, "arrai": [0, 5, 28], "update_repo": [0, 25], "repo_id": [0, 6, 24, 25], "updat": [0, 6, 17, 19], "repo": [0, 6, 14, 17, 19], "payload": [0, 5, 17, 26, 28], "metadata": [0, 1], "delete_repo": [0, 24], "create_repo": [0, 23], "get_repo_by_path": 0, "path": [0, 2, 5, 6, 7, 9, 13, 15, 18, 21, 23, 27, 28], "obtain": [0, 18], "repositori": [0, 5, 6, 19, 23], "exist": [0, 5, 6, 19, 23, 24, 25, 26, 27, 28], "doesn": 0, "t": [0, 6, 7, 18, 23, 24, 25, 26], "test_connect": [0, 2, 17, 19], "test": [0, 2, 17, 19, 21, 23, 24, 25, 27, 28], "ui": 0, "4": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28], "dev0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "experiment": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "featur": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28], "azure_default_ad_endpoint": 1, "login": [1, 18], "microsoftonlin": [1, 18], "azure_token_service_url": 1, "oauth2": 1, "token": [1, 5, 6, 17, 18, 19], "azure_metadata_service_token_url": 1, "169": 1, "254": 1, "ident": [1, 17, 18, 19], "azure_metadata_service_instance_url": 1, "instanc": [1, 5, 17, 19], "token_refresh_lead_tim": 1, "120": 1, "azure_management_endpoint": 1, "manag": [1, 17, 18, 19, 20], "core": [1, 21], "window": [1, 21], "net": [1, 21], "default_databricks_scop": 1, "2ff814a6": 1, "3304": 1, "4ab8": 1, "85cb": 1, "cd0e6f879c1d": 1, "unknown": [1, 20], "basehook": 1, "conn_name_attr": 1, "databricks_default": [1, 5, 6, 18], "conn_typ": 1, "extra_paramet": 1, "host": [1, 5, 6, 17, 18, 19], "use_azure_managed_ident": [1, 18], "azure_ad_endpoint": [1, 18], "azure_resource_id": [1, 18], "databricks_conn": 1, "get_conn": [1, 2], "user_agent_head": 1, "user_agent_valu": 1, "__aenter__": 1, "__aexit__": 1, "err": 1, "bearerauth": 1, "aiohttp": [1, 19], "basicauth": 1, "onli": [1, 2, 5, 7, 11, 18, 19, 21, 26, 27], "ship": 1, "bearer": [1, 18], "auth": [1, 18], "we": [1, 5, 6, 11, 13, 15, 23], "need": [1, 5, 9, 18, 19, 23, 24, 25, 28], "subclass": 1, "encod": [1, 18], "credenti": [1, 7, 18], "list_sql_endpoints_endpoint": 2, "sql": [2, 7, 17, 18, 19, 21, 27], "databrickssqlhook": [2, 7, 17, 19], "http_path": [2, 7, 18, 21, 27], "sql_endpoint_nam": [2, 7, 21, 27], "session_configur": [2, 7, 18], "http_header": [2, 7], "catalog": [2, 7], "schema": [2, 7], "common": [2, 5, 6, 17, 19], "dbapihook": [2, 17, 19], "specifi": [2, 5, 6, 7, 17, 18, 19, 21, 23, 26, 27], "should": [2, 5, 6, 7, 9, 18, 20, 27, 28], "either": [2, 5, 7, 9, 18, 24, 25, 26], "": [2, 5, 6, 7, 17, 18, 19, 20, 23, 28], "extra": [2, 5, 6, 7, 17, 18, 19], "must": [2, 5, 6, 7, 11, 18, 23, 27, 28], "abov": [2, 7, 20], "session": [2, 7, 18], "default": [2, 5, 6, 7, 9, 17, 27], "could": [2, 7, 18, 21, 27], "tupl": [2, 7, 9], "k": [2, 7], "v": [2, 7, 27], "pair": [2, 7], "set": [2, 5, 7, 13, 15, 18], "header": [2, 7, 17, 18, 21], "everi": [2, 5, 7, 9], "initi": [2, 7, 17, 19, 28], "requir": [2, 5, 6, 7, 18, 21, 23, 24, 25, 26, 27], "dbr": [2, 7], "9": [2, 7, 28], "addit": [2, 7, 26], "connector": [2, 7, 17, 18, 19], "object": [2, 5, 17, 18, 19, 28], "autocommit": 2, "handler": 2, "split_stat": 2, "return_last": 2, "command": [2, 5, 7, 17, 21, 28], "statement": [2, 7], "them": [2, 19], "execut": [2, 5, 6, 7, 13, 15, 25, 28], "sequenti": [2, 13, 15], "union": [2, 5, 7], "iter": [2, 7], "what": 2, "queri": [2, 7, 17, 27], "map": [2, 5, 7, 18], "render": [2, 5, 6, 7], "callabl": 2, "which": [2, 5, 7, 13, 15, 17, 18, 19, 28], "each": [2, 5, 26, 27, 28], "split": 2, "singl": [2, 7, 9, 28], "separ": [2, 17], "last": 2, "all": [2, 7, 11, 17, 18, 19, 26], "after": [2, 17], "express": [2, 7], "simpl": [2, 27], "abstract": 2, "bulk_dump": 2, "tabl": [2, 5, 7, 21, 27, 28], "tmp_file": 2, "dump": 2, "databas": 2, "tab": 2, "delimit": 2, "file": [2, 5, 7, 17, 19, 20, 21, 28], "target": 2, "bulk_load": 2, "load": [2, 17, 21], "databricks_sql": [3, 4, 8], "hook": [4, 17, 18, 19], "databricks_repo": [4, 8], "trigger": [4, 5, 17, 19, 26], "defer_method_nam": 5, "execute_complet": 5, "xcom_run_id_kei": 5, "xcom_run_page_url_kei": 5, "databricksjobrunlink": [5, 17, 19], "model": [5, 6, 7], "baseoperatorlink": 5, "construct": [5, 7], "link": [5, 17, 19, 20], "monitor": 5, "see": [5, 17, 18, 19, 23], "get_link": 5, "dttm": 5, "ti_kei": 5, "extern": 5, "system": [5, 18, 21, 23, 24, 25, 27, 28], "old": 5, "signatur": [5, 17, 19, 20], "datetim": 5, "That": 5, "still": 5, "support": [5, 7, 17, 18, 19, 21, 23, 28], "runtim": [5, 17, 19], "deprec": 5, "associ": 5, "taskinst": [5, 17, 19], "taskinstancekei": 5, "databrickssubmitrunoper": [5, 13, 15, 17, 19, 22], "spark_jar_task": [5, 28], "notebook_task": [5, 28], "spark_python_task": [5, 28], "spark_submit_task": [5, 28], "pipeline_task": [5, 28], "new_clust": [5, 13, 28], "existing_cluster_id": [5, 28], "run_nam": [5, 28], "polling_period_second": [5, 9], "30": [5, 9, 17], "databricks_retry_limit": [5, 6, 23, 24, 25], "databricks_retry_delai": [5, 6, 23, 24, 25], "databricks_retry_arg": 5, "do_xcom_push": [5, 7], "idempotency_token": 5, "access_control_list": 5, "wait_for_termin": [5, 17, 19], "git_sourc": [5, 17, 19], "baseoper": [5, 6, 7], "There": [5, 18, 20, 26, 27, 28], "three": [5, 28], "wai": [5, 18, 26, 27, 28], "instanti": [5, 9, 26, 28], "For": [5, 7, 13, 15, 17, 18, 19, 20, 26], "more": [5, 6, 7, 13, 15, 17, 18, 19, 26, 28], "how": [5, 7, 17, 20], "take": [5, 7, 26, 28], "guid": [5, 7, 17, 20], "runsubmittaskset": 5, "100": 5, "item": 5, "jobsrunssubmit": 5, "directli": [5, 26, 28], "e": [5, 9, 18], "merg": [5, 28], "thei": [5, 9, 11, 28], "conflict": [5, 28], "dure": [5, 28], "preced": [5, 28], "overrid": [5, 17, 19, 28], "top": [5, 19, 20, 26, 28], "level": [5, 17, 26, 28], "kei": [5, 6, 20, 28], "templat": [5, 6, 7, 17, 19], "about": [5, 13, 15, 17, 20], "jinja": [5, 6, 7], "main": [5, 6, 7, 17, 19, 20, 28], "jar": [5, 13, 15, 28], "actual": [5, 23], "OR": 5, "field": [5, 6, 11, 17, 18, 19], "jobssparkjartask": 5, "notebook": [5, 13, 15, 28], "jobsnotebooktask": 5, "python": [5, 17, 19, 20, 28], "jobssparkpythontask": 5, "jobssparksubmittask": 5, "delta": [5, 28], "live": [5, 28], "pipelin": [5, 28], "least": [5, 19, 23], "pipeline_id": 5, "jobspipelinetask": 5, "spec": [5, 28], "new": [5, 13, 15, 17, 19, 26, 28], "except": [5, 6, 11, 17, 19, 23, 26], "when": [5, 6, 7, 9, 11, 17, 18, 19, 28], "jobsclusterspecnewclust": 5, "managedlibrarieslibrari": 5, "By": [5, 6, 9, 20], "task_id": [5, 21, 23, 24, 25, 27, 28], "superclass": 5, "guarante": 5, "idempot": 5, "alreadi": [5, 6, 20, 23], "doe": [5, 11], "instead": [5, 17, 18, 28], "64": 5, "charact": [5, 17], "repres": [5, 17, 19, 27], "access": [5, 18], "control": [5, 7, 9, 28], "acl": 5, "consist": 5, "subject": [5, 17], "user_nam": 5, "user": [5, 6, 14, 17, 18, 19, 20, 23, 24, 25, 28], "group_nam": 5, "group": 5, "permission_level": 5, "document": [5, 7, 17, 18, 19, 21, 23, 26, 27], "timeout": 5, "mean": 5, "To": [5, 6, 20, 23, 24, 25], "authent": [5, 6, 7, 17, 19], "leav": [5, 6, 18], "empti": [5, 6, 17, 18, 19], "rate": [5, 9], "poll": [5, 9], "backend": [5, 6, 23, 24, 25], "unreach": [5, 6, 23, 24, 25], "Its": [5, 6], "greater": [5, 6], "than": [5, 6], "equal": [5, 6], "push": [5, 7], "remot": 5, "git": [5, 6, 23, 25], "template_field": [5, 6, 7, 17], "sequenc": [5, 6, 7, 17], "template_ext": [5, 7, 17, 19], "tpl": 5, "ui_color": 5, "1cb1c2": 5, "ui_fgcolor": 5, "fff": 5, "operator_extra_link": 5, "context": [5, 6, 7, 9, 17], "deriv": [5, 6, 7], "get_template_context": [5, 6, 7], "on_kil": 5, "cleanup": [5, 9, 17], "subprocess": 5, "kill": 5, "thread": 5, "multiprocess": 5, "within": 5, "clean": [5, 17], "ghost": 5, "process": [5, 9], "behind": 5, "databrickssubmitrundeferrableoper": [5, 22], "deferr": [5, 17, 19, 26, 28], "event": [5, 9, 11], "databricksrunnowoper": [5, 17, 19, 22], "notebook_param": [5, 26], "python_param": [5, 26], "jar_param": [5, 26], "spark_submit_param": [5, 26], "python_named_param": 5, "two": [5, 13, 15, 26], "In": [5, 9, 13, 15, 26, 28], "typic": [5, 26, 28], "our": [5, 17, 26, 28], "through": [5, 26, 28], "exampl": [5, 13, 15, 17, 18, 19, 20, 26], "42": 5, "dry": 5, "oldest": 5, "consid": [5, 18], "1457570074236": 5, "notebook_run": [5, 28], "anoth": [5, 9, 26, 28], "accomplish": [5, 26, 28], "thing": [5, 26, 28], "exactli": [5, 26, 28], "one": [5, 11, 20, 26, 28], "your": [5, 19], "code": [5, 7, 20, 25], "would": 5, "like": [5, 9, 23, 24, 25], "dougla": 5, "adam": 5, "org": [5, 20], "apach": [5, 20], "sparkpi": 5, "where": [5, 28], "both": [5, 13, 15, 18, 28], "AND": [5, 28], "togeth": [5, 7, 28], "python_named_paramet": [5, 26], "It": [5, 20, 26, 28], "mutual": 5, "exclus": 5, "g": [5, 9, 18], "john": 5, "ag": 5, "35": 5, "dbutil": 5, "widget": 5, "upon": 5, "cannot": [5, 17, 19], "conjunct": 5, "exce": 5, "10": [5, 17, 19, 28], "000": 5, "byte": 5, "line": [5, 17], "overwrit": 5, "wheel": [5, 19], "script": [5, 17, 20], "databricksrunnowdeferrableoper": [5, 22], "databricksreposcreateoper": [6, 22], "git_url": [6, 23], "git_provid": [6, 23], "branch": [6, 23, 25], "tag": [6, 23, 25], "repo_path": [6, 14, 23, 24, 25], "ignore_existing_repo": [6, 23], "check": [6, 17, 19, 20, 28], "guess": [6, 23], "format": [6, 7, 21, 27], "folder": [6, 17, 20], "directori": [6, 18, 20, 23], "checkout": [6, 23], "don": [6, 23], "throw": [6, 11, 23, 26], "__git_providers__": 6, "__aws_code_commit_regexp__": 6, "__repos_path_regexp__": 6, "static": [6, 17], "__detect_repo_provider__": 6, "databricksreposupdateoper": [6, 22], "patch": [6, 17], "omit": 6, "databricksreposdeleteoper": [6, 22], "databrickssqloper": [7, 17, 18, 19, 22], "output_path": [7, 27], "output_format": [7, 27], "csv": 7, "csv_param": 7, "client_paramet": 7, "recogn": 7, "end": 7, "write": 7, "select": [7, 20], "possibl": [7, 17], "jsonl": [7, 27], "dictwrit": 7, "_output_path": 7, "template_fields_render": 7, "copy_into_approved_format": 7, "avro": [7, 21], "orc": [7, 21], "parquet": [7, 21], "text": [7, 21], "binaryfil": [7, 21], "databrickscopyintooper": [7, 17, 19, 22], "table_nam": [7, 21], "file_loc": [7, 21], "file_format": [7, 21], "pattern": 7, "expression_list": 7, "storage_credenti": 7, "encrypt": 7, "format_opt": [7, 21], "force_copi": [7, 21], "copy_opt": 7, "valid": [7, 11, 20], "copi": [7, 21], "INTO": [7, 21], "piec": 7, "locat": [7, 21], "import": [7, 17, 19, 20], "regex": 7, "match": [7, 17, 19, 20], "configur": [7, 21, 27], "against": [7, 27], "uniti": 7, "storag": 7, "destin": 7, "forc": 7, "also": [7, 20, 28], "row": 7, "integ": [7, 23, 24, 25], "n": 7, "right": 7, "_file_loc": 7, "_file": 7, "_table_nam": 7, "basetrigg": 9, "databricksexecutiontrigg": [9, 11], "handl": [9, 17, 19], "logic": 9, "commun": 9, "serial": 9, "reconstruct": 9, "keyword": 9, "re": [9, 18], "asynchron": 9, "yield": 9, "whenev": 9, "fire": 9, "off": 9, "finish": [9, 27], "thu": 9, "immedi": 9, "resum": 9, "veri": 9, "quickli": 9, "mai": [9, 17, 18], "workload": 9, "being": 9, "move": [9, 17, 19], "multi": 9, "defer": 9, "assum": 9, "persist": 9, "reli": 9, "longer": 9, "normalise_json_cont": 11, "json_path": 11, "normal": 11, "non": [11, 17], "numer": 11, "boolean": [11, 17, 18, 19], "reason": 11, "why": [11, 20], "becaus": [11, 13, 15, 28], "render_templ": 11, "fail": [11, 17, 19], "convert": [11, 17, 19], "understand": 11, "validate_trigger_ev": 11, "correct": [11, 17, 19, 20], "receiv": 11, "dag": [13, 15, 17, 19], "upload": [13, 15], "dbf": [13, 15, 28], "downstream": [13, 15], "depend": [13, 15, 17, 26], "NOT": [13, 15], "until": [13, 15], "complet": [13, 15, 19], "successfulli": [13, 15], "definit": [13, 15], "ha": [13, 15], "env_id": [13, 14, 15], "dag_id": [13, 14, 15], "example_databricks_oper": 13, "test_run": [13, 14, 15], "default_arg": [14, 17], "example_databricks_repos_oper": 14, "domain": [14, 23, 24, 25], "demo": [14, 23, 24, 25], "example_databricks_sql_oper": 15, "connection_id": [15, 21, 27], "my_connect": 15, "example_databrick": [16, 28], "example_databricks_repo": [16, 23, 24, 25], "example_databricks_sql": [16, 21, 27], "commit": 17, "chang": 17, "high": 17, "changelog": 17, "2022": [17, 19], "08": 17, "12": [17, 20, 28], "4d32f61fd0": 17, "fix": 17, "x": [17, 19, 28], "25674": [17, 19], "e5ac6c7cfb": 17, "prepar": 17, "releas": [17, 19, 25], "august": 17, "25618": 17, "52f2f5bfa8": 17, "07": 17, "agent": [17, 19], "25578": [17, 19], "0255a0a5e7": 17, "04": 17, "deep_string_coerc": [17, 19], "25394": [17, 19], "679a85325a": 17, "03": 17, "correctli": [17, 19], "25427": [17, 19], "82f842ffc5": 17, "28": 17, "24599": 17, "54a8c4fd2a": 17, "27": [17, 19], "improv": [17, 19], "25260": [17, 19], "7438707747": 17, "26": 17, "telemetri": [17, 19], "25115": [17, 19], "df00436569": 17, "22": 17, "unifi": [17, 19], "23971": [17, 19], "2f70daf5ac": 17, "18": 17, "implement": [17, 19], "25114": [17, 19], "13": 17, "d2459a241b": 17, "add": [17, 18, 19], "juli": 17, "25030": 17, "8dfe7bf5ff": 17, "11": [17, 20, 28], "ad": [17, 18, 19], "24945": [17, 19], "acaa0635c8": 17, "automat": [17, 19], "detect": [17, 28], "lazi": 17, "log": 17, "interpol": 17, "24910": 17, "46bbfdade0": 17, "class": [17, 19, 21, 27, 28], "24836": [17, 19], "96b01a8012": 17, "05": 17, "remov": [17, 19, 20], "bad": 17, "codebas": 17, "24841": 17, "0de31bd73a": 17, "06": 17, "29": 17, "insid": [17, 18], "24672": 17, "510a6bab45": 17, "yaml": 17, "24702": 17, "ed37c3a0e8": 17, "24617": [17, 19], "9c59831ee7": 17, "21": 17, "functool": [17, 19], "compat": [17, 19], "cached_properti": [17, 19], "24582": [17, 19], "09": 17, "dcdcf3a2b8": 17, "rc2": 17, "24307": 17, "717a7588bc": 17, "descript": 17, "doubl": 17, "min": 17, "24292": 17, "aeabe994b3": 17, "24231": 17, "027b707d21": 17, "explanatori": 17, "contributor": [17, 18], "24229": 17, "ddf9013098": 17, "aip": 17, "47": 17, "migrat": [17, 19], "design": 17, "22442": 17, "24203": 17, "acf89510cd": 17, "defin": [17, 18, 19], "23622": [17, 19], "23641": [17, 19], "92ddcf4ac6": 17, "introduc": [17, 26, 28], "flake8": 17, "implicit": 17, "concat": 17, "plugin": 17, "23873": 17, "6150d28323": 17, "19736": [17, 19], "cf5a78e91c": 17, "unboundlocalerror": [17, 19], "23815": [17, 19], "d0a5b3a4f2": 17, "23620": [17, 19], "75c60923e0": 17, "23631": 17, "428a439953": 17, "f": 17, "concaten": [17, 19], "23591": 17, "a58506b2a6": 17, "address": 17, "review": 17, "comment": 17, "6a3d6cc32b": 17, "dbsql": [17, 19], "7b3bf4e435": 17, "switch": [17, 18, 19], "f02b0b6b40": 17, "further": [17, 19], "23199": [17, 19], "8b6b0848a3": 17, "23": 17, "brees": 17, "build": [17, 20], "pull": 17, "verifi": [17, 19], "imag": 17, "23104": 17, "40831144be": 17, "march": 17, "22979": 17, "7be57eb256": 17, "22886": [17, 19], "aa8c08db38": 17, "22885": [17, 19], "6933022e94": 17, "mypi": 17, "error": [17, 19, 28], "22884": 17, "56ab82ed7a": 17, "mid": 17, "april": 17, "22819": 17, "1b12c93ed3": 17, "31": 17, "refactor": [17, 19], "hoc": [17, 19], "22571": [17, 19], "95169d1d07": 17, "22541": [17, 19], "352d7f72dd": 17, "22422": [17, 19], "c063fc688c": 17, "black": 17, "precommit": 17, "22521": 17, "d7dbfb7e26": 17, "bugfix": 17, "22383": 17, "cc920963a6": 17, "15": 17, "22278": [17, 19], "14": 17, "16adc035b1": 17, "classifi": 17, "22226": 17, "12e9e2c695": 17, "429": [17, 19], "well": [17, 19, 20], "21852": [17, 19], "af9d85ccd8": 17, "some": [17, 19], "22221": [17, 19], "4014194320": 17, "22076": [17, 19], "f5b96315fe": 17, "feb": 17, "22056": 17, "62bf1276f6": 17, "01": 17, "show": [17, 19], "21709": [17, 19], "27d19e7626": 17, "02": 17, "21363": [17, 19], "a1845c68f9": 17, "21663": [17, 19], "7cca82495b": 17, "21494": [17, 19], "0a2d0d1ecb": 17, "18925": [17, 19], "21530": [17, 19], "d94fa37830": 17, "januari": [17, 19], "delai": [17, 19], "21439": [17, 19], "6c3a67d4fc": 17, "2021": [17, 20], "21257": 17, "602abe8394": 17, "20": 17, "sphinx": 17, "autoapi": 17, "typehint": 17, "20951": 17, "f77417eb0d": 17, "k8": 17, "pypi": [17, 19], "20614": 17, "97496ba2b4": 17, "decemb": 17, "20523": 17, "0bf424f37f": 17, "20598": 17, "d56e7b56bb": 17, "friendli": 17, "20571": 17, "a0821235fb": 17, "everywher": 17, "20565": 17, "c5c18c54fa": 17, "verif": [17, 19], "20550": [17, 19], "d3b3161f0d": 17, "attr": [17, 19], "20540": [17, 19], "58afc19377": 17, "20536": [17, 19], "e7659d08b0": 17, "20526": [17, 19], "cad39274d9": 17, "20265": 17, "820bfed515": 17, "20205": 17, "66f94f95c2": 17, "db": [17, 19], "__init__": [17, 19], "20180": [17, 19], "545ca59ba9": 17, "unhid": 17, "entri": 17, "20128": 17, "637db1a0ba": 17, "novemb": 17, "20086": 17, "728e94a47": 17, "19835": 17, "4925b37b66": 17, "expir": [17, 19], "20036": [17, 19], "853576d901": 17, "19882": 17, "11998848a4": 17, "24": 17, "19723": [17, 19], "56bdfe7a84": 17, "allow": [17, 18, 19, 26, 28], "azur": [17, 18, 19], "sp": [17, 19], "cloud": [17, 18, 19], "19722": [17, 19], "244627e3da": 17, "pat": [17, 18, 19], "password": [17, 18, 19], "19585": [17, 19], "0a4a8bdb94": 17, "19544": [17, 19], "8ae878953b": 17, "19412": [17, 19], "28b51fb7bd": 17, "aad": [17, 18, 19], "19335": [17, 19], "3a0c455855": 17, "19443": [17, 19], "d9567eb106": 17, "octob": 17, "19321": 17, "f5ad26dcdd": 17, "fixup": [17, 19], "19099": [17, 19], "840ea3efb9": 17, "septemb": 17, "18613": 17, "ef037e7021": 17, "start_dat": 17, "misc": 17, "18597": 17, "0b7b13372f": 17, "18339": [17, 19], "0a68588479": 17, "17890": 17, "be75dcd39c": 17, "meta": 17, "76ed2a49c6": 17, "19": 17, "lazili": 17, "17682": 17, "87f408b1e7": 17, "17116": 17, "b916b75079": 17, "17015": 17, "866a601b76": 17, "pylint": 17, "toolchain": 17, "16682": 17, "bbc627a3da": 17, "16501": 17, "cbf8001d76": 17, "16": 17, "synchron": 17, "buggfix": 17, "16464": 17, "1fba5402bb": 17, "june": 17, "16405": 17, "9c94b72d44": 17, "16294": 17, "37681bca00": 17, "auto": [17, 19], "appli": [17, 19], "apply_default": [17, 19], "decor": [17, 19], "15667": [17, 19], "807ad32ce5": 17, "pip": [17, 19, 20], "15576": 17, "df143aee8d": 17, "rework": 17, "15444": 17, "49cae1f052": 17, "17": 17, "15410": 17, "68e4c4dcb0": 17, "backport": 17, "14886": 17, "88bdcfa0df": 17, "wave": 17, "14013": 17, "ac2f72c98d": 17, "13767": 17, "a9ac2b040b": 17, "flynt": 17, "13732": 17, "3fd5ef3555": 17, "miss": 17, "logo": 17, "integr": [17, 18], "13717": 17, "295d66f914": 17, "2020": 17, "grammar": 17, "warn": [17, 20], "13380": 17, "6cf76d7ac0": 17, "typo": 17, "upgrad": [17, 19], "13148": 17, "32971a1a2d": 17, "12955": 17, "b40dffa085": 17, "renam": 17, "rema": 17, "modul": 17, "12917": 17, "9b39f24780": 17, "dynam": 17, "form": 17, "per": 17, "12558": 17, "bd90136aaf": 17, "12681": 17, "c34ef853c8": 17, "12444": 17, "0080354502": 17, "readm": [17, 19], "0b2": 17, "12449": 17, "7ca0b6f121": 17, "markdownlint": 17, "rule": 17, "md003": 17, "head": 17, "style": 17, "12427": 17, "12438": 17, "ae7cb4a1e2": 17, "wrong": 17, "hash": 17, "12390": 17, "6889a333cf": 17, "ref": 17, "12366": 17, "7825e8f590": 17, "12304": 17, "b027223132": 17, "12316": 17, "85a18e13d9": 17, "project": 17, "cross": 17, "12212": 17, "59eb5de78c": 17, "come": 17, "0beta1": 17, "12206": 17, "b2a28d1590": 17, "12082": 17, "7e0d08e1f0": 17, "12175": 17, "4e8f9cc8d0": 17, "formmatt": 17, "9550": 17, "8c42cf1b00": 17, "pyupgrad": 17, "11447": 17, "5a439e84eb": 17, "2a1": 17, "11855": 17, "872b1566a1": 17, "gener": [17, 18], "setup": 17, "11826": 17, "349b0811c3": 17, "d200": 17, "pydocstyl": 17, "11688": 17, "16e7129719": 17, "11487": 17, "0a0e1af800": 17, "broken": 17, "markdown": 17, "toc": 17, "11249": 17, "ca4238eb4d": 17, "month": 17, "11242": 17, "5220e4c384": 17, "11238": 17, "54353f8745": 17, "increas": 17, "coverag": 17, "five": 17, "differ": [17, 18, 20], "11170": 17, "966a06d96b": 17, "fetch": 17, "suppli": 17, "10762": 17, "9549274d11": 17, "8b1": 17, "10818": 17, "fdd9b6f65b": 17, "10543": 17, "bfefcce0c9": 17, "rest": [17, 28], "so": 17, "10462": 17, "3696c34c28": 17, "word": 17, "10528": 17, "2f2d8dbfaf": 17, "noinspect": 17, "nativ": 17, "intellij": 17, "10525": 17, "ee7ca128a1": 17, "refernc": 17, "10483": 17, "cdec301254": 17, "sensor": 17, "10205": 17, "7d24b088cd": 17, "stop": 17, "example_dag": 17, "9985": 17, "e13a14c873": 17, "whitespac": 17, "relat": [17, 18], "9458": 17, "d0e7db4024": 17, "fresh": 17, "9408": 17, "12af6a0800": 17, "final": 17, "23rc1": 17, "9404": 17, "c7e5bce57f": 17, "candid": 17, "9370": 17, "f6bd817a3a": 17, "transfer": 17, "9320": 17, "0b0e4f7a4c": 17, "rc3": 17, "relas": 17, "9026": 17, "00642a46d0": 17, "remain": 17, "wrongli": 17, "8994": 17, "f1073381ed": 17, "8846": 17, "375d1ca229": 17, "8898": 17, "12c5e5d8a": 17, "8891": 17, "f3521fb0e3": 17, "regener": 17, "8886": 17, "92585ca4cb": 17, "autom": 17, "8807": 17, "649935e8c": 17, "8472": 17, "_do_api_cal": 17, "8473": 17, "16903ba3a6": 17, "8474": 17, "8475": 17, "5648dfbc30": 17, "super": 17, "amazon": 17, "cloudant": 17, "7827": 17, "3320e432a1": 17, "6817": 17, "keep": 17, "face": 17, "untouch": 17, "7517": 17, "4d03e33c11": 17, "py": [17, 21, 23, 24, 25, 27, 28], "replac": 17, "explicit": 17, "md": [17, 19], "squash": 17, "rebas": 17, "7456": 17, "97a429f9d0": 17, "6714": 17, "magic": 17, "utf": 17, "8": [17, 28], "7338": 17, "83c037873f": 17, "6674": [17, 20], "accord": 17, "7287": 17, "c42a375e79": 17, "6644": 17, "7265": 17, "sever": 18, "airflow": [18, 20, 23, 24, 25, 26, 28], "person": 18, "recommend": [18, 20], "usernam": 18, "account": [18, 21], "discourag": 18, "activ": 18, "princip": 18, "secret": 18, "outsid": 18, "owner": [18, 20], "permiss": 18, "vm": 18, "assign": 18, "work": 18, "sent": 18, "basic": 18, "plan": 18, "reus": 18, "simplehttpoper": 18, "secur": 18, "necessari": 18, "azure_tenant_id": 18, "tenant": 18, "resourc": 18, "isn": [18, 24, 25, 26], "special": 18, "govcloud": 18, "china": 18, "germani": 18, "protocol": 18, "de": 18, "flag": 18, "packag": 18, "uri": [18, 21], "syntax": 18, "export": 18, "airflow_conn_databricks_default": 18, "yourtoken": 18, "below": [19, 20], "minimum": 19, "via": [19, 20, 23, 24, 25, 26, 28], "those": [19, 20], "order": 19, "checksum": [19, 20], "site": 19, "sdist": [19, 20], "asc": [19, 20], "sha512": [19, 20], "avail": [19, 20, 28], "explain": 19, "polici": 19, "github": [19, 23], "blob": 19, "mistakenli": 19, "install_requir": 19, "22382": 19, "optimis": 19, "due": 19, "otherwis": 19, "manual": 19, "download": 20, "provid": [20, 21, 23, 24, 25, 27, 28], "databrick": [20, 21, 26, 27, 28], "offici": 20, "choos": 20, "drop": [20, 27], "down": 20, "left": 20, "whl": 20, "origin": 20, "softwar": 20, "foundat": 20, "pgp": 20, "essenti": 20, "sha": 20, "gpg": 20, "relev": 20, "distribut": 20, "mirror": 20, "pgpk": 20, "ka": 20, "binari": 20, "pgpv": 20, "tar": 20, "gz": 20, "made": 20, "sat": 20, "sep": 20, "49": 20, "54": 20, "bst": 20, "rsa": 20, "cde15c6e4d3a8ec4ecf4ba4b6674e08ad7de406f": 20, "issuer": 20, "kaxilnaik": 20, "good": 20, "kaxil": 20, "naik": 20, "aka": 20, "gmail": 20, "certifi": 20, "trust": 20, "indic": 20, "belong": 20, "primari": 20, "fingerprint": 20, "cde1": 20, "5c6e": 20, "4d3a": 20, "8ec4": 20, "ecf4": 20, "ba4b": 20, "e08a": 20, "d7de": 20, "406f": 20, "worri": 20, "certif": 20, "sign": 20, "server": 20, "previou": 20, "step": 20, "know": 20, "sum": 20, "shasum": 20, "512": 20, "diff": 20, "local": 20, "bin": 20, "bash": 20, "package_vers": 20, "package_nam": 20, "provider_download_dir": 20, "mktemp": 20, "d": 20, "dep": 20, "dest": 20, "curl": 20, "apache_airflow_providers_databrick": 20, "py3": 20, "l": 20, "o": 20, "echo": 20, "la": 20, "onc": 20, "instruct": 20, "chapter": 20, "temporari": 20, "One": [21, 27], "usag": [21, 23, 24, 25, 27, 28], "copy_into": 21, "import_csv": 21, "my_tabl": 21, "abfss": 21, "df": 21, "my": 21, "input": [23, 24, 25], "user_email": [23, 24, 25], "repo_nam": [23, 24, 25], "decim": [23, 24, 25], "usual": 25, "worker": [26, 28], "effect": [26, 28], "extens": 27, "new_lin": 27, "select_data": 27, "my_airflow_t": 27, "store": 27, "select_into_fil": 27, "select_data_into_fil": 27, "tmp": 27, "perform": 27, "create_and_populate_t": 27, "insert": 27, "create_fil": 27, "create_and_populate_from_fil": 27, "With": 28, "approach": 28, "full": 28, "over": 28, "underli": 28, "harder": 28, "lack": 28, "spark_vers": 28, "db3": 28, "scala2": 28, "num_work": 28, "notebook_path": 28, "preparedata": 28, "here": 28, "invok": 28, "node_type_id": 28, "r3": 28, "xlarg": 28, "aws_attribut": 28, "on_demand": 28, "notebook_task_param": 28, "main_class_nam": 28, "processdata": 28, "lib": 28, "etl": 28}, "objects": {"airflow.providers": [[4, 0, 0, "-", "databricks"]], "airflow.providers.databricks": [[3, 0, 0, "-", "hooks"], [8, 0, 0, "-", "operators"], [10, 0, 0, "-", "triggers"], [12, 0, 0, "-", "utils"]], "airflow.providers.databricks.hooks": [[0, 0, 0, "-", "databricks"], [1, 0, 0, "-", "databricks_base"], [2, 0, 0, "-", "databricks_sql"]], "airflow.providers.databricks.hooks.databricks": [[0, 1, 1, "", "CANCEL_RUN_ENDPOINT"], [0, 2, 1, "", "DatabricksHook"], [0, 1, 1, "", "GET_RUN_ENDPOINT"], [0, 1, 1, "", "INSTALL_LIBS_ENDPOINT"], [0, 1, 1, "", "LIST_JOBS_ENDPOINT"], [0, 1, 1, "", "OUTPUT_RUNS_JOB_ENDPOINT"], [0, 1, 1, "", "RESTART_CLUSTER_ENDPOINT"], [0, 1, 1, "", "RUN_LIFE_CYCLE_STATES"], [0, 1, 1, "", "RUN_NOW_ENDPOINT"], [0, 2, 1, "", "RunState"], [0, 1, 1, "", "SPARK_VERSIONS_ENDPOINT"], [0, 1, 1, "", "START_CLUSTER_ENDPOINT"], [0, 1, 1, "", "SUBMIT_RUN_ENDPOINT"], [0, 1, 1, "", "TERMINATE_CLUSTER_ENDPOINT"], [0, 1, 1, "", "UNINSTALL_LIBS_ENDPOINT"], [0, 1, 1, "", "WORKSPACE_GET_STATUS_ENDPOINT"]], "airflow.providers.databricks.hooks.databricks.DatabricksHook": [[0, 3, 1, "", "a_get_run"], [0, 3, 1, "", "a_get_run_page_url"], [0, 3, 1, "", "a_get_run_state"], [0, 3, 1, "", "cancel_run"], [0, 3, 1, "", "create_repo"], [0, 3, 1, "", "delete_repo"], [0, 3, 1, "", "find_job_id_by_name"], [0, 3, 1, "", "get_job_id"], [0, 3, 1, "", "get_repo_by_path"], [0, 3, 1, "", "get_run"], [0, 3, 1, "", "get_run_output"], [0, 3, 1, "", "get_run_page_url"], [0, 3, 1, "", "get_run_state"], [0, 3, 1, "", "get_run_state_lifecycle"], [0, 3, 1, "", "get_run_state_message"], [0, 3, 1, "", "get_run_state_result"], [0, 3, 1, "", "get_run_state_str"], [0, 4, 1, "", "hook_name"], [0, 3, 1, "", "install"], [0, 3, 1, "", "list_jobs"], [0, 3, 1, "", "restart_cluster"], [0, 3, 1, "", "run_now"], [0, 3, 1, "", "start_cluster"], [0, 3, 1, "", "submit_run"], [0, 3, 1, "", "terminate_cluster"], [0, 3, 1, "", "test_connection"], [0, 3, 1, "", "uninstall"], [0, 3, 1, "", "update_repo"]], "airflow.providers.databricks.hooks.databricks.RunState": [[0, 3, 1, "", "__eq__"], [0, 3, 1, "", "__repr__"], [0, 3, 1, "", "from_json"], [0, 3, 1, "", "is_successful"], [0, 3, 1, "", "is_terminal"], [0, 3, 1, "", "to_json"]], "airflow.providers.databricks.hooks.databricks_base": [[1, 1, 1, "", "AZURE_DEFAULT_AD_ENDPOINT"], [1, 1, 1, "", "AZURE_MANAGEMENT_ENDPOINT"], [1, 1, 1, "", "AZURE_METADATA_SERVICE_INSTANCE_URL"], [1, 1, 1, "", "AZURE_METADATA_SERVICE_TOKEN_URL"], [1, 1, 1, "", "AZURE_TOKEN_SERVICE_URL"], [1, 2, 1, "", "BaseDatabricksHook"], [1, 2, 1, "", "BearerAuth"], [1, 1, 1, "", "DEFAULT_DATABRICKS_SCOPE"], [1, 1, 1, "", "TOKEN_REFRESH_LEAD_TIME"]], "airflow.providers.databricks.hooks.databricks_base.BaseDatabricksHook": [[1, 3, 1, "", "__aenter__"], [1, 3, 1, "", "__aexit__"], [1, 4, 1, "", "conn_name_attr"], [1, 4, 1, "", "conn_type"], [1, 3, 1, "", "databricks_conn"], [1, 4, 1, "", "default_conn_name"], [1, 4, 1, "", "extra_parameters"], [1, 3, 1, "", "get_conn"], [1, 3, 1, "", "host"], [1, 3, 1, "", "user_agent_header"], [1, 3, 1, "", "user_agent_value"]], "airflow.providers.databricks.hooks.databricks_base.BearerAuth": [[1, 3, 1, "", "encode"]], "airflow.providers.databricks.hooks.databricks_sql": [[2, 2, 1, "", "DatabricksSqlHook"], [2, 1, 1, "", "LIST_SQL_ENDPOINTS_ENDPOINT"]], "airflow.providers.databricks.hooks.databricks_sql.DatabricksSqlHook": [[2, 3, 1, "", "bulk_dump"], [2, 3, 1, "", "bulk_load"], [2, 3, 1, "", "get_conn"], [2, 4, 1, "", "hook_name"], [2, 3, 1, "", "run"], [2, 3, 1, "", "test_connection"]], "airflow.providers.databricks.operators": [[5, 0, 0, "-", "databricks"], [6, 0, 0, "-", "databricks_repos"], [7, 0, 0, "-", "databricks_sql"]], "airflow.providers.databricks.operators.databricks": [[5, 1, 1, "", "DEFER_METHOD_NAME"], [5, 2, 1, "", "DatabricksJobRunLink"], [5, 2, 1, "", "DatabricksRunNowDeferrableOperator"], [5, 2, 1, "", "DatabricksRunNowOperator"], [5, 2, 1, "", "DatabricksSubmitRunDeferrableOperator"], [5, 2, 1, "", "DatabricksSubmitRunOperator"], [5, 1, 1, "", "XCOM_RUN_ID_KEY"], [5, 1, 1, "", "XCOM_RUN_PAGE_URL_KEY"]], "airflow.providers.databricks.operators.databricks.DatabricksJobRunLink": [[5, 3, 1, "", "get_link"], [5, 4, 1, "", "name"]], "airflow.providers.databricks.operators.databricks.DatabricksRunNowDeferrableOperator": [[5, 3, 1, "", "execute"], [5, 3, 1, "", "execute_complete"]], "airflow.providers.databricks.operators.databricks.DatabricksRunNowOperator": [[5, 3, 1, "", "execute"], [5, 3, 1, "", "on_kill"], [5, 4, 1, "", "operator_extra_links"], [5, 4, 1, "", "template_ext"], [5, 4, 1, "", "template_fields"], [5, 4, 1, "", "ui_color"], [5, 4, 1, "", "ui_fgcolor"]], "airflow.providers.databricks.operators.databricks.DatabricksSubmitRunDeferrableOperator": [[5, 3, 1, "", "execute"], [5, 3, 1, "", "execute_complete"]], "airflow.providers.databricks.operators.databricks.DatabricksSubmitRunOperator": [[5, 3, 1, "", "execute"], [5, 3, 1, "", "on_kill"], [5, 4, 1, "", "operator_extra_links"], [5, 4, 1, "", "template_ext"], [5, 4, 1, "", "template_fields"], [5, 4, 1, "", "ui_color"], [5, 4, 1, "", "ui_fgcolor"]], "airflow.providers.databricks.operators.databricks_repos": [[6, 2, 1, "", "DatabricksReposCreateOperator"], [6, 2, 1, "", "DatabricksReposDeleteOperator"], [6, 2, 1, "", "DatabricksReposUpdateOperator"]], "airflow.providers.databricks.operators.databricks_repos.DatabricksReposCreateOperator": [[6, 4, 1, "", "__aws_code_commit_regexp__"], [6, 3, 1, "", "__detect_repo_provider__"], [6, 4, 1, "", "__git_providers__"], [6, 4, 1, "", "__repos_path_regexp__"], [6, 3, 1, "", "execute"], [6, 4, 1, "", "template_fields"]], "airflow.providers.databricks.operators.databricks_repos.DatabricksReposDeleteOperator": [[6, 3, 1, "", "execute"], [6, 4, 1, "", "template_fields"]], "airflow.providers.databricks.operators.databricks_repos.DatabricksReposUpdateOperator": [[6, 3, 1, "", "execute"], [6, 4, 1, "", "template_fields"]], "airflow.providers.databricks.operators.databricks_sql": [[7, 1, 1, "", "COPY_INTO_APPROVED_FORMATS"], [7, 2, 1, "", "DatabricksCopyIntoOperator"], [7, 2, 1, "", "DatabricksSqlOperator"]], "airflow.providers.databricks.operators.databricks_sql.DatabricksCopyIntoOperator": [[7, 3, 1, "", "execute"], [7, 4, 1, "", "template_fields"]], "airflow.providers.databricks.operators.databricks_sql.DatabricksSqlOperator": [[7, 3, 1, "", "execute"], [7, 4, 1, "", "template_ext"], [7, 4, 1, "", "template_fields"], [7, 4, 1, "", "template_fields_renderers"]], "airflow.providers.databricks.triggers": [[9, 0, 0, "-", "databricks"]], "airflow.providers.databricks.triggers.databricks": [[9, 1, 1, "", "BaseTrigger"], [9, 2, 1, "", "DatabricksExecutionTrigger"]], "airflow.providers.databricks.triggers.databricks.DatabricksExecutionTrigger": [[9, 3, 1, "", "run"], [9, 3, 1, "", "serialize"]], "airflow.providers.databricks.utils": [[11, 0, 0, "-", "databricks"]], "airflow.providers.databricks.utils.databricks": [[11, 5, 1, "", "normalise_json_content"], [11, 5, 1, "", "validate_trigger_event"]], "tests.system.providers": [[16, 0, 0, "-", "databricks"]], "tests.system.providers.databricks": [[13, 0, 0, "-", "example_databricks"], [14, 0, 0, "-", "example_databricks_repos"], [15, 0, 0, "-", "example_databricks_sql"]], "tests.system.providers.databricks.example_databricks": [[13, 1, 1, "", "DAG_ID"], [13, 1, 1, "", "ENV_ID"], [13, 1, 1, "", "new_cluster"], [13, 1, 1, "", "test_run"]], "tests.system.providers.databricks.example_databricks_repos": [[14, 1, 1, "", "DAG_ID"], [14, 1, 1, "", "ENV_ID"], [14, 1, 1, "", "default_args"], [14, 1, 1, "", "repo_path"], [14, 1, 1, "", "test_run"]], "tests.system.providers.databricks.example_databricks_sql": [[15, 1, 1, "", "DAG_ID"], [15, 1, 1, "", "ENV_ID"], [15, 1, 1, "", "connection_id"], [15, 1, 1, "", "test_run"]]}, "objtypes": {"0": "py:module", "1": "py:data", "2": "py:class", "3": "py:method", "4": "py:attribute", "5": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "data", "Python data"], "2": ["py", "class", "Python class"], "3": ["py", "method", "Python method"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"]}, "titleterms": {"airflow": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 17, 19], "provid": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19], "databrick": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 23, 24, 25], "hook": [0, 1, 2, 3], "modul": [0, 1, 2, 5, 6, 7, 9, 11, 13, 14, 15], "content": [0, 1, 2, 5, 6, 7, 9, 11, 13, 14, 15, 19], "class": [0, 1, 2, 5, 6, 7, 9], "attribut": [0, 1, 2, 5, 7, 9], "databricks_bas": 1, "databricks_sql": [2, 7], "submodul": [3, 8, 10, 12, 16], "subpackag": 4, "oper": [5, 6, 7, 8, 21, 22, 23, 24, 25, 26, 27, 28], "databricks_repo": 6, "trigger": [9, 10], "util": [11, 12], "function": 11, "test": [13, 14, 15, 16], "system": [13, 14, 15, 16], "example_databrick": 13, "example_databricks_repo": 14, "example_databricks_sql": 15, "packag": [17, 19, 20], "apach": [17, 19], "3": [17, 19], "2": [17, 19], "0": [17, 19], "1": [17, 19], "7": [17, 19], "6": [17, 19], "5": [17, 19], "4": [17, 19], "connect": 18, "authent": 18, "default": 18, "id": 18, "configur": 18, "guid": 19, "refer": 19, "resourc": 19, "commit": 19, "instal": [19, 20], "requir": 19, "cross": 19, "depend": 19, "download": 19, "offici": 19, "changelog": 19, "featur": 19, "bug": 19, "fix": 19, "break": 19, "chang": 19, "misc": 19, "from": [20, 27], "sourc": 20, "releas": 20, "integr": 20, "verifi": 20, "pypi": 20, "databrickscopyintooper": 21, "us": [21, 23, 24, 25, 26, 27, 28], "exampl": [21, 23, 24, 25, 27, 28], "import": 21, "csv": 21, "data": [21, 27], "databricksreposcreateoper": 23, "creat": 23, "repo": [23, 24, 25], "databricksreposdeleteoper": 24, "delet": 24, "specifi": [24, 25, 28], "path": [24, 25], "databricksreposupdateoper": 25, "updat": 25, "databricksrunnowoper": 26, "databricksrunnowdeferrableoper": 26, "databrickssqloper": 27, "select": 27, "file": 27, "execut": 27, "multipl": 27, "statement": 27, "databrickssubmitrunoper": 28, "paramet": 28, "json": 28, "name": 28, "databrickssubmitrundeferrableoper": 28}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx": 56}})