| Search.setIndex({docnames:["_api/airflow/providers/apache/spark/hooks/index","_api/airflow/providers/apache/spark/hooks/spark_jdbc/index","_api/airflow/providers/apache/spark/hooks/spark_jdbc_script/index","_api/airflow/providers/apache/spark/hooks/spark_sql/index","_api/airflow/providers/apache/spark/hooks/spark_submit/index","_api/airflow/providers/apache/spark/index","_api/airflow/providers/apache/spark/operators/index","_api/airflow/providers/apache/spark/operators/spark_jdbc/index","_api/airflow/providers/apache/spark/operators/spark_sql/index","_api/airflow/providers/apache/spark/operators/spark_submit/index","connections/spark","index","operators"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":1,"sphinx.ext.intersphinx":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["_api/airflow/providers/apache/spark/hooks/index.rst","_api/airflow/providers/apache/spark/hooks/spark_jdbc/index.rst","_api/airflow/providers/apache/spark/hooks/spark_jdbc_script/index.rst","_api/airflow/providers/apache/spark/hooks/spark_sql/index.rst","_api/airflow/providers/apache/spark/hooks/spark_submit/index.rst","_api/airflow/providers/apache/spark/index.rst","_api/airflow/providers/apache/spark/operators/index.rst","_api/airflow/providers/apache/spark/operators/spark_jdbc/index.rst","_api/airflow/providers/apache/spark/operators/spark_sql/index.rst","_api/airflow/providers/apache/spark/operators/spark_submit/index.rst","connections/spark.rst","index.rst","operators.rst"],objects:{"airflow.providers.apache":{spark:[5,0,0,"-"]},"airflow.providers.apache.spark":{hooks:[0,0,0,"-"],operators:[6,0,0,"-"]},"airflow.providers.apache.spark.hooks":{spark_jdbc:[1,0,0,"-"],spark_jdbc_script:[2,0,0,"-"],spark_sql:[3,0,0,"-"],spark_submit:[4,0,0,"-"]},"airflow.providers.apache.spark.hooks.spark_jdbc":{SparkJDBCHook:[1,1,1,""]},"airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook":{_build_jdbc_application_arguments:[1,2,1,""],_resolve_jdbc_connection:[1,2,1,""],conn_name_attr:[1,3,1,""],conn_type:[1,3,1,""],default_conn_name:[1,3,1,""],get_conn:[1,2,1,""],hook_name:[1,3,1,""],submit_jdbc_job:[1,2,1,""]},"airflow.providers.apache.spark.hooks.spark_jdbc_script":{SPARK_READ_FROM_JDBC:[2,4,1,""],SPARK_WRITE_TO_JDBC:[2,4,1,""],_create_spark_session:[2,5,1,""],_parse_arguments:[2,5,1,""],_run_spark:[2,5,1,""],set_common_options:[2,5,1,""],spark_read_from_jdbc:[2,5,1,""],spark_write_to_jdbc:[2,5,1,""]},"airflow.providers.apache.spark.hooks.spark_sql":{SparkSqlHook:[3,1,1,""]},"airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook":{_prepare_command:[3,2,1,""],conn_name_attr:[3,3,1,""],conn_type:[3,3,1,""],default_conn_name:[3,3,1,""],get_conn:[3,2,1,""],hook_name:[3,3,1,""],kill:[3,2,1,""],run_query:[3,2,1,""]},"airflow.providers.apache.spark.hooks.spark_submit":{SparkSubmitHook:[4,1,1,""]},"airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook":{_build_spark_driver_kill_command:[4,2,1,""],_build_spark_submit_command:[4,2,1,""],_build_track_driver_status_command:[4,2,1,""],_get_spark_binary_path:[4,2,1,""],_mask_cmd:[4,2,1,""],_process_spark_status_log:[4,2,1,""],_process_spark_submit_log:[4,2,1,""],_resolve_connection:[4,2,1,""],_resolve_should_track_driver_status:[4,2,1,""],_start_driver_status_tracking:[4,2,1,""],conn_name_attr:[4,3,1,""],conn_type:[4,3,1,""],default_conn_name:[4,3,1,""],get_conn:[4,2,1,""],get_ui_field_behaviour:[4,2,1,""],hook_name:[4,3,1,""],on_kill:[4,2,1,""],submit:[4,2,1,""]},"airflow.providers.apache.spark.operators":{spark_jdbc:[7,0,0,"-"],spark_sql:[8,0,0,"-"],spark_submit:[9,0,0,"-"]},"airflow.providers.apache.spark.operators.spark_jdbc":{SparkJDBCOperator:[7,1,1,""]},"airflow.providers.apache.spark.operators.spark_jdbc.SparkJDBCOperator":{_get_hook:[7,2,1,""],execute:[7,2,1,""],on_kill:[7,2,1,""]},"airflow.providers.apache.spark.operators.spark_sql":{SparkSqlOperator:[8,1,1,""]},"airflow.providers.apache.spark.operators.spark_sql.SparkSqlOperator":{_get_hook:[8,2,1,""],execute:[8,2,1,""],on_kill:[8,2,1,""],template_ext:[8,3,1,""],template_fields:[8,3,1,""]},"airflow.providers.apache.spark.operators.spark_submit":{SparkSubmitOperator:[9,1,1,""]},"airflow.providers.apache.spark.operators.spark_submit.SparkSubmitOperator":{_get_hook:[9,2,1,""],execute:[9,2,1,""],on_kill:[9,2,1,""],template_fields:[9,3,1,""],ui_color:[9,3,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","attribute","Python attribute"],"4":["py","data","Python data"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:attribute","4":"py:data","5":"py:function"},terms:{"1000":[1,7],"1000m":[1,3,4,7,8,9],"1024":[1,7],"5432":2,"case":10,"char":[1,7],"class":[1,3,4,7,8,9],"default":[1,2,3,4,7,8,9],"int":[1,2,3,4,7,8,9],"return":[3,4],"static":4,"true":[3,8],"while":[4,9],For:[4,7,8,9,12],Has:4,The:[1,3,4,7,8,9,10,12],_applic:9,_application_arg:9,_build_jdbc_application_argu:1,_build_spark_driver_kill_command:4,_build_spark_submit_command:4,_build_track_driver_status_command:4,_conf:9,_create_spark_sess:2,_driver_class_path:9,_driver_id:4,_env_var:9,_exclude_packag:9,_file:9,_get_hook:[7,8,9],_get_spark_binary_path:4,_jar:9,_keytab:9,_mask_cmd:4,_name:9,_packag:9,_parse_argu:2,_prepare_command:3,_princip:9,_process_spark_status_log:4,_process_spark_submit_log:4,_proxy_us:9,_py_fil:9,_resolve_connect:4,_resolve_jdbc_connect:1,_resolve_should_track_driver_statu:4,_run_spark:2,_sql:8,_start_driver_status_track:4,actual:3,add:[1,7],addit:[1,4,7,9],administr:[1,4,7,9],after:4,again:4,airflow:12,alia:4,all:[3,4,8,9,12],alloc:[1,4,7,9],also:[1,7,12],ani:[1,2,4,7,8,9],api:11,append:[1,3,4,7,12],applic:[4,9,10,12],application_arg:[4,9],arbitrari:[3,4,8,9],archiv:[4,9],arg:[1,2],argument:[2,3,4,9],around:[3,4,9],arrow:9,assum:[7,10],avail:[3,4,7,8,9],bar:12,base:[1,3,4,7,8,9,12],basehook:[3,4],baseoper:[8,9],batch:[1,7],batch_siz:[1,2,7],been:4,behaviour:4,being:[4,9],between:[4,9,10],bin:10,binari:[3,4,7,9,10],bool:[1,2,3,4,7,8,9],both:[1,7],bound:[1,7],build:10,call:[7,8,9],can:[1,4,7,9,10,12],cannot:[1,7],cap:[1,7],care:12,classpath:[1,4,7,9,12],cleanli:4,cli:12,client:[4,10],cluster:[4,9,10,12],cmd:3,cmd_type:[1,7,12],column:[1,7],comma:[4,9],command:[3,4,9,10,12],comment:[1,7],conf:[3,4,8,9],configur:[1,3,4,7,8,9,12],conn_id:[3,4,8,9],conn_name_attr:[1,3,4],conn_typ:[1,3,4],connect:[1,2,4,7,9,11,12],connection_cmd:4,connection_id:[3,4,8,9],construct:[3,4],contain:[1,3,4,7,8,9],context:[7,8,9],coordin:[4,9],core:[1,3,4,7,8,9],creat:[1,7],create_table_column_typ:[1,2,7],custom:4,dag:11,data:[1,2,7,12],databas:[1,7,12],dataframewrit:12,debug:[1,4,7,9],default_conn_nam:[1,3,4],definit:12,depend:[1,4,7,9,12],deploi:[4,10,12],determin:4,dict:[1,3,4,7,8,9],dictionari:10,differ:[1,7,12],directli:4,directori:[4,9],distro:[4,9,10],divid:10,document:12,don:10,driver:[1,2,4,7,9,10,12],driver_class_path:[4,9],driver_memori:[1,4,7,9],drop:[1,7],due:4,each:[4,9],effect:[1,7],egg:[1,4,7,9],either:9,enabl:[3,10],env_var:[4,9],environ:[4,9],error:4,etc:[1,7],exampl:[4,9,11,12],example_dag:12,example_spark_dag:12,exclud:[4,9],exclude_packag:[4,9],execut:[3,4,7,8,9,10],executor:[1,3,4,7,8,9],executor_cor:[1,3,4,7,8,9],executor_memori:[1,3,4,7,8,9],exit:4,extend:[1,7],extern:10,extra:[3,4,9,10],extract:4,fail:4,failur:4,fals:[1,4,7,9],fetch:[1,7],fetch_siz:[1,2,7],field:4,file:[1,3,4,7,8,9,12],finish:4,flag:[1,3,4,7,8,9],flow:[1,7],follow:10,foo:12,format:[1,3,7,8],from:[1,2,7,12],full:[1,3,4,7,8,9],further:12,get:[2,4,8,12],get_conn:[1,3,4],get_ui_field_behaviour:4,given:[4,9],guid:[7,8,9,11],has:4,here:2,hive:12,home:[9,10],hook:[5,9,10],hook_nam:[1,3,4],host:[3,8,10],how:[7,8,9],hql:[8,12],imperson:[4,9],includ:[4,9],info:4,inform:[1,4,7,8,9,12],initi:4,input:4,insert:[1,7],instead:[1,7],invalid:[4,9],iter:4,itr:4,its:[4,12],jar:[1,4,7,9,12],java:[4,9],java_class:[4,9],jdbc:[1,2,7,10,12],jdbc_conn:1,jdbc_conn_id:[1,7],jdbc_driver:[1,7,12],jdbc_tabl:[1,2,7,12],jdbc_to_spark:[1,2,7,12],jdbc_to_spark_job:12,jdbc_truncat:[1,7],job:[1,3,4,7,8,9],json:[10,12],k8s:[4,9],kerbero:[1,4,7,9],keytab:[1,3,4,7,8,9],kick:[4,9],kill:[3,4],known:4,kubernet:[4,10],kwarg:[1,3,4,7,8,9],launch:[3,4,8,9,12],least:4,level:4,list:[2,3,4,9],local:[3,8,10,12],localhost:2,log4j:4,log:4,logger:4,logging_mixin:4,loggingmixin:4,look:[7,8,9,12],lower:[1,7],lower_bound:[1,2,7],made:[1,7],mai:[4,9,10],main:[4,9,12],manag:[1,7,12],manual:4,master:[3,4,8,12],maven:[4,9],maximum:[1,7],memori:[1,3,4,7,8,9],meso:[3,4,8,9],metastor:[1,7,12],metastore_t:[1,2,7,12],miss:4,mode:[1,4,7,9,10,12],model:[8,9],more:[7,8,9],multipl:10,must:[1,7,12],name:[1,2,3,4,7,8,9,10],namespac:10,need:4,node:10,non:4,none:[1,2,3,4,7,8,9],num_executor:[1,3,4,7,8,9],num_partit:[1,2,7],number:[1,3,4,7,8,9],numer:[1,7],object:[4,9],off:[4,9],on_kil:[4,7,8,9],onli:[1,3,4,7,8,9],open:[1,7],oper:[1,5,10,11],option:[1,2,3,4,7,8,9,10],org:[4,12],otherwis:10,out:[4,10],output:[3,4],over:4,overwrit:[1,7,12],packag:[4,9],param:4,paramet:[1,2,3,4,7,8,9,10,12],parquet:[1,7],pars:4,partit:[1,7],partition_column:[1,2,7],pass:[1,3,4,7,8,9,10],password:2,path:[1,3,4,7,8,9,10,12],per:[1,3,4,7,8,9],perform:[1,7,12],place:[4,9],point:10,poll:[4,9],popen:[3,4],port:[3,8,10],possibl:[1,4,7,12],postgresql:12,present:10,previous:4,princip:[1,3,4,7,8,9],process:[4,9],prop:[3,8],properti:[1,3,4,7,8,9],provid:12,proxy_us:[4,9],py_fil:[4,9],python:[1,4,7,9,10,11,12],queri:[3,4,8,12],queue:[3,8,10],quota:10,ran:4,rang:[1,7],reader:2,recoveri:4,recreat:[1,7],refer:11,relaunch:4,remark:4,remot:[3,4,9],repositori:[4,9],request:4,requir:[3,4,9,10,12],resolv:[4,9],resourc:[2,10,11],restart:4,root:2,round:[1,7],run:[1,4,7,8,9,12],run_queri:3,same:[1,7],save:[1,7],save_format:[1,2,7,12],save_mod:[1,2,7,12],saveast:12,schedul:4,schema:[1,7],script:12,search:[4,9],second:[4,9],see:[3,4],select:12,self:[1,3,4,7,8,9],separ:[4,9],serial:[4,9],server:12,servic:12,set:[1,4,7,9,12],set_common_opt:2,should:[1,4,7],simultan:[1,7],size:[1,7],some:[4,9,10],sourc:[1,2,3,4,7,8,9,12],spark2:[4,9,10],spark_app_nam:[1,7],spark_binari:[4,9],spark_conf:[1,7],spark_conn_id:[1,7],spark_default:[1,4,9,10],spark_fil:[1,7],spark_hom:[4,12],spark_jar:[1,7,12],spark_jdbc:[0,5,6],spark_jdbc_script:[0,5],spark_py_fil:[1,7],spark_read_from_jdbc:2,spark_sess:2,spark_sourc:2,spark_sql:[0,5,6],spark_sql_default:[3,8,10],spark_submit:[0,1,5,6,7],spark_to_jdbc:[1,2,7,12],spark_to_jdbc_job:12,spark_write_to_jdbc:2,sparkjdbchook:1,sparkjdbcoper:7,sparksess:2,sparksqlhook:[3,8],sparksqloper:8,sparksubmithook:[1,4,7,9],sparksubmitoper:[7,9],specif:[1,4,7,9],specifi:[1,7,10],sql:[1,3,7,8,10,12],sql_job:12,src:12,standalon:[3,4,8,9],standard:10,start:4,statu:[4,9],status_poll_interv:[4,9],str:[1,2,3,4,7,8,9],string:[3,8],submit:[1,3,4,7,8,9,10,12],submit_jdbc_job:1,submit_job:12,subprocess:[3,4],subsequ:4,successfulli:4,supervis:4,suppli:[4,9],support:[4,9,10,12],syntax:[1,7],tabl:[1,2,7,12],tag:4,take:[1,7,8,9,12],task_id:12,templat:[8,9,12],template_ext:8,template_field:[8,9],temporarili:4,them:[4,9],thi:[1,3,4,7,8,9],those:4,through:4,too:[4,9],total:[3,4,8,9],total_executor_cor:[3,4,8,9],track:4,transfer:[1,2,7,12],trip:[1,7],truncat:[1,2,7],type:[1,7,10,11],ui_color:9,unabl:4,union:[3,4],unknown:4,unrecover:4,unzip:4,upload:[1,4,7,9],upper:[1,7],upper_bound:[1,2,7],url:[2,10],use:[1,4,7,8,9,10,12],used:[1,4,7,9,10],useful:4,user:[2,4,9,10],uses:12,using:[10,12],usual:[1,7],util:4,valid:[1,7],valu:[1,3,7,8],varchar:[1,7],variabl:[4,9],verbos:[1,3,4,7,8,9],via:10,view:12,wai:[1,7],wait:[4,9],when:[1,4,7,9],whether:[1,3,4,7,8,9,10],which:[1,4,7,10,12],work:[4,9],worker:[3,4,8,9,10],wrapper:[3,4,9],write:12,writer:2,written:[1,7],yarn:[3,4,8,9,10],yarn_queu:[3,8],yet:4,you:[1,7,12],your:10,zero:4,zip:[1,4,7,9]},titles:["<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_jdbc</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_jdbc_script</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_sql</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_submit</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_jdbc</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_sql</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_submit</span></code>","Apache Spark Connection","<code class=\"docutils literal notranslate\"><span class=\"pre\">apache-airflow-providers-apache-spark</span></code>","Apache Spark Operators"],titleterms:{"default":10,IDs:10,Using:12,airflow:[0,1,2,3,4,5,6,7,8,9,11],apach:[0,1,2,3,4,5,6,7,8,9,10,11,12],configur:10,connect:10,content:[1,2,3,4,7,8,9,11],hook:[0,1,2,3,4],modul:[1,2,3,4,7,8,9],oper:[6,7,8,9,12],prerequisit:12,provid:[0,1,2,3,4,5,6,7,8,9,11],refer:12,spark:[0,1,2,3,4,5,6,7,8,9,10,11,12],spark_jdbc:[1,7],spark_jdbc_script:2,spark_sql:[3,8],spark_submit:[4,9],sparkjdbcoper:12,sparksqloper:12,sparksubmitoper:12,submodul:[0,6],subpackag:5}}) |