Search.setIndex({docnames:["_api/airflow/providers/apache/spark/hooks/index","_api/airflow/providers/apache/spark/hooks/spark_jdbc/index","_api/airflow/providers/apache/spark/hooks/spark_jdbc_script/index","_api/airflow/providers/apache/spark/hooks/spark_sql/index","_api/airflow/providers/apache/spark/hooks/spark_submit/index","_api/airflow/providers/apache/spark/index","_api/airflow/providers/apache/spark/operators/index","_api/airflow/providers/apache/spark/operators/spark_jdbc/index","_api/airflow/providers/apache/spark/operators/spark_sql/index","_api/airflow/providers/apache/spark/operators/spark_submit/index","commits","connections/spark","index","operators"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":1,"sphinx.ext.intersphinx":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["_api/airflow/providers/apache/spark/hooks/index.rst","_api/airflow/providers/apache/spark/hooks/spark_jdbc/index.rst","_api/airflow/providers/apache/spark/hooks/spark_jdbc_script/index.rst","_api/airflow/providers/apache/spark/hooks/spark_sql/index.rst","_api/airflow/providers/apache/spark/hooks/spark_submit/index.rst","_api/airflow/providers/apache/spark/index.rst","_api/airflow/providers/apache/spark/operators/index.rst","_api/airflow/providers/apache/spark/operators/spark_jdbc/index.rst","_api/airflow/providers/apache/spark/operators/spark_sql/index.rst","_api/airflow/providers/apache/spark/operators/spark_submit/index.rst","commits.rst","connections/spark.rst","index.rst","operators.rst"],objects:{"airflow.providers.apache":{spark:[5,0,0,"-"]},"airflow.providers.apache.spark":{hooks:[0,0,0,"-"],operators:[6,0,0,"-"]},"airflow.providers.apache.spark.hooks":{spark_jdbc:[1,0,0,"-"],spark_jdbc_script:[2,0,0,"-"],spark_sql:[3,0,0,"-"],spark_submit:[4,0,0,"-"]},"airflow.providers.apache.spark.hooks.spark_jdbc":{SparkJDBCHook:[1,1,1,""]},"airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook":{conn_name_attr:[1,2,1,""],conn_type:[1,2,1,""],default_conn_name:[1,2,1,""],get_conn:[1,3,1,""],hook_name:[1,2,1,""],submit_jdbc_job:[1,3,1,""]},"airflow.providers.apache.spark.hooks.spark_jdbc_script":{SPARK_READ_FROM_JDBC:[2,4,1,""],SPARK_WRITE_TO_JDBC:[2,4,1,""],set_common_options:[2,5,1,""],spark_read_from_jdbc:[2,5,1,""],spark_write_to_jdbc:[2,5,1,""]},"airflow.providers.apache.spark.hooks.spark_sql":{SparkSqlHook:[3,1,1,""]},"airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook":{conn_name_attr:[3,2,1,""],conn_type:[3,2,1,""],default_conn_name:[3,2,1,""],get_conn:[3,3,1,""],hook_name:[3,2,1,""],kill:[3,3,1,""],run_query:[3,3,1,""]},"airflow.providers.apache.spark.hooks.spark_submit":{SparkSubmitHook:[4,1,1,""]},"airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook":{conn_name_attr:[4,2,1,""],conn_type:[4,2,1,""],default_conn_name:[4,2,1,""],get_conn:[4,3,1,""],get_ui_field_behaviour:[4,3,1,""],hook_name:[4,2,1,""],on_kill:[4,3,1,""],submit:[4,3,1,""]},"airflow.providers.apache.spark.operators":{spark_jdbc:[7,0,0,"-"],spark_sql:[8,0,0,"-"],spark_submit:[9,0,0,"-"]},"airflow.providers.apache.spark.operators.spark_jdbc":{SparkJDBCOperator:[7,1,1,""]},"airflow.providers.apache.spark.operators.spark_jdbc.SparkJDBCOperator":{execute:[7,3,1,""],on_kill:[7,3,1,""]},"airflow.providers.apache.spark.operators.spark_sql":{SparkSqlOperator:[8,1,1,""]},"airflow.providers.apache.spark.operators.spark_sql.SparkSqlOperator":{execute:[8,3,1,""],on_kill:[8,3,1,""],template_ext:[8,2,1,""],template_fields:[8,2,1,""]},"airflow.providers.apache.spark.operators.spark_submit":{SparkSubmitOperator:[9,1,1,""]},"airflow.providers.apache.spark.operators.spark_submit.SparkSubmitOperator":{execute:[9,3,1,""],on_kill:[9,3,1,""],template_fields:[9,2,1,""],ui_color:[9,2,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","attribute","Python attribute"],"3":["py","method","Python method"],"4":["py","data","Python data"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:attribute","3":"py:method","4":"py:data","5":"py:function"},terms:{"00642a46d":10,"008035450":10,"024b4bf96":10,"0481b9a95":10,"067806d59":10,"0873070e0":10,"0a0e1af80":10,"0b0e4f7a4":10,"0b2":10,"0beta1":10,"1000":[1,7],"1000m":[1,3,4,7,8,9],"10170":10,"1024":[1,7],"10468":10,"10483":10,"10528":10,"10530":10,"10533":10,"10543":10,"11032":10,"11238":10,"11242":10,"11249":10,"11447":10,"11448":10,"11487":10,"11688":10,"11826":10,"11855":10,"12082":10,"12206":10,"12212":10,"12304":10,"12366":10,"12390":10,"12444":10,"12449":10,"12558":10,"12681":10,"12917":10,"12955":10,"12af6a080":10,"12c5e5d8a":10,"13073":10,"13148":10,"13380":10,"13732":10,"13767":10,"13a827d80":10,"14013":10,"14187":[10,12],"1427e4acb":10,"14487":10,"14823":[10,12],"14886":10,"15304":10,"15393":10,"15534":10,"15576":10,"15667":[10,12],"15794":[10,12],"16294":10,"16405":10,"16464":10,"16e712971":10,"1fba5402b":10,"2020":10,"2021":10,"2327aa5a2":10,"23rc1":10,"295d66f91":10,"2a1":10,"32971a1a2":10,"33f0cd265":10,"349b0811c":10,"3696c34c2":10,"375d1ca22":10,"37681bca0":10,"3b3287d7a":10,"40bf8f28f":10,"487b5cc50":10,"4b031d39e":10,"4bde99f13":10,"4d74ac211":10,"4e8f9cc8d":10,"5090fb0c8":10,"5220e4c38":10,"5432":2,"589d6dec9":10,"59eb5de78":10,"5a439e84":10,"5b2fe0e74":10,"5c86e3d50":10,"6515":10,"6539":10,"657384615":10,"6714":10,"6889a333c":10,"68e4c4dcb":10,"6949":10,"6cf76d7ac":10,"7024":10,"7025":10,"7026":10,"7142":10,"7338":10,"7506c73f1":10,"7575":10,"7676":10,"7677":10,"7749":10,"7802":10,"7820":10,"7825e8f59":10,"7c206a82a":10,"7d24b088c":10,"7e6372a68":10,"807ad32c":10,"8170":10,"8305":10,"85a18e13d":10,"872b1566a":10,"8730":10,"8787":10,"87969a350":10,"8807":10,"8886":10,"8891":10,"8898":10,"88bdcfa0d":10,"8994":10,"8c42cf1b0":10,"9015beb31":10,"9026":10,"9044":10,"92585ca4c":10,"9290":10,"9320":10,"9370":10,"9404":10,"9408":10,"9491":10,"9550":10,"9615":10,"9729":10,"9784":10,"97a429f9d":10,"9985":10,"9b39f2478":10,"9c94b72d4":10,"case":11,"char":[1,7],"class":[1,3,4,7,8,9,10,12],"default":[1,2,3,4,7,8,9,10],"export":11,"final":10,"function":10,"int":[1,2,3,4,7,8,9],"new":10,"return":4,"static":4,"super":10,"switch":10,"true":[3,8],"while":[4,9],Added:10,For:[4,7,8,9,10,11,13],The:[1,3,4,7,8,9,11,13],Use:[10,12],_applic:9,_application_arg:9,_conf:9,_driver_class_path:9,_env_var:9,_exclude_packag:9,_file:9,_jar:9,_keytab:9,_name:9,_packag:9,_princip:9,_proxy_us:9,_py_fil:9,_sql:8,a9ac2b040:10,about:10,ac2f72c98:10,actual:3,add:[1,7,10],addit:[1,4,7,9],administr:[1,4,7,9],ae7cb4a1:10,after:10,aip:10,airflow:13,airflow_conn_spark_default:11,alia:4,all:[3,4,8,9,11,12,13],alloc:[1,4,7,9],allow:10,also:[1,7,13],ani:[1,2,4,7,8,9],api:12,append:[1,3,7,13],appli:[10,12],applic:[4,9,11,13],application_arg:[4,9],apply_default:[10,12],arbitrari:[3,4,8,9],archiv:[4,9],arg:1,argument:[3,4,9,10],around:[3,4,9],arrow:9,assig:10,assign:10,assum:[7,11],augment:10,auto:[10,12],autom:10,automat:[10,12],avail:[3,4,7,8,9],b2a28d159:10,b40dffa08:10,b59042b5a:10,backport:10,bar:13,base:[1,3,4,7,8,9,13],basehook:[3,4],baseoper:[8,9],batch:[1,7,10],batch_siz:[1,2,7],bd90136aa:10,be1451b0:10,befor:10,behaviour:4,being:[4,9],between:[4,9,11],bin:11,binari:[3,4,7,9,11],black:10,bool:[1,2,3,4,7,8,9],both:[1,7],bound:[1,7],broken:10,buggfix:10,build:[10,11],c34ef853c:10,c7e5bce57:10,ca4238eb4:10,call:[7,8,9,10],can:[1,4,7,9,11,12,13],candid:10,cannot:[1,7],cap:[1,7],capit:10,care:13,cbf8001d7:10,chang:10,changelog:10,check:10,classpath:[1,4,7,9,13],cleanup:10,cli:13,client:11,cluster:[4,9,11,13],cmd:3,cmd_type:[1,7,13],code:10,column:[1,7],com:11,come:10,comma:[4,9],command:[3,4,9,10,11,13],comment:[1,7,10],commit:[10,12],compat:10,complet:12,compon:11,conf:[3,4,8,9,10],configur:[1,3,4,7,8,9,13],conn_id:[3,4,8,9],conn_name_attr:[1,3,4],conn_typ:[1,3,4],connect:[1,2,3,4,7,8,9,10,12,13],connection_id:[3,4,8,9],contain:[1,3,4,7,8,9],context:[7,8,9],coordin:[4,9],core:[1,3,4,7,8,9],creat:[1,7],create_table_column_typ:[1,2,7],cross:10,custom:4,d0e7db402:10,d1bce91bb:10,d200:10,d202:10,d305876be:10,d403:10,d76026545:10,dag:12,data:[1,2,7,13],databas:[1,7,13],dataframewrit:13,debug:[1,4,7,9],decor:[10,12],default_arg:10,default_conn_nam:[1,3,4],definit:13,depend:[1,4,7,9,10,13],deploi:[11,13],deploy_mod:11,descript:10,detail:[10,12],detect:10,dev:10,dict:[1,3,4,7,8,9,10],dictionari:11,differ:[1,7,13],directori:[4,9],distro:[4,9,11],divid:11,doc:10,docstr:10,document:[10,12,13],driver:[1,2,4,7,9,11,13],driver_class_path:[4,9],driver_memori:[1,4,7,9],drop:[1,7],due:12,dynam:10,each:[4,9],ee7ca128a:10,effect:[1,7],egg:[1,4,7,9],either:9,enabl:[10,11],encod:11,enforc:10,ensur:10,env_var:[4,9],environ:[4,9,10,11],error:10,etc:[1,7],exampl:[4,9,11,12,13],example_dag:[10,13],example_spark_dag:13,except:[10,12],exclud:[4,9],exclude_packag:[4,9],execut:[3,4,7,8,9,11],executor:[1,3,4,7,8,9],executor_cor:[1,3,4,7,8,9],executor_memori:[1,3,4,7,8,9],exist:12,explicit:10,extend:[1,7],extern:11,extra:[3,4,9,11],f3521fb0e:10,f3e87c503:10,f6bd817a3:10,f9c9e9c38:10,fals:[1,4,7,9],fdd9b6f65:10,featur:10,fetch:[1,7],fetch_siz:[1,2,7],field:[4,10],file:[1,3,4,7,8,9,10,12,13],first:[10,12],fix:10,flag:[1,3,4,7,8,9],flow:[1,7],flynt:10,follow:11,foo:13,form:10,format:[1,3,7,8],formmatt:10,fresh:10,from:[1,2,7,10,13],full:[1,3,4,7,8,9],further:13,gener:10,get:[2,10,13],get_conn:[1,3,4],get_ui_field_behaviour:4,given:[4,9],grammar:10,guid:[7,8,9,10,12],handl:10,hash:10,have:12,here:2,high:10,hive:13,home:[9,11],hook:[5,9,10,11],hook_nam:[1,3,4],host:[3,8,11],how:[7,8,9],hql:[8,13],http:10,imperson:[4,9],implement:10,improv:10,includ:[4,9,10],increas:10,info:10,inform:[1,7,8,9,10,13],initi:12,insert:[1,7],instal:10,instead:[1,7],integr:10,introduc:10,invalid:[4,9],its:[10,13],jar:[1,4,7,9,13],java:[4,9],java_class:[4,9],jdbc:[1,2,7,10,11,13],jdbc_conn_id:[1,7],jdbc_driver:[1,7,13],jdbc_tabl:[1,2,7,13],jdbc_to_spark:[1,2,7,13],jdbc_to_spark_job:13,jdbc_truncat:[1,7],job:[1,3,4,7,8,9],json:[10,11,13],june:10,k8s:[4,9],keep:10,kerbero:[1,4,7,9,10],keytab:[1,3,4,7,8,9],keyword:10,kick:[4,9],kill:[3,4,10],kube:11,kubernet:[4,10,11,12],kwarg:[1,3,4,7,8,9],lack:10,latest:10,launch:[3,4,8,9,13],least:12,level:10,link:10,list:[3,4,9,10,12],local:[3,8,11,13],localhost:2,log:[4,10,12],logging_mixin:4,loggingmixin:4,look:[7,8,9,13],lower:[1,7],lower_bound:[1,2,7],made:[1,7],magic:10,mai:[4,9,11],main:[4,9,13],make:[10,12],manag:[1,7,13],manual:12,markdown:10,mask:10,master:[3,8,13],match:10,maven:[4,9],maximum:[1,7],memori:[1,3,4,7,8,9],meso:[3,4,8,9],messag:10,metastor:[1,7,13],metastore_t:[1,2,7,13],migrat:12,mode:[1,4,7,9,13],model:[8,9],modul:10,mongo:11,month:10,more:[7,8,9,10],move:10,multipl:11,must:[1,7,13],mypi:10,mysparkclust:11,name:[1,2,3,4,7,8,9,10,11],namespac:[10,11],next:10,node:11,none:[1,2,3,4,7,8,9,10],note:[10,11],num_executor:[1,3,4,7,8,9],num_partit:[1,2,7],number:[1,3,4,7,8,9,10],numer:[1,7],object:[4,9],octob:10,off:[4,9],on_kil:[4,7,8,9],onli:[1,3,4,7,8,9,10],open:[1,7],oper:[1,5,10,11,12],option:[1,3,4,7,8,9,11],org:13,other:10,otherwis:[11,12],out:[10,11],overwrit:[1,7,13],packag:[4,9],page:10,param:4,paramet:[1,2,3,4,7,8,9,10,11,13],parquet:[1,7],partit:[1,7],partition_column:[1,2,7],pass:[1,3,4,7,8,9,10,11],password:[2,10],path:[1,3,4,7,8,9,11,13],per:[1,3,4,7,8,9,10],perform:[1,7,13],pip:10,place:[4,9],point:[10,11],poll:[4,9],popen:[3,4],popular:10,port:[3,8,11],possibl:[1,4,7,13],postgresql:13,prepar:10,present:11,princip:[1,3,4,7,8,9],process:[4,9,10],project:10,prop:[3,8],properli:10,properti:[1,3,4,7,8,9],provid:13,proxy_us:[4,9],py_fil:[4,9],pydocstyl:10,pylint:10,pypi:[10,12],pyspark:12,python:[1,4,7,9,10,11,12,13],pyupgrad:10,queri:[3,8,13],queue:[3,8,11],quota:11,rang:[1,7],rc3:10,reader:2,readm:[10,12],recreat:[1,7],redund:[10,12],ref:10,refer:[10,12],refernc:10,regener:10,relas:10,releas:[10,12],rema:10,remain:10,remot:[3,4,9],remov:[10,12],renam:10,replac:10,repositori:[4,9,12],requir:[3,4,9,11,13],resolv:[4,9],resourc:[2,11,12],respect:10,root:2,round:[1,7],run:[1,4,7,8,9,10,12,13],run_queri:[3,10],same:[1,7,11],save:[1,7],save_format:[1,2,7,13],save_mod:[1,2,7,13],saveast:13,schema:[1,7],script:[10,13],search:[4,9],second:[4,9],see:[3,4,10],select:13,self:[1,3,4,7,8,9],separ:[4,9,10],serial:[4,9],server:13,servic:13,set:[1,3,4,7,8,9,13],set_common_opt:2,setup:10,should:[1,4,7,11],signatur:10,simultan:[1,7],size:[1,7],some:[4,9,11],sourc:[1,2,3,4,7,8,9,13],spark2:[4,9,11],spark_app_nam:[1,7],spark_binari:[4,9,11],spark_conf:[1,7],spark_conn_id:[1,4,7,9],spark_default:[1,4,9,11],spark_fil:[1,7],spark_hom:[4,11,13],spark_jar:[1,7,13],spark_jdbc:[0,5,6],spark_jdbc_script:[0,5,10],spark_py_fil:[1,7],spark_read_from_jdbc:2,spark_sess:2,spark_sourc:2,spark_sql:[0,5,6],spark_sql_default:[3,8,11],spark_submit:[0,1,5,6,7],spark_to_jdbc:[1,2,7,13],spark_to_jdbc_job:13,spark_write_to_jdbc:2,sparkjdbchook:1,sparkjdbcoper:7,sparksess:2,sparksqlhook:[3,8,10,12],sparksqloper:[8,10],sparksubmithook:[1,4,7,9],sparksubmitoper:[7,9,10],specif:[1,4,7,9],specifi:[1,7,11],sql:[1,3,7,8,11,13],sql_job:13,src:13,standalon:[3,4,8,9],standard:11,start_dat:10,statu:[4,9],status_poll_interv:[4,9],stop:10,str:[1,2,3,4,7,8,9],string:[3,8,10,11],subject:10,submit:[1,3,4,7,8,9,10,11,13],submit_jdbc_job:1,submit_job:13,subprocess:[3,4],suppli:[4,9],support:[4,9,10,11,13],surround:10,synchron:10,syntax:[1,7,11],tabl:[1,2,7,13],tag:4,take:[1,7,8,9,13],task_id:13,templat:[8,9,13],template_ext:8,template_field:[8,9],test:10,text:10,them:[4,9],thi:[1,3,4,7,8,9,10,12],toc:10,token:10,too:[4,9],tool:10,top:12,total:[3,4,8,9],total_executor_cor:[3,4,8,9],transfer:[1,2,7,10,13],trip:[1,7],truncat:[1,2,7],type:[1,7,10,11,12],typo:10,ui_color:9,unzip:4,updat:[10,12],upgrad:[10,12],upload:[1,4,7,9],upper:[1,7],upper_bound:[1,2,7],uri:11,url:[2,11],use:[1,4,7,8,9,10,11,12,13],used:[1,4,7,9,11],user:[2,4,9,11],uses:13,using:[10,11,13],usual:[1,7],utf:10,util:4,valid:[1,7,10],valu:[1,3,7,8],varchar:[1,7],variabl:[4,9,10,11],verbos:[1,3,4,7,8,9,10],version:[10,12],via:[11,12],view:13,wai:[1,7],wait:[4,9],want:12,warn:10,wave:10,when:[1,4,7,9,11],whether:[1,3,4,7,8,9,11],which:[1,7,11,13],whitespac:10,without:[10,12],word:10,work:[4,9],worker:[3,4,8,9,11],wrapper:[3,4,9],write:13,writer:2,written:[1,7],wrong:10,wrongli:10,yarn:[3,4,8,9,10,11],yarn_queu:[3,8],you:[1,7,11,12,13],your:[11,12],zip:[1,4,7,9]},titles:["<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_jdbc</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_jdbc_script</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_sql</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.hooks.spark_submit</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_jdbc</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_sql</span></code>","<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.spark.operators.spark_submit</span></code>","Package apache-airflow-providers-apache-spark","Apache Spark Connection","<code class=\"docutils literal notranslate\"><span class=\"pre\">apache-airflow-providers-apache-spark</span></code>","Apache Spark Operators"],titleterms:{"break":12,"default":11,IDs:11,Using:13,airflow:[0,1,2,3,4,5,6,7,8,9,10,12],apach:[0,1,2,3,4,5,6,7,8,9,10,11,12,13],bug:12,chang:12,changelog:12,configur:11,connect:11,content:[1,2,3,4,7,8,9,12],fix:12,hook:[0,1,2,3,4],instal:12,modul:[1,2,3,4,7,8,9],oper:[6,7,8,9,13],packag:[10,12],pip:12,prerequisit:13,provid:[0,1,2,3,4,5,6,7,8,9,10,12],refer:13,requir:12,spark:[0,1,2,3,4,5,6,7,8,9,10,11,12,13],spark_jdbc:[1,7],spark_jdbc_script:2,spark_sql:[3,8],spark_submit:[4,9],sparkjdbcoper:13,sparksqloper:13,sparksubmitoper:13,submodul:[0,6],subpackag:5}})