| <!doctype html> |
| <html class="docs-version-current" lang="en" dir="ltr"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <meta name="generator" content="Docusaurus v2.0.0-beta.17"> |
| <link rel="preconnect" href="https://www.google-analytics.com"> |
| <script>window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)},ga.l=+new Date,ga("create","G-133LHD3B3N","auto"),ga("set","anonymizeIp",!0),ga("send","pageview")</script> |
| <script async src="https://www.google-analytics.com/analytics.js"></script> |
| <link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=G-133LHD3B3N"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-133LHD3B3N",{anonymize_ip:!0})</script> |
| <link rel="search" type="application/opensearchdescription+xml" title="Superset" href="/opensearch.xml"> |
| <script src="/script/matomo.js"></script><title data-rh="true">Async Queries via Celery | Superset</title><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://superset.apache.org/docs/installation/async-queries-celery"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Async Queries via Celery | Superset"><meta data-rh="true" name="description" content="Async Queries via Celery"><meta data-rh="true" property="og:description" content="Async Queries via Celery"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://superset.apache.org/docs/installation/async-queries-celery"><link data-rh="true" rel="alternate" href="https://superset.apache.org/docs/installation/async-queries-celery" hreflang="en"><link data-rh="true" rel="alternate" href="https://superset.apache.org/docs/installation/async-queries-celery" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://WR5FASX5ED-dsn.algolia.net" crossorigin="anonymous"><link rel="stylesheet" href="/assets/css/styles.ba1779ef.css"> |
| <link rel="preload" href="/assets/js/runtime~main.9b07dba6.js" as="script"> |
| <link rel="preload" href="/assets/js/main.f67bdcb9.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region"><a href="#" class="skipToContent_ZgBM">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle clean-btn" type="button" tabindex="0"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/superset-logo-horiz.svg" alt="Superset Logo" class="themedImage_W2Cr themedImage--light_TfLj"><img src="/img/superset-logo-horiz-dark.svg" alt="Superset Logo" class="themedImage_W2Cr themedImage--dark_oUvU"></div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" class="navbar__link">Documentation</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/intro">Getting Started</a></li><li><a class="dropdown__link" href="/docs/intro">Tutorial</a></li><li><a class="dropdown__link" href="/docs/frequently-asked-questions">FAQ</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community">Resources</a></li><li><a href="https://github.com/apache/superset" target="_blank" rel="noopener noreferrer" class="dropdown__link"><span>GitHub<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li><a href="https://join.slack.com/t/apache-superset/shared_invite/zt-16jvzmoi8-sI7jKWp~xc2zYRe~NqiY9Q" target="_blank" rel="noopener noreferrer" class="dropdown__link"><span>Slack<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li><a href="https://lists.apache.org/list.html?dev@superset.apache.org" target="_blank" rel="noopener noreferrer" class="dropdown__link"><span>Mailing List<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li><a href="https://stackoverflow.com/questions/tagged/superset+apache-superset" target="_blank" rel="noopener noreferrer" class="dropdown__link"><span>Stack Overflow<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li></ul></div></div><div class="navbar__items navbar__items--right"><a href="https://github.com/apache/superset" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link" class="github-logo-container"></a><div class="searchBox_qEbK"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div class="main-wrapper docs-wrapper docs-doc-page"><div class="docPage_P2Lg"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_RiI4" type="button"></button><aside class="theme-doc-sidebar-container docSidebarContainer_rKC_"><div class="sidebar_CW9Y"><nav class="menu thin-scrollbar menu_SkdO"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/intro">Introduction</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" href="/docs/installation/installing-superset-using-docker-compose">Installation and Configuration</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/installing-superset-using-docker-compose">Installing Locally Using Docker Compose</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/installing-superset-from-scratch">Installing From Scratch</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/configuring-superset">Configuring Superset</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/networking-settings">Additional Networking Settings</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/cache">Caching</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/event-logging">Event Logging</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/upgrading-superset">Upgrading Superset</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/installation/async-queries-celery">Async Queries via Celery</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/alerts-reports">Alerts and Reports</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/sql-templating">SQL Templating</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/installation/running-on-kubernetes">Running on Kubernetes</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/databases/installing-database-drivers">Connecting to Databases</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/creating-charts-dashboards/creating-your-first-dashboard">Creating Charts and Dashboards</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/miscellaneous/country-map-tools">Miscellaneous</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" href="/docs/contributing/contributing-page">Contributing</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/frequently-asked-questions">Frequently Asked Questions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/api">API</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/security">Security</a></li></ul></nav></div></aside><main class="docMainContainer_TCnq"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_DM6M"><div class="docItemContainer_vinB"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Xlws" aria-label="breadcrumbs"><ul class="breadcrumbs"><li class="breadcrumbs__item"><a class="breadcrumbs__link breadcrumbsItemLink_e5ie" href="/">🏠</a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link breadcrumbsItemLink_e5ie">Installation and Configuration</span></li><li class="breadcrumbs__item breadcrumbs__item--active"><a class="breadcrumbs__link breadcrumbsItemLink_e5ie" href="/docs/installation/async-queries-celery">Async Queries via Celery</a></li></ul></nav><div class="tocCollapsible_jdIR theme-doc-toc-mobile tocMobile_TmEX"><button type="button" class="clean-btn tocCollapsibleButton_Fzxq">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_mojV" id="async-queries-via-celery">Async Queries via Celery<a class="hash-link" href="#async-queries-via-celery" title="Direct link to heading"></a></h2><h3 class="anchor anchorWithStickyNavbar_mojV" id="celery">Celery<a class="hash-link" href="#celery" title="Direct link to heading"></a></h3><p>On large analytic databases, it’s common to run queries that execute for minutes or hours. To enable |
| support for long running queries that execute beyond the typical web request’s timeout (30-60 |
| seconds), it is necessary to configure an asynchronous backend for Superset which consists of:</p><ul><li>one or many Superset workers (which is implemented as a Celery worker), and can be started with |
| the <code>celery worker</code> command, run <code>celery worker --help</code> to view the related options.</li><li>a celery broker (message queue) for which we recommend using Redis or RabbitMQ</li><li>a results backend that defines where the worker will persist the query results</li></ul><p>Configuring Celery requires defining a <code>CELERY_CONFIG</code> in your <code>superset_config.py</code>. Both the worker |
| and web server processes should have the same configuration.</p><div class="codeBlockContainer_I0IT language-python theme-code-block"><div class="codeBlockContent_wNvx python"><pre tabindex="0" class="prism-code language-python codeBlock_jd64 thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#393A34"><span class="token keyword" style="color:#00009f">class</span><span class="token plain"> </span><span class="token class-name">CeleryConfig</span><span class="token punctuation" style="color:#393A34">(</span><span class="token builtin">object</span><span class="token punctuation" style="color:#393A34">)</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> broker_url </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'redis://localhost:6379/0'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> imports </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'superset.sql_lab'</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'superset.tasks'</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> result_backend </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'redis://localhost:6379/0'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> worker_log_level </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'DEBUG'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> worker_prefetch_multiplier </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> task_acks_late </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token boolean" style="color:#36acaa">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> task_annotations </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'sql_lab.get_sql_results'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'rate_limit'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'100/s'</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">}</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'email_reports.send'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'rate_limit'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'1/s'</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'time_limit'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">120</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'soft_time_limit'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">150</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'ignore_result'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token boolean" style="color:#36acaa">True</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">}</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> beat_schedule </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'email_reports.schedule_hourly'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'task'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'email_reports.schedule_hourly'</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token string" style="color:#e3116c">'schedule'</span><span class="token punctuation" style="color:#393A34">:</span><span class="token plain"> crontab</span><span class="token punctuation" style="color:#393A34">(</span><span class="token plain">minute</span><span class="token operator" style="color:#393A34">=</span><span class="token number" style="color:#36acaa">1</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"> hour</span><span class="token operator" style="color:#393A34">=</span><span class="token string" style="color:#e3116c">'*'</span><span class="token punctuation" style="color:#393A34">)</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">}</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">CELERY_CONFIG </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> CeleryConfig</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_wuS7 clean-btn">Copy</button></div></div><p>To start a Celery worker to leverage the configuration, run the following command:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#393A34"><span class="token plain">celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_wuS7 clean-btn">Copy</button></div></div><p>To start a job which schedules periodic background jobs, run the following command:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#393A34"><span class="token plain">celery --app=superset.tasks.celery_app:app beat</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_wuS7 clean-btn">Copy</button></div></div><p>To setup a result backend, you need to pass an instance of a derivative of from |
| cachelib.base.BaseCache to the RESULTS_BACKEND configuration key in your superset_config.py. You can |
| use Memcached, Redis, S3 (<a href="https://pypi.python.org/pypi/s3werkzeugcache" target="_blank" rel="noopener noreferrer">https://pypi.python.org/pypi/s3werkzeugcache</a>), memory or the file system |
| (in a single server-type setup or for testing), or to write your own caching interface. Your |
| <code>superset_config.py</code> may look something like:</p><div class="codeBlockContainer_I0IT language-python theme-code-block"><div class="codeBlockContent_wNvx python"><pre tabindex="0" class="prism-code language-python codeBlock_jd64 thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#393A34"><span class="token comment" style="color:#999988;font-style:italic"># On S3</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token keyword" style="color:#00009f">from</span><span class="token plain"> s3cache</span><span class="token punctuation" style="color:#393A34">.</span><span class="token plain">s3cache </span><span class="token keyword" style="color:#00009f">import</span><span class="token plain"> S3Cache</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">S3_CACHE_BUCKET </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'foobar-superset'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">S3_CACHE_KEY_PREFIX </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'sql_lab_result'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">RESULTS_BACKEND </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> S3Cache</span><span class="token punctuation" style="color:#393A34">(</span><span class="token plain">S3_CACHE_BUCKET</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"> S3_CACHE_KEY_PREFIX</span><span class="token punctuation" style="color:#393A34">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># On Redis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token keyword" style="color:#00009f">from</span><span class="token plain"> cachelib</span><span class="token punctuation" style="color:#393A34">.</span><span class="token plain">redis </span><span class="token keyword" style="color:#00009f">import</span><span class="token plain"> RedisCache</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">RESULTS_BACKEND </span><span class="token operator" style="color:#393A34">=</span><span class="token plain"> RedisCache</span><span class="token punctuation" style="color:#393A34">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> host</span><span class="token operator" style="color:#393A34">=</span><span class="token string" style="color:#e3116c">'localhost'</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"> port</span><span class="token operator" style="color:#393A34">=</span><span class="token number" style="color:#36acaa">6379</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"> key_prefix</span><span class="token operator" style="color:#393A34">=</span><span class="token string" style="color:#e3116c">'superset_results'</span><span class="token punctuation" style="color:#393A34">)</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_wuS7 clean-btn">Copy</button></div></div><p>For performance gains, <a href="https://github.com/msgpack/msgpack-python" target="_blank" rel="noopener noreferrer">MessagePack</a> and |
| <a href="https://arrow.apache.org/docs/python/" target="_blank" rel="noopener noreferrer">PyArrow</a> are now used for results serialization. This can be |
| disabled by setting <code>RESULTS_BACKEND_USE_MSGPACK = False</code> in your <code>superset_config.py</code>, should any |
| issues arise. Please clear your existing results cache store when upgrading an existing environment.</p><p><strong>Important Notes</strong></p><ul><li><p>It is important that all the worker nodes and web servers in the Superset cluster <em>share a common |
| metadata database</em>. This means that SQLite will not work in this context since it has limited |
| support for concurrency and typically lives on the local file system.</p></li><li><p>There should <em>only be one instance of celery beat running</em> in your entire setup. If not, |
| background jobs can get scheduled multiple times resulting in weird behaviors like duplicate |
| delivery of reports, higher than expected load / traffic etc.</p></li><li><p>SQL Lab will <em>only run your queries asynchronously if</em> you enable <strong>Asynchronous Query Execution</strong> |
| in your database settings (Sources > Databases > Edit record).</p></li></ul><h3 class="anchor anchorWithStickyNavbar_mojV" id="celery-flower">Celery Flower<a class="hash-link" href="#celery-flower" title="Direct link to heading"></a></h3><p>Flower is a web based tool for monitoring the Celery cluster which you can install from pip:</p><div class="codeBlockContainer_I0IT language-python theme-code-block"><div class="codeBlockContent_wNvx python"><pre tabindex="0" class="prism-code language-python codeBlock_jd64 thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#393A34"><span class="token plain">pip install flower</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_wuS7 clean-btn">Copy</button></div></div><p>You can run flower using:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#393A34"><span class="token plain">celery --app=superset.tasks.celery_app:app flower</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_wuS7 clean-btn">Copy</button></div></div></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/superset/tree/master/docs/docs/installation/async-queries-celery.mdx" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_dcUD" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_foO9"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/docs/installation/upgrading-superset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Upgrading Superset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/docs/installation/alerts-reports"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Alerts and Reports</div></a></div></nav></div></div><div class="col col--3"><div class="tableOfContents_cNA8 thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#async-queries-via-celery" class="table-of-contents__link toc-highlight">Async Queries via Celery</a><ul><li><a href="#celery" class="table-of-contents__link toc-highlight">Celery</a></li><li><a href="#celery-flower" class="table-of-contents__link toc-highlight">Celery Flower</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2022, |
| The <a href="https://www.apache.org/" target="_blank" rel="noreferrer">Apache Software Foundation</a>, |
| Licensed under the Apache <a href="https://apache.org/licenses/LICENSE-2.0" target="_blank" rel="noreferrer">License</a>. <br> |
| <small>Apache Superset, Apache, Superset, the Superset logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation. All other products or name brands are trademarks of their respective holders, including The Apache Software Foundation. |
| <a href="https://www.apache.org/" target="_blank">Apache Software Foundation</a> resources</small><br> |
| <small> |
| <a href="https://www.apache.org/security/" target="_blank" rel="noreferrer">Security</a> | |
| <a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noreferrer">Donate</a> | |
| <a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noreferrer">Thanks</a> | |
| <a href="https://apache.org/events/current-event" target="_blank" rel="noreferrer">Events</a> | |
| <a href="https://apache.org/licenses/" target="_blank" rel="noreferrer">License</a> |
| </small></div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.9b07dba6.js"></script> |
| <script src="/assets/js/main.f67bdcb9.js"></script> |
| </body> |
| </html> |
