| {"version":3,"sources":["webpack:///./src/pages/docs/installation/async_queries_celery.mdx"],"names":["_frontmatter","MDXLayout","DefaultLayout","MDXContent","components","props","mdxType","isMDXComponent"],"mappings":"wPAMaA,G,UAAe,S,sOAE5B,IAKMC,EAAYC,IACH,SAASC,EAAT,GAGZ,IAFDC,EAEC,EAFDA,WACGC,EACF,8BACD,OAAO,YAACJ,EAAD,eAAeI,EAAf,CAAsBD,WAAYA,EAAYE,QAAQ,cAC3D,iBAAQ,CACN,GAAM,4BADR,4BAGA,iBAAQ,CACN,GAAM,UADR,UAGA,4TAGA,sBAEE,iBAAQ,CACN,WAAc,MADhB,wGAIE,yBAAgB,CACd,WAAc,MADhB,iBAJF,iBAQE,yBAAgB,CACd,WAAc,MADhB,wBARF,iCAcA,iBAAQ,CACN,WAAc,MADhB,kFAKA,iBAAQ,CACN,WAAc,MADhB,mFAKF,+DACE,yBAAgB,CACd,WAAc,KADhB,iBADF,YAKE,yBAAgB,CACd,WAAc,KADhB,sBALF,mFAUA,uBAAK,mBAAU,CACX,UAAa,kBACb,WAAc,OAFb,21BAiCL,2GACA,uBAAK,mBAAU,CACX,WAAc,OADb,oFAIL,4GACA,uBAAK,mBAAU,CACX,WAAc,OADb,sDAIL,yOAGE,gBAAO,CACL,KAAQ,+CACR,WAAc,KAFhB,gDAHF,+HAUE,yBAAgB,CACd,WAAc,KADhB,sBAVF,6BAcA,uBAAK,mBAAU,CACX,UAAa,kBACb,WAAc,OAFb,sVAcL,+CACE,gBAAO,CACL,KAAQ,4CACR,WAAc,KAFhB,eADF,SAOE,gBAAO,CACL,KAAQ,wCACR,WAAc,KAFhB,WAPF,6EAaE,yBAAgB,CACd,WAAc,KADhB,uCAbF,YAiBE,yBAAgB,CACd,WAAc,KADhB,sBAjBF,sHAsBA,qBAAG,qBAAY,CACX,WAAc,KADf,oBAGH,sBAEE,iBAAQ,CACN,WAAc,MAGd,gBAAO,CACL,WAAc,MADhB,qFAGE,iBAAQ,CACN,WAAc,KADhB,qCAHF,uJAaF,iBAAQ,CACN,WAAc,MAGd,gBAAO,CACL,WAAc,MADhB,gBAGE,iBAAQ,CACN,WAAc,KADhB,+CAHF,gMAaF,iBAAQ,CACN,WAAc,MAGd,gBAAO,CACL,WAAc,MADhB,gBAGE,iBAAQ,CACN,WAAc,KADhB,2CAHF,eAOE,qBAAY,CACV,WAAc,KADhB,gCAPF,sEAgBJ,iBAAQ,CACN,GAAM,iBADR,iBAGA,qHACA,uBAAK,mBAAU,CACX,UAAa,kBACb,WAAc,OAFb,yBAKL,kDACA,uBAAK,mBAAU,CACX,WAAc,OADb,8D,oOAQTH,EAAWI,gBAAiB","file":"component---src-pages-docs-installation-async-queries-celery-mdx-42f2f7b46e8dbf32f887.js","sourcesContent":["import * as React from 'react'\n /* @jsx mdx */\nimport { mdx } from '@mdx-js/react';\n/* @jsx mdx */\n\nimport DefaultLayout from \"/Users/evan/GitHub/incubator-superset_preset/docs/node_modules/gatsby-theme-docz/src/base/Layout.js\";\nexport const _frontmatter = {};\n\nconst makeShortcode = name => function MDXDefaultShortcode(props) {\n console.warn(\"Component \" + name + \" was not imported, exported, or provided by MDXProvider as global scope\");\n return <div {...props} />;\n};\n\nconst MDXLayout = DefaultLayout;\nexport default function MDXContent({\n components,\n ...props\n}) {\n return <MDXLayout {...props} components={components} mdxType=\"MDXLayout\">\n <h2 {...{\n \"id\": \"async-queries-via-celery\"\n }}>{`Async Queries via Celery`}</h2>\n <h3 {...{\n \"id\": \"celery\"\n }}>{`Celery`}</h3>\n <p>{`On large analytic databases, it’s common to run queries that execute for minutes or hours. To enable\nsupport for long running queries that execute beyond the typical web request’s timeout (30-60\nseconds), it is necessary to configure an asynchronous backend for Superset which consists of:`}</p>\n <ul>\n\n <li {...{\n \"parentName\": \"ul\"\n }}>{`one or many Superset workers (which is implemented as a Celery worker), and can be started with\nthe `}\n <inlineCode {...{\n \"parentName\": \"li\"\n }}>{`celery worker`}</inlineCode>\n {` command, run `}\n <inlineCode {...{\n \"parentName\": \"li\"\n }}>{`celery worker --help`}</inlineCode>\n {` to view the related options.`}</li>\n\n\n <li {...{\n \"parentName\": \"ul\"\n }}>{`a celery broker (message queue) for which we recommend using Redis or RabbitMQ`}</li>\n\n\n <li {...{\n \"parentName\": \"ul\"\n }}>{`a results backend that defines where the worker will persist the query results`}</li>\n\n </ul>\n <p>{`Configuring Celery requires defining a `}\n <inlineCode {...{\n \"parentName\": \"p\"\n }}>{`CELERY_CONFIG`}</inlineCode>\n {` in your `}\n <inlineCode {...{\n \"parentName\": \"p\"\n }}>{`superset_config.py`}</inlineCode>\n {`. Both the worker\nand web server processes should have the same configuration.`}</p>\n <pre><code {...{\n \"className\": \"language-python\",\n \"parentName\": \"pre\"\n }}>{`class CeleryConfig(object):\n BROKER_URL = 'redis://localhost:6379/0'\n CELERY_IMPORTS = (\n 'superset.sql_lab',\n 'superset.tasks',\n )\n CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'\n CELERYD_LOG_LEVEL = 'DEBUG'\n CELERYD_PREFETCH_MULTIPLIER = 10\n CELERY_ACKS_LATE = True\n CELERY_ANNOTATIONS = {\n 'sql_lab.get_sql_results': {\n 'rate_limit': '100/s',\n },\n 'email_reports.send': {\n 'rate_limit': '1/s',\n 'time_limit': 120,\n 'soft_time_limit': 150,\n 'ignore_result': True,\n },\n }\n CELERYBEAT_SCHEDULE = {\n 'email_reports.schedule_hourly': {\n 'task': 'email_reports.schedule_hourly',\n 'schedule': crontab(minute=1, hour='*'),\n },\n }\n\nCELERY_CONFIG = CeleryConfig\n`}</code></pre>\n <p>{`To start a Celery worker to leverage the configuration, run the following command:`}</p>\n <pre><code {...{\n \"parentName\": \"pre\"\n }}>{`celery worker --app=superset.tasks.celery_app:app --pool=prefork -O fair -c 4\n`}</code></pre>\n <p>{`To start a job which schedules periodic background jobs, run the following command:`}</p>\n <pre><code {...{\n \"parentName\": \"pre\"\n }}>{`celery beat --app=superset.tasks.celery_app:app\n`}</code></pre>\n <p>{`To setup a result backend, you need to pass an instance of a derivative of from\ncachelib.base.BaseCache to the RESULTS_BACKEND configuration key in your superset_config.py. You can\nuse Memcached, Redis, S3 (`}\n <a {...{\n \"href\": \"https://pypi.python.org/pypi/s3werkzeugcache\",\n \"parentName\": \"p\"\n }}>{`https://pypi.python.org/pypi/s3werkzeugcache`}</a>\n {`), memory or the file system\n(in a single server-type setup or for testing), or to write your own caching interface. Your\n`}\n <inlineCode {...{\n \"parentName\": \"p\"\n }}>{`superset_config.py`}</inlineCode>\n {` may look something like:`}</p>\n <pre><code {...{\n \"className\": \"language-python\",\n \"parentName\": \"pre\"\n }}>{`# On S3\nfrom s3cache.s3cache import S3Cache\nS3_CACHE_BUCKET = 'foobar-superset'\nS3_CACHE_KEY_PREFIX = 'sql_lab_result'\nRESULTS_BACKEND = S3Cache(S3_CACHE_BUCKET, S3_CACHE_KEY_PREFIX)\n\n# On Redis\nfrom cachelib.redis import RedisCache\nRESULTS_BACKEND = RedisCache(\n host='localhost', port=6379, key_prefix='superset_results')\n`}</code></pre>\n <p>{`For performance gains, `}\n <a {...{\n \"href\": \"https://github.com/msgpack/msgpack-python\",\n \"parentName\": \"p\"\n }}>{`MessagePack`}</a>\n {` and\n`}\n <a {...{\n \"href\": \"https://arrow.apache.org/docs/python/\",\n \"parentName\": \"p\"\n }}>{`PyArrow`}</a>\n {` are now used for results serialization. This can be\ndisabled by setting `}\n <inlineCode {...{\n \"parentName\": \"p\"\n }}>{`RESULTS_BACKEND_USE_MSGPACK = False`}</inlineCode>\n {` in your `}\n <inlineCode {...{\n \"parentName\": \"p\"\n }}>{`superset_config.py`}</inlineCode>\n {`, should any\nissues arise. Please clear your existing results cache store when upgrading an existing environment.`}</p>\n <p><strong {...{\n \"parentName\": \"p\"\n }}>{`Important Notes`}</strong></p>\n <ul>\n\n <li {...{\n \"parentName\": \"ul\"\n }}>\n\n <p {...{\n \"parentName\": \"li\"\n }}>{`It is important that all the worker nodes and web servers in the Superset cluster `}\n <em {...{\n \"parentName\": \"p\"\n }}>{`share a common\nmetadata database`}</em>\n {`. This means that SQLite will not work in this context since it has limited\nsupport for concurrency and typically lives on the local file system.`}</p>\n\n </li>\n\n\n <li {...{\n \"parentName\": \"ul\"\n }}>\n\n <p {...{\n \"parentName\": \"li\"\n }}>{`There should `}\n <em {...{\n \"parentName\": \"p\"\n }}>{`only be one instance of celery beat running`}</em>\n {` in your entire setup. If not,\nbackground jobs can get scheduled multiple times resulting in weird behaviors like duplicate\ndelivery of reports, higher than expected load / traffic etc.`}</p>\n\n </li>\n\n\n <li {...{\n \"parentName\": \"ul\"\n }}>\n\n <p {...{\n \"parentName\": \"li\"\n }}>{`SQL Lab will `}\n <em {...{\n \"parentName\": \"p\"\n }}>{`only run your queries asynchronously if`}</em>\n {` you enable `}\n <strong {...{\n \"parentName\": \"p\"\n }}>{`Asynchronous Query Execution`}</strong>\n {`\nin your database settings (Sources > Databases > Edit record).`}</p>\n\n </li>\n\n </ul>\n <h3 {...{\n \"id\": \"celery-flower\"\n }}>{`Celery Flower`}</h3>\n <p>{`Flower is a web based tool for monitoring the Celery cluster which you can install from pip:`}</p>\n <pre><code {...{\n \"className\": \"language-python\",\n \"parentName\": \"pre\"\n }}>{`pip install flower\n`}</code></pre>\n <p>{`You can run flower using:`}</p>\n <pre><code {...{\n \"parentName\": \"pre\"\n }}>{`celery flower --app=superset.tasks.celery_app:app\n`}</code></pre>\n\n </MDXLayout>;\n}\n;\nMDXContent.isMDXComponent = true;\n "],"sourceRoot":""} |