| <!DOCTYPE html> | |
| <html lang="en" data-content_root="../../../" data-theme="auto"> | |
| <head> | |
| <meta charset="utf-8" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> | |
| <title>datafusion.plan — Apache DataFusion in Python documentation</title> | |
| <script data-cfasync="false"> | |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto"; | |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto"; | |
| </script> | |
| <!-- | |
| this give us a css class that will be invisible only if js is disabled | |
| --> | |
| <noscript> | |
| <style> | |
| .pst-js-only { display: none !important; } | |
| </style> | |
| </noscript> | |
| <!-- Loaded before other Sphinx assets --> | |
| <link href="../../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> | |
| <link href="../../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> | |
| <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" /> | |
| <link rel="stylesheet" type="text/css" href="../../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" /> | |
| <link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css?v=4ae1632d" /> | |
| <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css?v=76b2166b" /> | |
| <link rel="stylesheet" type="text/css" href="../../../_static/theme_overrides.css?v=4af573bc" /> | |
| <!-- So that users can add custom icons --> | |
| <script src="../../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script> | |
| <!-- Pre-loaded scripts that we'll load fully later --> | |
| <link rel="preload" as="script" href="../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" /> | |
| <link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" /> | |
| <script src="../../../_static/documentation_options.js?v=5929fcd5"></script> | |
| <script src="../../../_static/doctools.js?v=9bcbadda"></script> | |
| <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> | |
| <script src="../../../_static/clipboard.min.js?v=a7894cd8"></script> | |
| <script src="../../../_static/copybutton.js?v=f281be69"></script> | |
| <script>DOCUMENTATION_OPTIONS.pagename = 'autoapi/datafusion/plan/index';</script> | |
| <script src="../../../_static/toc-toggle.js?v=36375f01"></script> | |
| <link rel="icon" href="../../../_static/favicon.svg"/> | |
| <link rel="index" title="Index" href="../../../genindex.html" /> | |
| <link rel="search" title="Search" href="../../../search.html" /> | |
| <link rel="next" title="datafusion.record_batch" href="../record_batch/index.html" /> | |
| <link rel="prev" title="datafusion.options" href="../options/index.html" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> | |
| <meta name="docsearch:language" content="en"/> | |
| <meta name="docsearch:version" content="" /> | |
| </head> | |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto"> | |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> | |
| <div id="pst-scroll-pixel-helper"></div> | |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> | |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> | |
| <dialog id="pst-search-dialog"> | |
| <form class="bd-search d-flex align-items-center" | |
| action="../../../search.html" | |
| method="get"> | |
| <i class="fa-solid fa-magnifying-glass"></i> | |
| <input type="search" | |
| class="form-control" | |
| name="q" | |
| placeholder="Search the docs ..." | |
| aria-label="Search the docs ..." | |
| autocomplete="off" | |
| autocorrect="off" | |
| autocapitalize="off" | |
| spellcheck="false"/> | |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> | |
| </form> | |
| </dialog> | |
| <div class="pst-async-banner-revealer d-none"> | |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> | |
| </div> | |
| <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> | |
| <div class="bd-header__inner bd-page-width"> | |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> | |
| <span class="fa-solid fa-bars"></span> | |
| </button> | |
| <div class="col-lg-3 navbar-header-items__start"> | |
| <div class="navbar-item"> | |
| <a class="navbar-brand logo" href="../../../index.html"> | |
| <img src="../../../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/> | |
| <img src="../../../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/> | |
| </a></div> | |
| </div> | |
| <div class="col-lg-9 navbar-header-items"> | |
| <div class="me-auto navbar-header-items__center"> | |
| <div class="navbar-item"> | |
| <nav> | |
| <ul class="bd-navbar-elements navbar-nav"> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../user-guide/index.html"> | |
| User Guide | |
| </a> | |
| </li> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../extension-guide/index.html"> | |
| Extension Guide | |
| </a> | |
| </li> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../contributor-guide/index.html"> | |
| Contributor Guide | |
| </a> | |
| </li> | |
| <li class="nav-item current active"> | |
| <a class="nav-link nav-internal" href="../../index.html"> | |
| API Reference | |
| </a> | |
| </li> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../links.html"> | |
| Links | |
| </a> | |
| </li> | |
| </ul> | |
| </nav></div> | |
| </div> | |
| <div class="navbar-header-items__end"> | |
| <div class="navbar-item navbar-persistent--container"> | |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> | |
| <i class="fa-solid fa-magnifying-glass"></i> | |
| <span class="search-button__default-text">Search</span> | |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> | |
| </button> | |
| </div> | |
| <div class="navbar-item"><ul class="navbar-icon-links" | |
| aria-label="Icon Links"> | |
| <li class="nav-item"> | |
| <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i> | |
| <span class="sr-only">GitHub</span></a> | |
| </li> | |
| <li class="nav-item"> | |
| <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i> | |
| <span class="sr-only">Rust API docs (docs.rs)</span></a> | |
| </li> | |
| </ul></div> | |
| <div class="navbar-item"> | |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> | |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> | |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> | |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> | |
| </button></div> | |
| </div> | |
| </div> | |
| <div class="navbar-persistent--mobile"> | |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> | |
| <i class="fa-solid fa-magnifying-glass"></i> | |
| <span class="search-button__default-text">Search</span> | |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> | |
| </button> | |
| </div> | |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> | |
| <span class="fa-solid fa-outdent"></span> | |
| </button> | |
| </div> | |
| </header> | |
| <div class="bd-container"> | |
| <div class="bd-container__inner bd-page-width"> | |
| <dialog id="pst-primary-sidebar-modal"></dialog> | |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> | |
| <div class="sidebar-header-items sidebar-primary__section"> | |
| <div class="sidebar-header-items__center"> | |
| <div class="navbar-item"> | |
| <nav> | |
| <ul class="bd-navbar-elements navbar-nav"> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../user-guide/index.html"> | |
| User Guide | |
| </a> | |
| </li> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../extension-guide/index.html"> | |
| Extension Guide | |
| </a> | |
| </li> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../contributor-guide/index.html"> | |
| Contributor Guide | |
| </a> | |
| </li> | |
| <li class="nav-item current active"> | |
| <a class="nav-link nav-internal" href="../../index.html"> | |
| API Reference | |
| </a> | |
| </li> | |
| <li class="nav-item "> | |
| <a class="nav-link nav-internal" href="../../../links.html"> | |
| Links | |
| </a> | |
| </li> | |
| </ul> | |
| </nav></div> | |
| </div> | |
| <div class="sidebar-header-items__end"> | |
| <div class="navbar-item"><ul class="navbar-icon-links" | |
| aria-label="Icon Links"> | |
| <li class="nav-item"> | |
| <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i> | |
| <span class="sr-only">GitHub</span></a> | |
| </li> | |
| <li class="nav-item"> | |
| <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i> | |
| <span class="sr-only">Rust API docs (docs.rs)</span></a> | |
| </li> | |
| </ul></div> | |
| <div class="navbar-item"> | |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> | |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> | |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> | |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> | |
| </button></div> | |
| </div> | |
| </div> | |
| <div class="sidebar-primary-items__start sidebar-primary__section"> | |
| <div class="sidebar-primary-item"> | |
| <nav class="bd-docs-nav bd-links" aria-label="Section Navigation"> | |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> | |
| <div class="bd-toc-item navbar-nav"> | |
| <ul class="current nav bd-sidenav"> | |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/introduction.html">Introduction</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/basics.html">Concepts</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/data-sources.html">Data Sources</a></li> | |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/views.html">Registering Views</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/basic-info.html">Basic Operations</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/select-and-filter.html">Column Selections</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/expressions.html">Expressions</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/arrow.html">Arrow</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/avro.html">Avro</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/csv.html">CSV</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/json.html">JSON</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/parquet.html">Parquet</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/table_provider.html">Custom Table Provider</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/configuration.html">Configuration</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/extensions.html">Using extension libraries</a></li> | |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/distributing-work/expressions.html">Shipping expressions to workers</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/distributing-work/query-engines.html">Distributed query engines</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/sql.html">SQL</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/upgrade-guides.html">Upgrade Guides</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../extension-guide/index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/why-ffi.html">Why FFI</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/capsule-protocol.html">The capsule protocol</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/table-providers.html">Providers and catalogs</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/functions.html">Functions and table functions</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/codecs.html">Extension codecs</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/bundles.html">Extension bundles</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/query-planners.html">Query planners</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/other-components.html">Optimizer rules and configuration</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/sessions.html">Sessions, handles, and lifetimes</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/checklist.html">Extension author checklist</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/introduction.html">Introduction</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li> | |
| <li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/ffi-internals.html">FFI framing internals</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="../../index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> | |
| <li class="toctree-l2 current active has-children"><a class="reference internal" href="../index.html">datafusion</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> | |
| <li class="toctree-l3"><a class="reference internal" href="../catalog/index.html">datafusion.catalog</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../context/index.html">datafusion.context</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../extensions/index.html">datafusion.extensions</a></li> | |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li> | |
| <li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../io/index.html">datafusion.io</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../ipc/index.html">datafusion.ipc</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../object_store/index.html">datafusion.object_store</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../options/index.html">datafusion.options</a></li> | |
| <li class="toctree-l3 current active"><a class="current reference internal" href="#">datafusion.plan</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../record_batch/index.html">datafusion.record_batch</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../substrait/index.html">datafusion.substrait</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../unparser/index.html">datafusion.unparser</a></li> | |
| <li class="toctree-l3"><a class="reference internal" href="../user_defined/index.html">datafusion.user_defined</a></li> | |
| </ul> | |
| </details></li> | |
| </ul> | |
| </details></li> | |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> | |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li> | |
| <li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li> | |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li> | |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li> | |
| </ul> | |
| </details></li> | |
| </ul> | |
| </div> | |
| </nav></div> | |
| </div> | |
| <div class="sidebar-primary-items__end sidebar-primary__section"> | |
| <div class="sidebar-primary-item"> | |
| <div id="ethical-ad-placement" | |
| class="flat" | |
| data-ea-publisher="readthedocs" | |
| data-ea-type="readthedocs-sidebar" | |
| data-ea-manual="true"> | |
| </div></div> | |
| </div> | |
| </div> | |
| <main id="main-content" class="bd-main" role="main"> | |
| <div class="bd-content"> | |
| <div class="bd-article-container"> | |
| <div class="bd-header-article d-print-none"> | |
| <div class="header-article-items header-article__inner"> | |
| <div class="header-article-items__start"> | |
| <div class="header-article-item"> | |
| <nav aria-label="Breadcrumb" class="d-print-none"> | |
| <ul class="bd-breadcrumbs"> | |
| <li class="breadcrumb-item breadcrumb-home"> | |
| <a href="../../../index.html" class="nav-link" aria-label="Home"> | |
| <i class="fa-solid fa-home"></i> | |
| </a> | |
| </li> | |
| <li class="breadcrumb-item"><a href="../../index.html" class="nav-link">API Reference</a></li> | |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">datafusion</a></li> | |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">datafusion.plan</span></li> | |
| </ul> | |
| </nav> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div id="searchbox"></div> | |
| <article class="bd-article"> | |
| <section id="module-datafusion.plan"> | |
| <span id="datafusion-plan"></span><h1>datafusion.plan<a class="headerlink" href="#module-datafusion.plan" title="Link to this heading">#</a></h1> | |
| <p>This module supports physical and logical plans in DataFusion.</p> | |
| <section id="classes"> | |
| <h2>Classes<a class="headerlink" href="#classes" title="Link to this heading">#</a></h2> | |
| <div class="pst-scrollable-table-container"><table class="autosummary longtable table autosummary"> | |
| <tbody> | |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ExecutionPlan</span></code></a></p></td> | |
| <td><p>Represent nodes in the DataFusion Physical Plan.</p></td> | |
| </tr> | |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><code class="xref py py-obj docutils literal notranslate"><span class="pre">LogicalPlan</span></code></a></p></td> | |
| <td><p>Logical Plan.</p></td> | |
| </tr> | |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.plan.Metric" title="datafusion.plan.Metric"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Metric</span></code></a></p></td> | |
| <td><p>A single execution metric with name, value, partition, and labels.</p></td> | |
| </tr> | |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.plan.MetricsSet" title="datafusion.plan.MetricsSet"><code class="xref py py-obj docutils literal notranslate"><span class="pre">MetricsSet</span></code></a></p></td> | |
| <td><p>A set of metrics for a single execution plan operator.</p></td> | |
| </tr> | |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.plan.PhysicalPartitioning" title="datafusion.plan.PhysicalPartitioning"><code class="xref py py-obj docutils literal notranslate"><span class="pre">PhysicalPartitioning</span></code></a></p></td> | |
| <td><p>How a physical plan's output rows are spread across its partitions.</p></td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| </section> | |
| <section id="module-contents"> | |
| <h2>Module Contents<a class="headerlink" href="#module-contents" title="Link to this heading">#</a></h2> | |
| <dl class="py class"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan"> | |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.plan.</span></span><span class="sig-name descname"><span class="pre">ExecutionPlan</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">plan</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.ExecutionPlan</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.plan.ExecutionPlan" title="Link to this definition">#</a></dt> | |
| <dd><p>Represent nodes in the DataFusion Physical Plan.</p> | |
| <p>This constructor should not be called by the end user.</p> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.__repr__"> | |
| <span class="sig-name descname"><span class="pre">__repr__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.__repr__" title="Link to this definition">#</a></dt> | |
| <dd><p>Print a string representation of the physical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.children"> | |
| <span class="sig-name descname"><span class="pre">children</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">ExecutionPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.children" title="Link to this definition">#</a></dt> | |
| <dd><p>Get a list of children <code class="docutils literal notranslate"><span class="pre">ExecutionPlan</span></code> that act as inputs to this plan.</p> | |
| <p>The returned list will be empty for leaf nodes such as scans, will contain a | |
| single value for unary nodes, or two values for binary nodes (such as joins).</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.collect_metrics"> | |
| <span class="sig-name descname"><span class="pre">collect_metrics</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.plan.MetricsSet" title="datafusion.plan.MetricsSet"><span class="pre">MetricsSet</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.collect_metrics" title="Link to this definition">#</a></dt> | |
| <dd><p>Return runtime statistics for each step of the query execution.</p> | |
| <p>DataFusion executes a query as a pipeline of operators — for example a | |
| data source scan, followed by a filter, followed by a projection. After | |
| the DataFrame has been executed (via | |
| <code class="xref py py-meth docutils literal notranslate"><span class="pre">collect()</span></code>, | |
| <code class="xref py py-meth docutils literal notranslate"><span class="pre">execute_stream()</span></code>, etc.), each operator | |
| records statistics such as how many rows it produced and how much CPU | |
| time it consumed.</p> | |
| <p>Each entry in the returned list corresponds to one operator that | |
| recorded metrics. The first element of the tuple is the operator’s | |
| description string — the same text shown by | |
| <a class="reference internal" href="#datafusion.plan.ExecutionPlan.display_indent" title="datafusion.plan.ExecutionPlan.display_indent"><code class="xref py py-meth docutils literal notranslate"><span class="pre">display_indent()</span></code></a> — which identifies both the operator type | |
| and its key parameters, for example <code class="docutils literal notranslate"><span class="pre">"FilterExec:</span> <span class="pre">column1@0</span> <span class="pre">></span> <span class="pre">1"</span></code> | |
| or <code class="docutils literal notranslate"><span class="pre">"DataSourceExec:</span> <span class="pre">partitions=1"</span></code>.</p> | |
| <dl class="field-list simple"> | |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> | |
| <dd class="field-odd"><p>A list of <code class="docutils literal notranslate"><span class="pre">(description,</span> <span class="pre">MetricsSet)</span></code> tuples ordered from the | |
| outermost operator (top of the execution tree) down to the | |
| data-source leaves. Only operators that recorded at least one | |
| metric are included. Returns an empty list if called before the | |
| DataFrame has been executed.</p> | |
| </dd> | |
| </dl> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.display"> | |
| <span class="sig-name descname"><span class="pre">display</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.display" title="Link to this definition">#</a></dt> | |
| <dd><p>Print the physical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.display_indent"> | |
| <span class="sig-name descname"><span class="pre">display_indent</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.display_indent" title="Link to this definition">#</a></dt> | |
| <dd><p>Print an indented form of the physical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.from_bytes"> | |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_bytes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">datafusion.context.SessionContext</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bytes</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">ExecutionPlan</span></a></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.from_bytes" title="Link to this definition">#</a></dt> | |
| <dd><p>Create an ExecutionPlan from serialized protobuf bytes.</p> | |
| <p>Decoding routes through the codecs installed on <code class="docutils literal notranslate"><span class="pre">ctx</span></code> with | |
| <code class="xref py py-meth docutils literal notranslate"><span class="pre">with_physical_extension_codec()</span></code>.</p> | |
| <p>Unlike <a class="reference internal" href="../index.html#datafusion.Expr.from_bytes" title="datafusion.Expr.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.Expr.from_bytes()</span></code></a>, <code class="docutils literal notranslate"><span class="pre">ctx</span></code> is required and | |
| positional, and there is no fallback to a worker or global context. | |
| <code class="docutils literal notranslate"><span class="pre">ctx</span></code> need share nothing with the session that encoded the plan: a | |
| scan over a table registered from record batches decodes here, because | |
| the batches travel inside the encoded scan.</p> | |
| <div class="admonition seealso"> | |
| <p class="admonition-title">See also</p> | |
| <p><a class="reference internal" href="#datafusion.plan.ExecutionPlan.to_bytes" title="datafusion.plan.ExecutionPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">to_bytes()</span></code></a>, <a class="reference internal" href="#datafusion.plan.LogicalPlan.from_bytes" title="datafusion.plan.LogicalPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">LogicalPlan.from_bytes()</span></code></a>.</p> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.from_proto"> | |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_proto</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">datafusion.context.SessionContext</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bytes</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">ExecutionPlan</span></a></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.from_proto" title="Link to this definition">#</a></dt> | |
| <dd><p>Deprecated alias for <a class="reference internal" href="#datafusion.plan.ExecutionPlan.from_bytes" title="datafusion.plan.ExecutionPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a>.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.metrics"> | |
| <span class="sig-name descname"><span class="pre">metrics</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.plan.MetricsSet" title="datafusion.plan.MetricsSet"><span class="pre">MetricsSet</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.metrics" title="Link to this definition">#</a></dt> | |
| <dd><p>Return metrics for this plan node, or None if this plan has no MetricsSet.</p> | |
| <p>Some operators (e.g. DataSourceExec) eagerly initialize a MetricsSet | |
| when the plan is created, so this may return a set even before | |
| execution. Metric <em>values</em> (such as <code class="docutils literal notranslate"><span class="pre">output_rows</span></code>) are only | |
| meaningful after the DataFrame has been executed.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.to_bytes"> | |
| <span class="sig-name descname"><span class="pre">to_bytes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">datafusion.context.SessionContext</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bytes</span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.to_bytes" title="Link to this definition">#</a></dt> | |
| <dd><p>Convert an ExecutionPlan into serialized protobuf bytes.</p> | |
| <p>When <code class="docutils literal notranslate"><span class="pre">ctx</span></code> is supplied, encoding routes through the codecs | |
| installed on it with | |
| <code class="xref py py-meth docutils literal notranslate"><span class="pre">with_physical_extension_codec()</span></code>.</p> | |
| <p>Unlike <a class="reference internal" href="#datafusion.plan.LogicalPlan.to_bytes" title="datafusion.plan.LogicalPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">LogicalPlan.to_bytes()</span></code></a>, a plan reading a table | |
| registered from record batches does round-trip: the batches travel | |
| inside the encoded scan.</p> | |
| <p>Round-tripping through this method and <a class="reference internal" href="#datafusion.plan.ExecutionPlan.from_bytes" title="datafusion.plan.ExecutionPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a> is how | |
| an extension library checks that its own codec claimed its nodes, | |
| rather than a codec installed earlier in the chain — see | |
| <a class="reference internal" href="../../../extension-guide/codecs.html#extension-codec-order"><span class="std std-ref">When codec order matters</span></a>.</p> | |
| <div class="admonition seealso"> | |
| <p class="admonition-title">See also</p> | |
| <p><a class="reference internal" href="#datafusion.plan.ExecutionPlan.from_bytes" title="datafusion.plan.ExecutionPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a>, <a class="reference internal" href="#datafusion.plan.LogicalPlan.to_bytes" title="datafusion.plan.LogicalPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">LogicalPlan.to_bytes()</span></code></a>.</p> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.to_proto"> | |
| <span class="sig-name descname"><span class="pre">to_proto</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bytes</span></span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan.to_proto" title="Link to this definition">#</a></dt> | |
| <dd><p>Deprecated alias for <a class="reference internal" href="#datafusion.plan.ExecutionPlan.to_bytes" title="datafusion.plan.ExecutionPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">to_bytes()</span></code></a>.</p> | |
| </dd></dl> | |
| <dl class="py attribute"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan._raw_plan"> | |
| <span class="sig-name descname"><span class="pre">_raw_plan</span></span><a class="headerlink" href="#datafusion.plan.ExecutionPlan._raw_plan" title="Link to this definition">#</a></dt> | |
| <dd></dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.output_partitioning"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">output_partitioning</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.plan.PhysicalPartitioning" title="datafusion.plan.PhysicalPartitioning"><span class="pre">PhysicalPartitioning</span></a></em><a class="headerlink" href="#datafusion.plan.ExecutionPlan.output_partitioning" title="Link to this definition">#</a></dt> | |
| <dd><p>Returns how this plan’s output rows are spread across its partitions.</p> | |
| <p>Where <a class="reference internal" href="#datafusion.plan.ExecutionPlan.partition_count" title="datafusion.plan.ExecutionPlan.partition_count"><code class="xref py py-attr docutils literal notranslate"><span class="pre">partition_count</span></code></a> gives only the number of partitions, | |
| this also reports the scheme, so a caller executing partitions | |
| separately can tell whether they are hash-distributed on known keys or | |
| merely counted. A plan does not necessarily partition the way it was | |
| asked to; see <a class="reference internal" href="../../../user-guide/configuration.html#checking-partitioning"><span class="std std-ref">Checking what the plan actually does</span></a>.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> | |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">SessionConfig</span><span class="p">()</span><span class="o">.</span><span class="n">with_target_partitions</span><span class="p">(</span><span class="mi">4</span><span class="p">))</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_record_batches</span><span class="p">(</span><span class="s2">"t"</span><span class="p">,</span> <span class="p">[</span> | |
| <span class="gp">... </span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})],</span> | |
| <span class="gp">... </span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})],</span> | |
| <span class="gp">... </span><span class="p">])</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"select a from t"</span><span class="p">)</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">output_partitioning</span> | |
| <span class="go">UnknownPartitioning(2)</span> | |
| </pre></div> | |
| </div> | |
| <p>A group-by redistributes rows, so the plan reports the keys:</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">grouped</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"select a, count(*) from t group by a"</span><span class="p">)</span> | |
| <span class="gp">>>> </span><span class="n">partitioning</span> <span class="o">=</span> <span class="n">grouped</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">output_partitioning</span> | |
| <span class="gp">>>> </span><span class="n">partitioning</span><span class="o">.</span><span class="n">scheme</span> | |
| <span class="go">'Hash'</span> | |
| <span class="gp">>>> </span><span class="n">partitioning</span><span class="o">.</span><span class="n">partition_count</span> | |
| <span class="go">4</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.ExecutionPlan.partition_count"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">partition_count</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#datafusion.plan.ExecutionPlan.partition_count" title="Link to this definition">#</a></dt> | |
| <dd><p>Returns the number of partitions in the physical plan.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> | |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">partition_count</span> | |
| <span class="go">1</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| </dd></dl> | |
| <dl class="py class"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan"> | |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.plan.</span></span><span class="sig-name descname"><span class="pre">LogicalPlan</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">plan</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.LogicalPlan</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.plan.LogicalPlan" title="Link to this definition">#</a></dt> | |
| <dd><p>Logical Plan.</p> | |
| <p>A <cite>LogicalPlan</cite> is a node in a tree of relational operators (such as | |
| Projection or Filter).</p> | |
| <p>Represents transforming an input relation (table) to an output relation | |
| (table) with a potentially different schema. Plans form a dataflow tree | |
| where data flows from leaves up to the root to produce the query result.</p> | |
| <p>A <cite>LogicalPlan</cite> can be created by the SQL query planner, the DataFrame API, | |
| or programmatically (for example custom query languages).</p> | |
| <p>This constructor should not be called by the end user.</p> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.__eq__"> | |
| <span class="sig-name descname"><span class="pre">__eq__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.__eq__" title="Link to this definition">#</a></dt> | |
| <dd><p>Test equality.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.__repr__"> | |
| <span class="sig-name descname"><span class="pre">__repr__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.__repr__" title="Link to this definition">#</a></dt> | |
| <dd><p>Generate a printable representation of the plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.display"> | |
| <span class="sig-name descname"><span class="pre">display</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.display" title="Link to this definition">#</a></dt> | |
| <dd><p>Print the logical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.display_graphviz"> | |
| <span class="sig-name descname"><span class="pre">display_graphviz</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.display_graphviz" title="Link to this definition">#</a></dt> | |
| <dd><p>Print the graph visualization of the logical plan.</p> | |
| <p>Returns a formattable structure that produces lines meant for graphical | |
| display using the <code class="docutils literal notranslate"><span class="pre">DOT</span></code> language. This format can be visualized using | |
| software from <a class="reference external" href="https://graphviz.org/">graphviz</a>.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.display_indent"> | |
| <span class="sig-name descname"><span class="pre">display_indent</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.display_indent" title="Link to this definition">#</a></dt> | |
| <dd><p>Print an indented form of the logical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.display_indent_schema"> | |
| <span class="sig-name descname"><span class="pre">display_indent_schema</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.display_indent_schema" title="Link to this definition">#</a></dt> | |
| <dd><p>Print an indented form of the schema for the logical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.from_bytes"> | |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_bytes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">datafusion.context.SessionContext</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bytes</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">LogicalPlan</span></a></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.from_bytes" title="Link to this definition">#</a></dt> | |
| <dd><p>Create a LogicalPlan from serialized protobuf bytes.</p> | |
| <p>Decoding routes through the codecs installed on <code class="docutils literal notranslate"><span class="pre">ctx</span></code> with | |
| <code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code>. | |
| Tables created in memory from record batches are currently not | |
| supported.</p> | |
| <p>Unlike <a class="reference internal" href="../index.html#datafusion.Expr.from_bytes" title="datafusion.Expr.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.Expr.from_bytes()</span></code></a>, <code class="docutils literal notranslate"><span class="pre">ctx</span></code> is required and | |
| positional, and there is no fallback to a worker or global context.</p> | |
| <div class="admonition seealso"> | |
| <p class="admonition-title">See also</p> | |
| <p><a class="reference internal" href="#datafusion.plan.LogicalPlan.to_bytes" title="datafusion.plan.LogicalPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">to_bytes()</span></code></a>, <a class="reference internal" href="#datafusion.plan.ExecutionPlan.from_bytes" title="datafusion.plan.ExecutionPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">ExecutionPlan.from_bytes()</span></code></a>, | |
| <a class="reference internal" href="../index.html#datafusion.Expr.from_bytes" title="datafusion.Expr.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.Expr.from_bytes()</span></code></a>.</p> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.from_proto"> | |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_proto</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">datafusion.context.SessionContext</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bytes</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">LogicalPlan</span></a></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.from_proto" title="Link to this definition">#</a></dt> | |
| <dd><p>Deprecated alias for <a class="reference internal" href="#datafusion.plan.LogicalPlan.from_bytes" title="datafusion.plan.LogicalPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a>.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.inputs"> | |
| <span class="sig-name descname"><span class="pre">inputs</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">LogicalPlan</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.inputs" title="Link to this definition">#</a></dt> | |
| <dd><p>Returns the list of inputs to the logical plan.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.to_bytes"> | |
| <span class="sig-name descname"><span class="pre">to_bytes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">datafusion.context.SessionContext</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bytes</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.to_bytes" title="Link to this definition">#</a></dt> | |
| <dd><p>Convert a LogicalPlan to serialized protobuf bytes.</p> | |
| <p>When <code class="docutils literal notranslate"><span class="pre">ctx</span></code> is supplied, encoding routes through the codecs | |
| installed on it with | |
| <code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code>, | |
| so extension codecs see the encode path. With <code class="docutils literal notranslate"><span class="pre">ctx=None</span></code> a | |
| default codec is used. Tables created in memory from record | |
| batches are currently not supported.</p> | |
| <div class="admonition seealso"> | |
| <p class="admonition-title">See also</p> | |
| <p><a class="reference internal" href="#datafusion.plan.LogicalPlan.from_bytes" title="datafusion.plan.LogicalPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a>, <a class="reference internal" href="#datafusion.plan.ExecutionPlan.to_bytes" title="datafusion.plan.ExecutionPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">ExecutionPlan.to_bytes()</span></code></a>, | |
| <a class="reference internal" href="../index.html#datafusion.Expr.to_bytes" title="datafusion.Expr.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.Expr.to_bytes()</span></code></a>.</p> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.to_proto"> | |
| <span class="sig-name descname"><span class="pre">to_proto</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bytes</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.to_proto" title="Link to this definition">#</a></dt> | |
| <dd><p>Deprecated alias for <a class="reference internal" href="#datafusion.plan.LogicalPlan.to_bytes" title="datafusion.plan.LogicalPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">to_bytes()</span></code></a>.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan.to_variant"> | |
| <span class="sig-name descname"><span class="pre">to_variant</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Any</span></span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan.to_variant" title="Link to this definition">#</a></dt> | |
| <dd><p>Convert the logical plan into its specific variant.</p> | |
| </dd></dl> | |
| <dl class="py attribute"> | |
| <dt class="sig sig-object py" id="datafusion.plan.LogicalPlan._raw_plan"> | |
| <span class="sig-name descname"><span class="pre">_raw_plan</span></span><a class="headerlink" href="#datafusion.plan.LogicalPlan._raw_plan" title="Link to this definition">#</a></dt> | |
| <dd></dd></dl> | |
| </dd></dl> | |
| <dl class="py class"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric"> | |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.plan.</span></span><span class="sig-name descname"><span class="pre">Metric</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">raw</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.Metric</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.plan.Metric" title="Link to this definition">#</a></dt> | |
| <dd><p>A single execution metric with name, value, partition, and labels.</p> | |
| <p>This constructor should not be called by the end user.</p> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric.__repr__"> | |
| <span class="sig-name descname"><span class="pre">__repr__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.Metric.__repr__" title="Link to this definition">#</a></dt> | |
| <dd><p>Return a string representation of the metric.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric.labels"> | |
| <span class="sig-name descname"><span class="pre">labels</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.Metric.labels" title="Link to this definition">#</a></dt> | |
| <dd><p>Return the labels associated with this metric.</p> | |
| <p>Labels provide additional context for a metric. For example:</p> | |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">metric</span><span class="o">.</span><span class="n">labels</span><span class="p">()</span> | |
| <span class="c1"># {'output_type': 'final'}</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py attribute"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric._raw"> | |
| <span class="sig-name descname"><span class="pre">_raw</span></span><a class="headerlink" href="#datafusion.plan.Metric._raw" title="Link to this definition">#</a></dt> | |
| <dd></dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric.name"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">name</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#datafusion.plan.Metric.name" title="Link to this definition">#</a></dt> | |
| <dd><p>The name of this metric (e.g. <code class="docutils literal notranslate"><span class="pre">output_rows</span></code>).</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric.partition"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">partition</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.Metric.partition" title="Link to this definition">#</a></dt> | |
| <dd><p>The 0-based partition index this metric applies to.</p> | |
| <p>Returns <code class="docutils literal notranslate"><span class="pre">None</span></code> for metrics that are not partition-specific (i.e. they | |
| apply globally across all partitions of the operator).</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric.value"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">value</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">datetime.datetime</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.Metric.value" title="Link to this definition">#</a></dt> | |
| <dd><p>The value of this metric.</p> | |
| <p>Returns an <code class="docutils literal notranslate"><span class="pre">int</span></code> for counters, gauges, and time-based metrics | |
| (nanoseconds), a <code class="xref py py-class docutils literal notranslate"><span class="pre">datetime</span></code> (UTC) for | |
| <code class="docutils literal notranslate"><span class="pre">start_timestamp</span></code> / <code class="docutils literal notranslate"><span class="pre">end_timestamp</span></code> metrics, or <code class="docutils literal notranslate"><span class="pre">None</span></code> | |
| when the value has not been set or is not representable.</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.Metric.value_as_datetime"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">value_as_datetime</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">datetime.datetime</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.Metric.value_as_datetime" title="Link to this definition">#</a></dt> | |
| <dd><p>The value as a UTC <code class="xref py py-class docutils literal notranslate"><span class="pre">datetime</span></code> for timestamp metrics.</p> | |
| <p>Returns <code class="docutils literal notranslate"><span class="pre">None</span></code> for all non-timestamp metrics and for timestamp | |
| metrics whose value has not been set (e.g. before execution).</p> | |
| </dd></dl> | |
| </dd></dl> | |
| <dl class="py class"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet"> | |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.plan.</span></span><span class="sig-name descname"><span class="pre">MetricsSet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">raw</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.MetricsSet</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.plan.MetricsSet" title="Link to this definition">#</a></dt> | |
| <dd><p>A set of metrics for a single execution plan operator.</p> | |
| <p>A physical plan operator runs independently across one or more partitions. | |
| <a class="reference internal" href="#datafusion.plan.MetricsSet.metrics" title="datafusion.plan.MetricsSet.metrics"><code class="xref py py-meth docutils literal notranslate"><span class="pre">metrics()</span></code></a> returns the raw per-partition <a class="reference internal" href="#datafusion.plan.Metric" title="datafusion.plan.Metric"><code class="xref py py-class docutils literal notranslate"><span class="pre">Metric</span></code></a> objects. | |
| The convenience properties (<a class="reference internal" href="#datafusion.plan.MetricsSet.output_rows" title="datafusion.plan.MetricsSet.output_rows"><code class="xref py py-attr docutils literal notranslate"><span class="pre">output_rows</span></code></a>, <a class="reference internal" href="#datafusion.plan.MetricsSet.elapsed_compute" title="datafusion.plan.MetricsSet.elapsed_compute"><code class="xref py py-attr docutils literal notranslate"><span class="pre">elapsed_compute</span></code></a>, | |
| etc.) automatically sum the named metric across <em>all</em> partitions, giving a | |
| single aggregate value for the operator as a whole.</p> | |
| <p>This constructor should not be called by the end user.</p> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.__repr__"> | |
| <span class="sig-name descname"><span class="pre">__repr__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.MetricsSet.__repr__" title="Link to this definition">#</a></dt> | |
| <dd><p>Return a string representation of the metrics set.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.metrics"> | |
| <span class="sig-name descname"><span class="pre">metrics</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="#datafusion.plan.Metric" title="datafusion.plan.Metric"><span class="pre">Metric</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.MetricsSet.metrics" title="Link to this definition">#</a></dt> | |
| <dd><p>Return all individual metrics in this set.</p> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.sum_by_name"> | |
| <span class="sig-name descname"><span class="pre">sum_by_name</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.plan.MetricsSet.sum_by_name" title="Link to this definition">#</a></dt> | |
| <dd><p>Sum the named metric across all partitions.</p> | |
| <p>Useful for accessing any metric not exposed as a first-class property. | |
| Returns <code class="docutils literal notranslate"><span class="pre">None</span></code> if no metric with the given name was recorded.</p> | |
| <dl class="field-list simple"> | |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> | |
| <dd class="field-odd"><p><strong>name</strong> – The metric name, e.g. <code class="docutils literal notranslate"><span class="pre">"output_rows"</span></code> or <code class="docutils literal notranslate"><span class="pre">"elapsed_compute"</span></code>.</p> | |
| </dd> | |
| </dl> | |
| </dd></dl> | |
| <dl class="py attribute"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet._raw"> | |
| <span class="sig-name descname"><span class="pre">_raw</span></span><a class="headerlink" href="#datafusion.plan.MetricsSet._raw" title="Link to this definition">#</a></dt> | |
| <dd></dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.elapsed_compute"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">elapsed_compute</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.MetricsSet.elapsed_compute" title="Link to this definition">#</a></dt> | |
| <dd><p>Total CPU time (in nanoseconds) spent inside this operator’s execute loop.</p> | |
| <p>Summed across all partitions. Returns <code class="docutils literal notranslate"><span class="pre">None</span></code> if no <code class="docutils literal notranslate"><span class="pre">elapsed_compute</span></code> | |
| metric was recorded.</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.output_rows"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">output_rows</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.MetricsSet.output_rows" title="Link to this definition">#</a></dt> | |
| <dd><p>Sum of output_rows across all partitions.</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.spill_count"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">spill_count</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.MetricsSet.spill_count" title="Link to this definition">#</a></dt> | |
| <dd><p>Number of times this operator spilled data to disk due to memory pressure.</p> | |
| <p>This is a count of spill events, not a byte count. Summed across all | |
| partitions. Returns <code class="docutils literal notranslate"><span class="pre">None</span></code> if no <code class="docutils literal notranslate"><span class="pre">spill_count</span></code> metric was recorded.</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.spilled_bytes"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">spilled_bytes</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.MetricsSet.spilled_bytes" title="Link to this definition">#</a></dt> | |
| <dd><p>Sum of spilled_bytes across all partitions.</p> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.MetricsSet.spilled_rows"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">spilled_rows</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.MetricsSet.spilled_rows" title="Link to this definition">#</a></dt> | |
| <dd><p>Sum of spilled_rows across all partitions.</p> | |
| </dd></dl> | |
| </dd></dl> | |
| <dl class="py class"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning"> | |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.plan.</span></span><span class="sig-name descname"><span class="pre">PhysicalPartitioning</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">partitioning</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.PhysicalPartitioning</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning" title="Link to this definition">#</a></dt> | |
| <dd><p>How a physical plan’s output rows are spread across its partitions.</p> | |
| <p>Returned by <a class="reference internal" href="#datafusion.plan.ExecutionPlan.output_partitioning" title="datafusion.plan.ExecutionPlan.output_partitioning"><code class="xref py py-attr docutils literal notranslate"><span class="pre">ExecutionPlan.output_partitioning</span></code></a>. This is the | |
| partitioning a built plan <em>has</em>. Distinct from | |
| <a class="reference internal" href="../expr/index.html#datafusion.expr.Partitioning" title="datafusion.expr.Partitioning"><code class="xref py py-class docutils literal notranslate"><span class="pre">datafusion.expr.Partitioning</span></code></a>, the <em>logical</em> partitioning a | |
| <code class="docutils literal notranslate"><span class="pre">Repartition</span></code> node records and hands back from | |
| <code class="docutils literal notranslate"><span class="pre">partitioning_scheme()</span></code> — a request, which the plan need not honour. See | |
| <a class="reference internal" href="../../../user-guide/configuration.html#checking-partitioning"><span class="std std-ref">Checking what the plan actually does</span></a>.</p> | |
| <p>This constructor should not be called by the end user.</p> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning.__eq__"> | |
| <span class="sig-name descname"><span class="pre">__eq__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning.__eq__" title="Link to this definition">#</a></dt> | |
| <dd><p>Compare two partitionings by scheme, count and hash expressions.</p> | |
| <p>Equality is structural, and does not mirror DataFusion’s own | |
| comparison of the underlying type, under which two | |
| <code class="docutils literal notranslate"><span class="pre">UnknownPartitioning</span></code> values of the same width are unequal.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">a</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">b</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">a</span><span class="o">.</span><span class="n">output_partitioning</span> <span class="o">==</span> <span class="n">b</span><span class="o">.</span><span class="n">output_partitioning</span> | |
| <span class="go">True</span> | |
| <span class="gp">>>> </span><span class="n">a</span><span class="o">.</span><span class="n">output_partitioning</span> <span class="o">==</span> <span class="s2">"UnknownPartitioning(1)"</span> | |
| <span class="go">False</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning.__hash__"> | |
| <span class="sig-name descname"><span class="pre">__hash__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">int</span></span></span><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning.__hash__" title="Link to this definition">#</a></dt> | |
| <dd><p>Hash the partitioning, consistently with <a class="reference internal" href="#datafusion.plan.PhysicalPartitioning.__eq__" title="datafusion.plan.PhysicalPartitioning.__eq__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__eq__()</span></code></a>.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">a</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">b</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="nb">len</span><span class="p">({</span><span class="n">a</span><span class="o">.</span><span class="n">output_partitioning</span><span class="p">,</span> <span class="n">b</span><span class="o">.</span><span class="n">output_partitioning</span><span class="p">})</span> | |
| <span class="go">1</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning.__repr__"> | |
| <span class="sig-name descname"><span class="pre">__repr__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning.__repr__" title="Link to this definition">#</a></dt> | |
| <dd><p>Print a string representation of the partitioning.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> | |
| <span class="gp">>>> </span><span class="nb">repr</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">output_partitioning</span><span class="p">)</span> | |
| <span class="go">'UnknownPartitioning(1)'</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py method"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning._key"> | |
| <span class="sig-name descname"><span class="pre">_key</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">int</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Ellipsis</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning._key" title="Link to this definition">#</a></dt> | |
| <dd></dd></dl> | |
| <dl class="py attribute"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning._raw_partitioning"> | |
| <span class="sig-name descname"><span class="pre">_raw_partitioning</span></span><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning._raw_partitioning" title="Link to this definition">#</a></dt> | |
| <dd></dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning.hash_expressions"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">hash_expressions</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning.hash_expressions" title="Link to this definition">#</a></dt> | |
| <dd><p>The expressions rows are hashed on, or <code class="docutils literal notranslate"><span class="pre">None</span></code> for other schemes.</p> | |
| <p>Physical expressions have no Python representation, so these are | |
| returned in their displayed form. <code class="docutils literal notranslate"><span class="pre">None</span></code> covers <code class="docutils literal notranslate"><span class="pre">"Range"</span></code> as well, | |
| whose ordering and split points this class does not expose.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> | |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">SessionConfig</span><span class="p">()</span><span class="o">.</span><span class="n">with_target_partitions</span><span class="p">(</span><span class="mi">4</span><span class="p">))</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_record_batches</span><span class="p">(</span><span class="s2">"t"</span><span class="p">,</span> <span class="p">[</span> | |
| <span class="gp">... </span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})],</span> | |
| <span class="gp">... </span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})],</span> | |
| <span class="gp">... </span><span class="p">])</span> | |
| <span class="gp">>>> </span><span class="n">scan</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"select a from t"</span><span class="p">)</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">scan</span><span class="o">.</span><span class="n">output_partitioning</span><span class="o">.</span><span class="n">hash_expressions</span> <span class="ow">is</span> <span class="kc">None</span> | |
| <span class="go">True</span> | |
| <span class="gp">>>> </span><span class="n">grouped</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"select a, count(*) from t group by a"</span><span class="p">)</span> | |
| <span class="gp">>>> </span><span class="n">grouped</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">output_partitioning</span><span class="o">.</span><span class="n">hash_expressions</span> | |
| <span class="go">['a@0']</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning.partition_count"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">partition_count</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">int</span></em><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning.partition_count" title="Link to this definition">#</a></dt> | |
| <dd><p>The number of partitions.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> | |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">output_partitioning</span><span class="o">.</span><span class="n">partition_count</span> | |
| <span class="go">1</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| <dl class="py property"> | |
| <dt class="sig sig-object py" id="datafusion.plan.PhysicalPartitioning.scheme"> | |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">scheme</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'RoundRobinBatch'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'Hash'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'Range'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'UnknownPartitioning'</span></span><span class="p"><span class="pre">]</span></span></em><a class="headerlink" href="#datafusion.plan.PhysicalPartitioning.scheme" title="Link to this definition">#</a></dt> | |
| <dd><p>Which partitioning scheme this is.</p> | |
| <p><code class="docutils literal notranslate"><span class="pre">"UnknownPartitioning"</span></code> means the plan knows how many partitions it | |
| has but nothing about how rows are distributed between them, which is | |
| the usual case for a file scan. <code class="docutils literal notranslate"><span class="pre">"RoundRobinBatch"</span></code> and <code class="docutils literal notranslate"><span class="pre">"Hash"</span></code> | |
| come from a repartition the optimizer inserted. <code class="docutils literal notranslate"><span class="pre">"Range"</span></code> only | |
| appears on a plan this package did not build; see | |
| <a class="reference internal" href="../../../user-guide/configuration.html#checking-partitioning"><span class="std std-ref">Checking what the plan actually does</span></a>.</p> | |
| <p class="rubric">Examples</p> | |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> | |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> | |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> | |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span><span class="o">.</span><span class="n">output_partitioning</span><span class="o">.</span><span class="n">scheme</span> | |
| <span class="go">'UnknownPartitioning'</span> | |
| </pre></div> | |
| </div> | |
| </dd></dl> | |
| </dd></dl> | |
| </section> | |
| </section> | |
| </article> | |
| <footer class="prev-next-footer d-print-none"> | |
| <div class="prev-next-area"> | |
| <a class="left-prev" | |
| href="../options/index.html" | |
| title="previous page"> | |
| <i class="fa-solid fa-angle-left"></i> | |
| <div class="prev-next-info"> | |
| <p class="prev-next-subtitle">previous</p> | |
| <p class="prev-next-title">datafusion.options</p> | |
| </div> | |
| </a> | |
| <a class="right-next" | |
| href="../record_batch/index.html" | |
| title="next page"> | |
| <div class="prev-next-info"> | |
| <p class="prev-next-subtitle">next</p> | |
| <p class="prev-next-title">datafusion.record_batch</p> | |
| </div> | |
| <i class="fa-solid fa-angle-right"></i> | |
| </a> | |
| </div> | |
| </footer> | |
| </div> | |
| <dialog id="pst-secondary-sidebar-modal"></dialog> | |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> | |
| <div class="sidebar-secondary-item"> | |
| <div | |
| id="pst-page-navigation-heading-2" | |
| class="page-toc tocsection onthispage"> | |
| <i class="fa-solid fa-list"></i> On this page | |
| </div> | |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> | |
| <ul class="visible nav section-nav flex-column"> | |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#classes">Classes</a></li> | |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#module-contents">Module Contents</a><ul class="visible nav section-nav flex-column"> | |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan</span></code></a><ul class="nav section-nav flex-column"> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.__repr__"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.__repr__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.children"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.children()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.collect_metrics"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.collect_metrics()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.display"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.display()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.display_indent"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.display_indent()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.from_bytes"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.from_bytes()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.from_proto"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.from_proto()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.metrics"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.metrics()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.to_bytes"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.to_bytes()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.to_proto"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.to_proto()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan._raw_plan"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan._raw_plan</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.output_partitioning"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.output_partitioning</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.ExecutionPlan.partition_count"><code class="docutils literal notranslate"><span class="pre">ExecutionPlan.partition_count</span></code></a></li> | |
| </ul> | |
| </li> | |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan"><code class="docutils literal notranslate"><span class="pre">LogicalPlan</span></code></a><ul class="nav section-nav flex-column"> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.__eq__"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.__eq__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.__repr__"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.__repr__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.display"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.display()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.display_graphviz"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.display_graphviz()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.display_indent"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.display_indent()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.display_indent_schema"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.display_indent_schema()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.from_bytes"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.from_bytes()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.from_proto"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.from_proto()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.inputs"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.inputs()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.to_bytes"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.to_bytes()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.to_proto"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.to_proto()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan.to_variant"><code class="docutils literal notranslate"><span class="pre">LogicalPlan.to_variant()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.LogicalPlan._raw_plan"><code class="docutils literal notranslate"><span class="pre">LogicalPlan._raw_plan</span></code></a></li> | |
| </ul> | |
| </li> | |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric"><code class="docutils literal notranslate"><span class="pre">Metric</span></code></a><ul class="nav section-nav flex-column"> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric.__repr__"><code class="docutils literal notranslate"><span class="pre">Metric.__repr__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric.labels"><code class="docutils literal notranslate"><span class="pre">Metric.labels()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric._raw"><code class="docutils literal notranslate"><span class="pre">Metric._raw</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric.name"><code class="docutils literal notranslate"><span class="pre">Metric.name</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric.partition"><code class="docutils literal notranslate"><span class="pre">Metric.partition</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric.value"><code class="docutils literal notranslate"><span class="pre">Metric.value</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.Metric.value_as_datetime"><code class="docutils literal notranslate"><span class="pre">Metric.value_as_datetime</span></code></a></li> | |
| </ul> | |
| </li> | |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet"><code class="docutils literal notranslate"><span class="pre">MetricsSet</span></code></a><ul class="nav section-nav flex-column"> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.__repr__"><code class="docutils literal notranslate"><span class="pre">MetricsSet.__repr__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.metrics"><code class="docutils literal notranslate"><span class="pre">MetricsSet.metrics()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.sum_by_name"><code class="docutils literal notranslate"><span class="pre">MetricsSet.sum_by_name()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet._raw"><code class="docutils literal notranslate"><span class="pre">MetricsSet._raw</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.elapsed_compute"><code class="docutils literal notranslate"><span class="pre">MetricsSet.elapsed_compute</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.output_rows"><code class="docutils literal notranslate"><span class="pre">MetricsSet.output_rows</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.spill_count"><code class="docutils literal notranslate"><span class="pre">MetricsSet.spill_count</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.spilled_bytes"><code class="docutils literal notranslate"><span class="pre">MetricsSet.spilled_bytes</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.MetricsSet.spilled_rows"><code class="docutils literal notranslate"><span class="pre">MetricsSet.spilled_rows</span></code></a></li> | |
| </ul> | |
| </li> | |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning</span></code></a><ul class="nav section-nav flex-column"> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning.__eq__"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning.__eq__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning.__hash__"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning.__hash__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning.__repr__"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning.__repr__()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning._key"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning._key()</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning._raw_partitioning"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning._raw_partitioning</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning.hash_expressions"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning.hash_expressions</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning.partition_count"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning.partition_count</span></code></a></li> | |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.plan.PhysicalPartitioning.scheme"><code class="docutils literal notranslate"><span class="pre">PhysicalPartitioning.scheme</span></code></a></li> | |
| </ul> | |
| </li> | |
| </ul> | |
| </li> | |
| </ul> | |
| </nav></div> | |
| </div></div> | |
| </div> | |
| <footer class="bd-footer-content"> | |
| </footer> | |
| </main> | |
| </div> | |
| </div> | |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> | |
| <script defer src="../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script> | |
| <script defer src="../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script> | |
| <!-- Based on pydata_sphinx_theme/footer.html --> | |
| <footer class="footer mt-5 mt-md-0"> | |
| <div class="container"> | |
| <div class="footer-item"> | |
| <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p> | |
| <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> | |
| </div> | |
| </div> | |
| </footer> | |
| </body> | |
| </html> |