| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../../../" data-theme="auto"> |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>datafusion.dataframe — Apache DataFusion in Python documentation</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto"; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto"; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| <link href="../../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css?v=4ae1632d" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/theme_overrides.css?v=4af573bc" /> |
| |
| <!-- So that users can add custom icons --> |
| <script src="../../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" /> |
| <link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" /> |
| |
| <script src="../../../_static/documentation_options.js?v=5929fcd5"></script> |
| <script src="../../../_static/doctools.js?v=9bcbadda"></script> |
| <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'autoapi/datafusion/dataframe/index';</script> |
| <script src="../../../_static/toc-toggle.js?v=36375f01"></script> |
| <link rel="icon" href="../../../_static/favicon.svg"/> |
| <link rel="index" title="Index" href="../../../genindex.html" /> |
| <link rel="search" title="Search" href="../../../search.html" /> |
| <link rel="next" title="datafusion.dataframe_formatter" href="../dataframe_formatter/index.html" /> |
| <link rel="prev" title="datafusion.context" href="../context/index.html" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="" /> |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto"> |
| |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/> |
| <img src="../../../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../user-guide/index.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../contributor-guide/index.html"> |
| Contributor Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../../index.html"> |
| API Reference |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../links.html"> |
| Links |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">Rust API docs (docs.rs)</span></a> |
| </li> |
| </ul></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../user-guide/index.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../contributor-guide/index.html"> |
| Contributor Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../../index.html"> |
| API Reference |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../links.html"> |
| Links |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">Rust API docs (docs.rs)</span></a> |
| </li> |
| </ul></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"> |
| <ul class="current nav bd-sidenav"> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/introduction.html">Introduction</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/basics.html">Concepts</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/data-sources.html">Data Sources</a></li> |
| |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/views.html">Registering Views</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/basic-info.html">Basic Operations</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/select-and-filter.html">Column Selections</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/expressions.html">Expressions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li> |
| |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/arrow.html">Arrow</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/avro.html">Avro</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/csv.html">CSV</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/json.html">JSON</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/parquet.html">Parquet</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../user-guide/io/table_provider.html">Custom Table Provider</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/configuration.html">Configuration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/distributing-work.html">Distributing work</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/sql.html">SQL</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/upgrade-guides.html">Upgrade Guides</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/introduction.html">Introduction</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/ffi.html">Python Extensions</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="../../index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l2 current active has-children"><a class="reference internal" href="../index.html">datafusion</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l3"><a class="reference internal" href="../catalog/index.html">datafusion.catalog</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../context/index.html">datafusion.context</a></li> |
| <li class="toctree-l3 current active"><a class="current reference internal" href="#">datafusion.dataframe</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3"><a class="reference internal" href="../io/index.html">datafusion.io</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../ipc/index.html">datafusion.ipc</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../object_store/index.html">datafusion.object_store</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../options/index.html">datafusion.options</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../plan/index.html">datafusion.plan</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../record_batch/index.html">datafusion.record_batch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../substrait/index.html">datafusion.substrait</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../unparser/index.html">datafusion.unparser</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../user_defined/index.html">datafusion.user_defined</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../../index.html" class="nav-link">API Reference</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">datafusion</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">datafusion.dataframe</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="module-datafusion.dataframe"> |
| <span id="datafusion-dataframe"></span><h1>datafusion.dataframe<a class="headerlink" href="#module-datafusion.dataframe" title="Link to this heading">#</a></h1> |
| <p><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> — lazy, chainable query representation.</p> |
| <p>A <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> is a logical plan over one or more data sources. |
| Methods that reshape the plan (<a class="reference internal" href="#datafusion.dataframe.DataFrame.select" title="datafusion.dataframe.DataFrame.select"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.select()</span></code></a>, |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.filter" title="datafusion.dataframe.DataFrame.filter"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.filter()</span></code></a>, <a class="reference internal" href="#datafusion.dataframe.DataFrame.aggregate" title="datafusion.dataframe.DataFrame.aggregate"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.aggregate()</span></code></a>, |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.sort" title="datafusion.dataframe.DataFrame.sort"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.sort()</span></code></a>, <a class="reference internal" href="#datafusion.dataframe.DataFrame.join" title="datafusion.dataframe.DataFrame.join"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.join()</span></code></a>, |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.limit" title="datafusion.dataframe.DataFrame.limit"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.limit()</span></code></a>, the set-operation methods, …) return a new |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and do no work until a terminal method such as |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.collect" title="datafusion.dataframe.DataFrame.collect"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.collect()</span></code></a>, <a class="reference internal" href="#datafusion.dataframe.DataFrame.to_pydict" title="datafusion.dataframe.DataFrame.to_pydict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.to_pydict()</span></code></a>, |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.show" title="datafusion.dataframe.DataFrame.show"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.show()</span></code></a>, or one of the <code class="docutils literal notranslate"><span class="pre">write_*</span></code> methods is called.</p> |
| <p>DataFrames are produced from a |
| <a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>, typically via |
| <a class="reference internal" href="../context/index.html#datafusion.context.SessionContext.sql" title="datafusion.context.SessionContext.sql"><code class="xref py py-meth docutils literal notranslate"><span class="pre">sql()</span></code></a>, |
| <a class="reference internal" href="../context/index.html#datafusion.context.SessionContext.read_csv" title="datafusion.context.SessionContext.read_csv"><code class="xref py py-meth docutils literal notranslate"><span class="pre">read_csv()</span></code></a>, |
| <a class="reference internal" href="../context/index.html#datafusion.context.SessionContext.read_parquet" title="datafusion.context.SessionContext.read_parquet"><code class="xref py py-meth docutils literal notranslate"><span class="pre">read_parquet()</span></code></a>, or |
| <a class="reference internal" href="../context/index.html#datafusion.context.SessionContext.from_pydict" title="datafusion.context.SessionContext.from_pydict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_pydict()</span></code></a>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'b': [20, 30]}</span> |
| </pre></div> |
| </div> |
| <p>See <a class="reference internal" href="../../../user-guide/basics.html#user-guide-concepts"><span class="std std-ref">Concepts</span></a> in the online documentation for a high-level |
| overview of the execution model.</p> |
| <section id="classes"> |
| <h2>Classes<a class="headerlink" href="#classes" title="Link to this heading">#</a></h2> |
| <div class="pst-scrollable-table-container"><table class="autosummary longtable table autosummary"> |
| <tbody> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.Compression" title="datafusion.dataframe.Compression"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Compression</span></code></a></p></td> |
| <td><p>Enum representing the available compression types for Parquet files.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-obj docutils literal notranslate"><span class="pre">DataFrame</span></code></a></p></td> |
| <td><p>Two dimensional table representation of data.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">DataFrameWriteOptions</span></code></a></p></td> |
| <td><p>Writer options for DataFrame.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.ExplainFormat" title="datafusion.dataframe.ExplainFormat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ExplainFormat</span></code></a></p></td> |
| <td><p>Output format for explain plans.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.InsertOp" title="datafusion.dataframe.InsertOp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">InsertOp</span></code></a></p></td> |
| <td><p>Insert operation mode.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.ParquetColumnOptions" title="datafusion.dataframe.ParquetColumnOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ParquetColumnOptions</span></code></a></p></td> |
| <td><p>Parquet options for individual columns.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.ParquetWriterOptions" title="datafusion.dataframe.ParquetWriterOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ParquetWriterOptions</span></code></a></p></td> |
| <td><p>Advanced parquet writer options.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| </section> |
| <section id="module-contents"> |
| <h2>Module Contents<a class="headerlink" href="#module-contents" title="Link to this heading">#</a></h2> |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">Compression</span></span><a class="headerlink" href="#datafusion.dataframe.Compression" title="Link to this definition">#</a></dt> |
| <dd><p>Bases: <code class="xref py py-obj docutils literal notranslate"><span class="pre">enum.Enum</span></code></p> |
| <p>Enum representing the available compression types for Parquet files.</p> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.from_str"> |
| <em class="property"><span class="pre">classmethod</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_str</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.Compression" title="datafusion.dataframe.Compression"><span class="pre">Compression</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.Compression.from_str" title="Link to this definition">#</a></dt> |
| <dd><p>Convert a string to a Compression enum value.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>value</strong> – The string representation of the compression type.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>The Compression enum lowercase value.</p> |
| </dd> |
| <dt class="field-odd">Raises<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>ValueError</strong> – If the string does not match any Compression enum value.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.get_default_level"> |
| <span class="sig-name descname"><span class="pre">get_default_level</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.Compression.get_default_level" title="Link to this definition">#</a></dt> |
| <dd><p>Get the default compression level for the compression type.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>The default compression level for the compression type.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.BROTLI"> |
| <span class="sig-name descname"><span class="pre">BROTLI</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'brotli'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.BROTLI" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.GZIP"> |
| <span class="sig-name descname"><span class="pre">GZIP</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'gzip'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.GZIP" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.LZ4"> |
| <span class="sig-name descname"><span class="pre">LZ4</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'lz4'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.LZ4" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.LZ4_RAW"> |
| <span class="sig-name descname"><span class="pre">LZ4_RAW</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'lz4_raw'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.LZ4_RAW" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.SNAPPY"> |
| <span class="sig-name descname"><span class="pre">SNAPPY</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'snappy'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.SNAPPY" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.UNCOMPRESSED"> |
| <span class="sig-name descname"><span class="pre">UNCOMPRESSED</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'uncompressed'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.UNCOMPRESSED" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.Compression.ZSTD"> |
| <span class="sig-name descname"><span class="pre">ZSTD</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'zstd'</span></em><a class="headerlink" href="#datafusion.dataframe.Compression.ZSTD" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">DataFrame</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion._internal.DataFrame</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.dataframe.DataFrame" title="Link to this definition">#</a></dt> |
| <dd><p>Two dimensional table representation of data.</p> |
| <p>DataFrame objects are iterable; iterating over a DataFrame yields |
| <a class="reference internal" href="../index.html#datafusion.RecordBatch" title="datafusion.RecordBatch"><code class="xref py py-class docutils literal notranslate"><span class="pre">datafusion.RecordBatch</span></code></a> instances lazily.</p> |
| <p>See <a class="reference internal" href="../../../user-guide/basics.html#user-guide-concepts"><span class="std std-ref">Concepts</span></a> in the online documentation for more information.</p> |
| <p>This constructor is not to be used by the end user.</p> |
| <p>See <a class="reference internal" href="../context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a> for methods to |
| create a <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.__aiter__"> |
| <span class="sig-name descname"><span class="pre">__aiter__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">collections.abc.AsyncIterator</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../record_batch/index.html#datafusion.record_batch.RecordBatch" title="datafusion.record_batch.RecordBatch"><span class="pre">datafusion.record_batch.RecordBatch</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__aiter__" title="Link to this definition">#</a></dt> |
| <dd><p>Return an async iterator over this DataFrame’s record batches.</p> |
| <p>We’re using __aiter__ because we support Python < 3.10 where aiter() is not |
| available.</p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.__arrow_c_stream__"> |
| <span class="sig-name descname"><span class="pre">__arrow_c_stream__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">requested_schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">object</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">object</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__arrow_c_stream__" title="Link to this definition">#</a></dt> |
| <dd><p>Export the DataFrame as an Arrow C Stream.</p> |
| <p>The DataFrame is executed using DataFusion’s streaming APIs and exposed via |
| Arrow’s C Stream interface. Record batches are produced incrementally, so the |
| full result set is never materialized in memory.</p> |
| <p>When <code class="docutils literal notranslate"><span class="pre">requested_schema</span></code> is provided, DataFusion applies only simple |
| projections such as selecting a subset of existing columns or reordering |
| them. Column renaming, computed expressions, or type coercion are not |
| supported through this interface.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>requested_schema</strong> – Either a <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.Schema</span></code> or an Arrow C |
| Schema capsule (<code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code>) produced by |
| <code class="docutils literal notranslate"><span class="pre">schema._export_to_c_capsule()</span></code>. The DataFrame will attempt to |
| align its output with the fields and order specified by this schema.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>Arrow <code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code> object representing an <code class="docutils literal notranslate"><span class="pre">ArrowArrayStream</span></code>.</p> |
| </dd> |
| </dl> |
| <p>For practical usage patterns, see the Apache Arrow streaming |
| documentation: <a class="reference external" href="https://arrow.apache.org/docs/python/ipc.html#streaming">https://arrow.apache.org/docs/python/ipc.html#streaming</a>.</p> |
| <p>For details on DataFusion’s Arrow integration and DataFrame streaming, |
| see the user guide (user-guide/io/arrow and user-guide/dataframe/index).</p> |
| <p class="rubric">Notes</p> |
| <p>The Arrow C Data Interface PyCapsule details are documented by Apache |
| Arrow and can be found at: |
| <a class="reference external" href="https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html">https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html</a></p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.__getitem__"> |
| <span class="sig-name descname"><span class="pre">__getitem__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">key</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__getitem__" title="Link to this definition">#</a></dt> |
| <dd><p>Return a new <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with the specified column or columns.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>key</strong> – Column name or list of column names to select.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with the specified column or columns.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.__iter__"> |
| <span class="sig-name descname"><span class="pre">__iter__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">collections.abc.Iterator</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../record_batch/index.html#datafusion.record_batch.RecordBatch" title="datafusion.record_batch.RecordBatch"><span class="pre">datafusion.record_batch.RecordBatch</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__iter__" title="Link to this definition">#</a></dt> |
| <dd><p>Return an iterator over this DataFrame’s record batches.</p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.__repr__"> |
| <span class="sig-name descname"><span class="pre">__repr__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.__repr__" title="Link to this definition">#</a></dt> |
| <dd><p>Return a string representation of the DataFrame.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>String representation of the DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame._repr_html_"> |
| <span class="sig-name descname"><span class="pre">_repr_html_</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame._repr_html_" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.aggregate"> |
| <span class="sig-name descname"><span class="pre">aggregate</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">group_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">aggs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.aggregate" title="Link to this definition">#</a></dt> |
| <dd><p>Aggregates the rows of the current DataFrame.</p> |
| <p>By default each unique combination of the <code class="docutils literal notranslate"><span class="pre">group_by</span></code> columns |
| produces one row. To get multiple levels of subtotals in a |
| single pass, pass a |
| <a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet" title="datafusion.expr.GroupingSet"><code class="xref py py-class docutils literal notranslate"><span class="pre">GroupingSet</span></code></a> expression |
| (created via |
| <a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet.rollup" title="datafusion.expr.GroupingSet.rollup"><code class="xref py py-meth docutils literal notranslate"><span class="pre">rollup()</span></code></a>, |
| <a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet.cube" title="datafusion.expr.GroupingSet.cube"><code class="xref py py-meth docutils literal notranslate"><span class="pre">cube()</span></code></a>, or |
| <a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet.grouping_sets" title="datafusion.expr.GroupingSet.grouping_sets"><code class="xref py py-meth docutils literal notranslate"><span class="pre">grouping_sets()</span></code></a>) |
| as the <code class="docutils literal notranslate"><span class="pre">group_by</span></code> argument. See the |
| <a class="reference internal" href="../../../user-guide/common-operations/aggregations.html#aggregation"><span class="std std-ref">Aggregation</span></a> user guide for detailed examples.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>group_by</strong> – Sequence of expressions or column names to group |
| by, or <code class="docutils literal notranslate"><span class="pre">None</span></code> for aggregation over the whole DataFrame. |
| A <a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet" title="datafusion.expr.GroupingSet"><code class="xref py py-class docutils literal notranslate"><span class="pre">GroupingSet</span></code></a> expression may |
| be included to produce multiple grouping levels (rollup, |
| cube, or explicit grouping sets).</p></li> |
| <li><p><strong>aggs</strong> – Sequence of expressions to aggregate.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after aggregation.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Aggregate without grouping — <code class="docutils literal notranslate"><span class="pre">None</span></code> or an empty <code class="docutils literal notranslate"><span class="pre">group_by</span></code> |
| produces a single row:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">{</span><span class="s2">"team"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"x"</span><span class="p">,</span> <span class="s2">"x"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">],</span> <span class="s2">"score"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">]}</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">F</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"score"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"total"</span><span class="p">)])</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'total': [8]}</span> |
| </pre></div> |
| </div> |
| <p>Group by a column and produce one row per group:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[</span><span class="s2">"team"</span><span class="p">],</span> <span class="p">[</span><span class="n">F</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"score"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"total"</span><span class="p">)]</span> |
| <span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"team"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'team': ['x', 'y'], 'total': [3, 5]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.alias"> |
| <span class="sig-name descname"><span class="pre">alias</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">alias</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.alias" title="Link to this definition">#</a></dt> |
| <dd><p>Assign a table alias to this <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p>Replaces the qualifiers of the output columns with <code class="docutils literal notranslate"><span class="pre">alias</span></code>. Useful for |
| self-joins and any situation that needs an unambiguous table-style |
| qualifier (<code class="docutils literal notranslate"><span class="pre">alias.col</span></code>) for downstream references.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>alias</strong> – Table alias to apply to the DataFrame’s columns.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with columns re-qualified under <code class="docutils literal notranslate"><span class="pre">alias</span></code>.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Example</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"val"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"l"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"r"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">left</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">right</span><span class="p">,</span> <span class="n">left_on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">right_on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="s2">"id"</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"l.val"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"lval"</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"r.val"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"rval"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'id': [1, 2], 'lval': [10, 20], 'rval': [10, 20]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.cache"> |
| <span class="sig-name descname"><span class="pre">cache</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.cache" title="Link to this definition">#</a></dt> |
| <dd><p>Cache the DataFrame as a memory table.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Cached DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.cast"> |
| <span class="sig-name descname"><span class="pre">cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">mapping</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">pyarrow.DataType</span><span class="p"><span class="pre">[</span></span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.cast" title="Link to this definition">#</a></dt> |
| <dd><p>Cast one or more columns to a different data type.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>mapping</strong> – Mapped with column as key and column dtype as value.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after casting columns</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.col"> |
| <span class="sig-name descname"><span class="pre">col</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.col" title="Link to this definition">#</a></dt> |
| <dd><p>Alias for <a class="reference internal" href="#datafusion.dataframe.DataFrame.column" title="datafusion.dataframe.DataFrame.column"><code class="xref py py-meth docutils literal notranslate"><span class="pre">column()</span></code></a>.</p> |
| <div class="admonition seealso"> |
| <p class="admonition-title">See also</p> |
| <p><a class="reference internal" href="#datafusion.dataframe.DataFrame.column" title="datafusion.dataframe.DataFrame.column"><code class="xref py py-meth docutils literal notranslate"><span class="pre">column()</span></code></a></p> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.collect"> |
| <span class="sig-name descname"><span class="pre">collect</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.RecordBatch</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.collect" title="Link to this definition">#</a></dt> |
| <dd><p>Execute this <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and collect results into memory.</p> |
| <p>Prior to calling <code class="docutils literal notranslate"><span class="pre">collect</span></code>, modifying a DataFrame simply updates a plan |
| (no actual computation is performed). Calling <code class="docutils literal notranslate"><span class="pre">collect</span></code> triggers the |
| computation.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>List of <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code> collected from the DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.collect_column"> |
| <span class="sig-name descname"><span class="pre">collect_column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">column_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.Array</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pyarrow.ChunkedArray</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.collect_column" title="Link to this definition">#</a></dt> |
| <dd><p>Executes this <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> for a single column.</p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.collect_partitioned"> |
| <span class="sig-name descname"><span class="pre">collect_partitioned</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.RecordBatch</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.collect_partitioned" title="Link to this definition">#</a></dt> |
| <dd><p>Execute this DataFrame and collect all partitioned results.</p> |
| <p>This operation returns <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code> maintaining the input |
| partitioning.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><dl class="simple"> |
| <dt>List of list of <code class="xref py py-class docutils literal notranslate"><span class="pre">RecordBatch</span></code> collected from the</dt><dd><p>DataFrame.</p> |
| </dd> |
| </dl> |
| </p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.column"> |
| <span class="sig-name descname"><span class="pre">column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.column" title="Link to this definition">#</a></dt> |
| <dd><p>Return a fully qualified column expression for <code class="docutils literal notranslate"><span class="pre">name</span></code>.</p> |
| <p>Resolves an unqualified column name against this DataFrame’s schema |
| and returns an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code> whose underlying column reference |
| includes the table qualifier. This is especially useful after joins, |
| where the same column name may appear in multiple relations.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>name</strong> – Unqualified column name to look up.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A fully qualified column expression.</p> |
| </dd> |
| <dt class="field-odd">Raises<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>Exception</strong> – If the column is not found or is ambiguous (exists in |
| multiple relations).</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Resolve a column from a simple DataFrame:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">expr</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2]}</span> |
| </pre></div> |
| </div> |
| <p>Resolve qualified columns after a join:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="p">[</span><span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">joined</span> <span class="o">=</span> <span class="n">left</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">right</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">"inner"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">expr</span> <span class="o">=</span> <span class="n">joined</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"y"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">joined</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'id': [1, 2], 'y': [30, 40]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.count"> |
| <span class="sig-name descname"><span class="pre">count</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">int</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.count" title="Link to this definition">#</a></dt> |
| <dd><p>Return the total number of rows in this <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p>Note that this method will actually run a plan to calculate the |
| count, which may be slow for large or complicated DataFrames.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Number of rows in the DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.default_str_repr"> |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">default_str_repr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">batches</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.RecordBatch</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Schema</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">has_more</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_uuid</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.default_str_repr" title="Link to this definition">#</a></dt> |
| <dd><p>Return the default string representation of a DataFrame.</p> |
| <p>This method is used by the default formatter and implemented in Rust for |
| performance reasons.</p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.describe"> |
| <span class="sig-name descname"><span class="pre">describe</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.describe" title="Link to this definition">#</a></dt> |
| <dd><p>Return the statistics for this DataFrame.</p> |
| <p>Only summarized numeric datatypes at the moments and returns nulls |
| for non-numeric datatypes.</p> |
| <p>The output format is modeled after pandas.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>A summary DataFrame containing statistics.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.distinct"> |
| <span class="sig-name descname"><span class="pre">distinct</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.distinct" title="Link to this definition">#</a></dt> |
| <dd><p>Return a new <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with all duplicated rows removed.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>DataFrame after removing duplicates.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.distinct_on"> |
| <span class="sig-name descname"><span class="pre">distinct_on</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">on_expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">sort_expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">datafusion.expr.SortKey</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.distinct_on" title="Link to this definition">#</a></dt> |
| <dd><p>Deduplicate rows based on specific columns.</p> |
| <p>Returns a new DataFrame with one row per unique combination of the |
| <code class="docutils literal notranslate"><span class="pre">on_expr</span></code> columns, keeping the first row per group as determined by |
| <code class="docutils literal notranslate"><span class="pre">sort_expr</span></code>.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>on_expr</strong> – Expressions that determine uniqueness.</p></li> |
| <li><p><strong>select_expr</strong> – Expressions to include in the output.</p></li> |
| <li><p><strong>sort_expr</strong> – Optional sort expressions to determine which row to keep.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after deduplication.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Keep the row with the smallest <code class="docutils literal notranslate"><span class="pre">b</span></code> for each unique <code class="docutils literal notranslate"><span class="pre">a</span></code>:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">distinct_on</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)],</span> |
| <span class="gp">... </span> <span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)],</span> |
| <span class="gp">... </span> <span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">ascending</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">ascending</span><span class="o">=</span><span class="kc">True</span><span class="p">)],</span> |
| <span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2], 'b': [10, 30]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.drop"> |
| <span class="sig-name descname"><span class="pre">drop</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">columns</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.drop" title="Link to this definition">#</a></dt> |
| <dd><p>Drop arbitrary amount of columns.</p> |
| <p>Column names are case-sensitive and require double quotes to be dropped |
| if the original name is not strictly lower case.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>columns</strong> – Column names to drop from the dataframe.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with those columns removed in the projection.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>To drop a lower-cased column ‘a’</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">names</span> |
| <span class="go">['b']</span> |
| </pre></div> |
| </div> |
| <p>Or to drop an upper-cased column ‘A’</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"A"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">'"A"'</span><span class="p">)</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">names</span> |
| <span class="go">['b']</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.except_all"> |
| <span class="sig-name descname"><span class="pre">except_all</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.except_all" title="Link to this definition">#</a></dt> |
| <dd><p>Calculate the set difference of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p>Returns rows that are in this DataFrame but not in <code class="docutils literal notranslate"><span class="pre">other</span></code>.</p> |
| <p>The two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> must have exactly the same schema.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>other</strong> – DataFrame to calculate exception with.</p></li> |
| <li><p><strong>distinct</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, duplicate rows are removed from the result.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after set difference.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Remove rows present in <code class="docutils literal notranslate"><span class="pre">df2</span></code>:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">except_all</span><span class="p">(</span><span class="n">df2</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [3], 'b': [30]}</span> |
| </pre></div> |
| </div> |
| <p>Remove rows present in <code class="docutils literal notranslate"><span class="pre">df2</span></code> and deduplicate:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">except_all</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [3], 'b': [30]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.execute_stream"> |
| <span class="sig-name descname"><span class="pre">execute_stream</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../record_batch/index.html#datafusion.record_batch.RecordBatchStream" title="datafusion.record_batch.RecordBatchStream"><span class="pre">datafusion.record_batch.RecordBatchStream</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.execute_stream" title="Link to this definition">#</a></dt> |
| <dd><p>Executes this DataFrame and returns a stream over a single partition.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Record Batch Stream over a single partition.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.execute_stream_partitioned"> |
| <span class="sig-name descname"><span class="pre">execute_stream_partitioned</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../record_batch/index.html#datafusion.record_batch.RecordBatchStream" title="datafusion.record_batch.RecordBatchStream"><span class="pre">datafusion.record_batch.RecordBatchStream</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.execute_stream_partitioned" title="Link to this definition">#</a></dt> |
| <dd><p>Executes this DataFrame and returns a stream for each partition.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>One record batch stream per partition.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.execution_plan"> |
| <span class="sig-name descname"><span class="pre">execution_plan</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../plan/index.html#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">datafusion.plan.ExecutionPlan</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.execution_plan" title="Link to this definition">#</a></dt> |
| <dd><p>Return the execution/physical plan.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Execution plan.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.explain"> |
| <span class="sig-name descname"><span class="pre">explain</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">verbose</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">analyze</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.ExplainFormat" title="datafusion.dataframe.ExplainFormat"><span class="pre">ExplainFormat</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.explain" title="Link to this definition">#</a></dt> |
| <dd><p>Print an explanation of the DataFrame’s plan so far.</p> |
| <p>If <code class="docutils literal notranslate"><span class="pre">analyze</span></code> is specified, runs the plan and reports metrics.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>verbose</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, more details will be included.</p></li> |
| <li><p><strong>analyze</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the plan will run and metrics reported.</p></li> |
| <li><p><strong>format</strong> – Output format for the plan. Defaults to |
| <a class="reference internal" href="#datafusion.dataframe.ExplainFormat.INDENT" title="datafusion.dataframe.ExplainFormat.INDENT"><code class="xref py py-attr docutils literal notranslate"><span class="pre">ExplainFormat.INDENT</span></code></a>.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Show the plan in tree format:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExplainFormat</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">explain</span><span class="p">(</span><span class="nb">format</span><span class="o">=</span><span class="n">ExplainFormat</span><span class="o">.</span><span class="n">TREE</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Show plan with runtime metrics:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">explain</span><span class="p">(</span><span class="n">analyze</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.fill_null"> |
| <span class="sig-name descname"><span class="pre">fill_null</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">subset</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.fill_null" title="Link to this definition">#</a></dt> |
| <dd><p>Fill null values in specified columns with a value.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>value</strong> – Value to replace nulls with. Will be cast to match column type.</p></li> |
| <li><p><strong>subset</strong> – Optional list of column names to fill. If None, fills all columns.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with null values replaced where type casting is possible</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">col</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">filled</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">fill_null</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">filled</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pylist</span><span class="p">()</span> |
| <span class="go">[0, 1, 3]</span> |
| </pre></div> |
| </div> |
| <p class="rubric">Notes</p> |
| <ul class="simple"> |
| <li><p>Only fills nulls in columns where the value can be cast to the column type</p></li> |
| <li><p>For columns where casting fails, the original column is kept unchanged</p></li> |
| <li><p>For columns not in subset, the original column is kept unchanged</p></li> |
| </ul> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.filter"> |
| <span class="sig-name descname"><span class="pre">filter</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">predicates</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.filter" title="Link to this definition">#</a></dt> |
| <dd><p>Return a DataFrame for which <code class="docutils literal notranslate"><span class="pre">predicate</span></code> evaluates to <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p> |
| <p>Rows for which <code class="docutils literal notranslate"><span class="pre">predicate</span></code> evaluates to <code class="docutils literal notranslate"><span class="pre">False</span></code> or <code class="docutils literal notranslate"><span class="pre">None</span></code> are filtered |
| out. If more than one predicate is provided, these predicates will be |
| combined as a logical AND. Each <code class="docutils literal notranslate"><span class="pre">predicate</span></code> can be an |
| <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> created using helper functions such as |
| <a class="reference internal" href="../index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.col()</span></code></a> or <a class="reference internal" href="../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.lit()</span></code></a>, or a SQL expression string |
| that will be parsed against the DataFrame schema. If more complex logic is |
| required, see the logical operations in <a class="reference internal" href="../functions/index.html#module-datafusion.functions" title="datafusion.functions"><code class="xref py py-mod docutils literal notranslate"><span class="pre">functions</span></code></a>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [2, 3]}</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="s2">"a > 1"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [2, 3]}</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>predicates</strong> – Predicate expression(s) or SQL strings to filter the DataFrame.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after filtering.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.find_qualified_columns"> |
| <span class="sig-name descname"><span class="pre">find_qualified_columns</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">names</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.find_qualified_columns" title="Link to this definition">#</a></dt> |
| <dd><p>Return fully qualified column expressions for the given names.</p> |
| <p>This is a batch version of <a class="reference internal" href="#datafusion.dataframe.DataFrame.column" title="datafusion.dataframe.DataFrame.column"><code class="xref py py-meth docutils literal notranslate"><span class="pre">column()</span></code></a> — it resolves each |
| unqualified name against the DataFrame’s schema and returns a list |
| of qualified column expressions.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>names</strong> – Unqualified column names to look up.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>List of fully qualified column expressions, one per name.</p> |
| </dd> |
| <dt class="field-odd">Raises<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>Exception</strong> – If any column is not found or is ambiguous.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Resolve multiple columns at once:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="s2">"c"</span><span class="p">:</span> <span class="p">[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">exprs</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">find_qualified_columns</span><span class="p">(</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="o">*</span><span class="n">exprs</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2], 'c': [5, 6]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.head"> |
| <span class="sig-name descname"><span class="pre">head</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">n</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">5</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.head" title="Link to this definition">#</a></dt> |
| <dd><p>Return a new <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with a limited number of rows.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>n</strong> – Number of rows to take from the head of the DataFrame.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after limiting.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.intersect"> |
| <span class="sig-name descname"><span class="pre">intersect</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.intersect" title="Link to this definition">#</a></dt> |
| <dd><p>Calculate the intersection of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p>The two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> must have exactly the same schema.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>other</strong> – DataFrame to intersect with.</p></li> |
| <li><p><strong>distinct</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, duplicate rows are removed from the result.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after intersection.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Find rows common to both DataFrames:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">40</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">intersect</span><span class="p">(</span><span class="n">df2</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1], 'b': [10]}</span> |
| </pre></div> |
| </div> |
| <p>Intersect with deduplication:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">intersect</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1], 'b': [10]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.into_view"> |
| <span class="sig-name descname"><span class="pre">into_view</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">temporary</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../catalog/index.html#datafusion.catalog.Table" title="datafusion.catalog.Table"><span class="pre">datafusion.catalog.Table</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.into_view" title="Link to this definition">#</a></dt> |
| <dd><p>Convert <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> into a <a class="reference internal" href="../index.html#datafusion.Table" title="datafusion.Table"><code class="xref py py-class docutils literal notranslate"><span class="pre">Table</span></code></a>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT 1 AS value"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">view</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">into_view</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_table</span><span class="p">(</span><span class="s2">"values_view"</span><span class="p">,</span> <span class="n">view</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">result</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT value FROM values_view"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">result</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pylist</span><span class="p">()</span> |
| <span class="go">[1]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.join"> |
| <span class="sig-name descname"><span class="pre">join</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">how</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'inner'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'full'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'semi'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'anti'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'inner'</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">left_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">right_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">join_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">coalesce_duplicate_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.join" title="Link to this definition">#</a></dt> |
| <dt class="sig sig-object py"> |
| <span class="sig-name descname"><span class="pre">join</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">how</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'inner'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'full'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'semi'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'anti'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'inner'</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">left_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">right_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">join_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">coalesce_duplicate_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span></dt> |
| <dt class="sig sig-object py"> |
| <span class="sig-name descname"><span class="pre">join</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">how</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'inner'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'full'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'semi'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'anti'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'inner'</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">join_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">left_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">right_on</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">coalesce_duplicate_keys</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span></dt> |
| <dd><p>Join this <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with another <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">on</span></code> has to be provided or both <code class="docutils literal notranslate"><span class="pre">left_on</span></code> and <code class="docutils literal notranslate"><span class="pre">right_on</span></code> in |
| conjunction.</p> |
| <p>When non-key columns share the same name in both DataFrames, use |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.col" title="datafusion.dataframe.DataFrame.col"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.col()</span></code></a> on each DataFrame <strong>before</strong> the join to |
| obtain fully qualified column references that can disambiguate them. |
| See <a class="reference internal" href="#datafusion.dataframe.DataFrame.join_on" title="datafusion.dataframe.DataFrame.join_on"><code class="xref py py-meth docutils literal notranslate"><span class="pre">join_on()</span></code></a> for an example.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>right</strong> – Other DataFrame to join with.</p></li> |
| <li><p><strong>on</strong> – Column names to join on in both dataframes.</p></li> |
| <li><p><strong>how</strong> – Type of join to perform. Supported types are “inner”, “left”, |
| “right”, “full”, “semi”, “anti”.</p></li> |
| <li><p><strong>left_on</strong> – Join column of the left dataframe.</p></li> |
| <li><p><strong>right_on</strong> – Join column of the right dataframe.</p></li> |
| <li><p><strong>coalesce_duplicate_keys</strong> – When True, coalesce the columns |
| from the right DataFrame and left DataFrame |
| that have identical names in the <code class="docutils literal notranslate"><span class="pre">on</span></code> fields.</p></li> |
| <li><p><strong>join_keys</strong> – Tuple of two lists of column names to join on. [Deprecated]</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after join.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Inner-join two DataFrames on a shared column:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"val"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="s2">"label"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">,</span> <span class="s2">"d"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">left</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">right</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'id': [2, 3], 'val': [20, 30], 'label': ['b', 'c']}</span> |
| </pre></div> |
| </div> |
| <p>Left join to keep all rows from the left side:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">left</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">right</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">"left"</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'id': [1, 2, 3], 'val': [10, 20, 30], 'label': [None, 'b', 'c']}</span> |
| </pre></div> |
| </div> |
| <p>Use <code class="docutils literal notranslate"><span class="pre">left_on</span></code> / <code class="docutils literal notranslate"><span class="pre">right_on</span></code> when the key columns differ in name:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">right2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"rid"</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"label"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"b"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">left</span><span class="o">.</span><span class="n">join</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">right2</span><span class="p">,</span> <span class="n">left_on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">right_on</span><span class="o">=</span><span class="s2">"rid"</span> |
| <span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'id': [2, 3], 'val': [20, 30], 'rid': [2, 3], 'label': ['b', 'c']}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.join_on"> |
| <span class="sig-name descname"><span class="pre">join_on</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">on_exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">how</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'inner'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'full'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'semi'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'anti'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'inner'</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.join_on" title="Link to this definition">#</a></dt> |
| <dd><p>Join two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> using the specified expressions.</p> |
| <p>Join predicates must be <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> objects, typically |
| built with <a class="reference internal" href="../index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.col()</span></code></a>. On expressions are used to support |
| in-equality predicates. Equality predicates are correctly optimized.</p> |
| <p>Use <a class="reference internal" href="#datafusion.dataframe.DataFrame.col" title="datafusion.dataframe.DataFrame.col"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.col()</span></code></a> on each DataFrame <strong>before</strong> the join to |
| obtain fully qualified column references. These qualified references |
| can then be used in the join predicate and to disambiguate columns |
| with the same name when selecting from the result.</p> |
| <p class="rubric">Examples</p> |
| <p>Join with unique column names:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"c"</span><span class="p">,</span> <span class="s2">"d"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">left</span><span class="o">.</span><span class="n">join_on</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">right</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">==</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"x"</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2], 'x': ['a', 'b'], 'b': [1, 2], 'y': ['c', 'd']}</span> |
| </pre></div> |
| </div> |
| <p>Use <a class="reference internal" href="#datafusion.dataframe.DataFrame.col" title="datafusion.dataframe.DataFrame.col"><code class="xref py py-meth docutils literal notranslate"><span class="pre">col()</span></code></a> to disambiguate shared column names:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"val"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"val"</span><span class="p">:</span> <span class="p">[</span><span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">joined</span> <span class="o">=</span> <span class="n">left</span><span class="o">.</span><span class="n">join_on</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">right</span><span class="p">,</span> <span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span> <span class="o">==</span> <span class="n">right</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">),</span> <span class="n">how</span><span class="o">=</span><span class="s2">"inner"</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">joined</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">),</span> <span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"val"</span><span class="p">),</span> <span class="n">right</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"val"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"rval"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'id': [1, 2], 'val': [10, 20], 'rval': [30, 40]}</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>right</strong> – Other DataFrame to join with.</p></li> |
| <li><p><strong>on_exprs</strong> – single or multiple (in)-equality predicates.</p></li> |
| <li><p><strong>how</strong> – Type of join to perform. Supported types are “inner”, “left”, |
| “right”, “full”, “semi”, “anti”.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after join.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.limit"> |
| <span class="sig-name descname"><span class="pre">limit</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">count</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">offset</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">0</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.limit" title="Link to this definition">#</a></dt> |
| <dd><p>Return a new <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with a limited number of rows.</p> |
| <p>Results are returned in unspecified order unless the DataFrame is |
| explicitly sorted first via <a class="reference internal" href="#datafusion.dataframe.DataFrame.sort" title="datafusion.dataframe.DataFrame.sort"><code class="xref py py-meth docutils literal notranslate"><span class="pre">sort()</span></code></a> or <a class="reference internal" href="#datafusion.dataframe.DataFrame.sort_by" title="datafusion.dataframe.DataFrame.sort_by"><code class="xref py py-meth docutils literal notranslate"><span class="pre">sort_by()</span></code></a>.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>count</strong> – Number of rows to limit the DataFrame to.</p></li> |
| <li><p><strong>offset</strong> – Number of rows to skip.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after limiting.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Take the first two rows:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]})</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2]}</span> |
| </pre></div> |
| </div> |
| <p>Skip the first row then take two (paging):</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">offset</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [2, 3]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.logical_plan"> |
| <span class="sig-name descname"><span class="pre">logical_plan</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../plan/index.html#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">datafusion.plan.LogicalPlan</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.logical_plan" title="Link to this definition">#</a></dt> |
| <dd><p>Return the unoptimized <code class="docutils literal notranslate"><span class="pre">LogicalPlan</span></code>.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Unoptimized logical plan.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.optimized_logical_plan"> |
| <span class="sig-name descname"><span class="pre">optimized_logical_plan</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../plan/index.html#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">datafusion.plan.LogicalPlan</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.optimized_logical_plan" title="Link to this definition">#</a></dt> |
| <dd><p>Return the optimized <code class="docutils literal notranslate"><span class="pre">LogicalPlan</span></code>.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Optimized logical plan.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.parse_sql_expr"> |
| <span class="sig-name descname"><span class="pre">parse_sql_expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.parse_sql_expr" title="Link to this definition">#</a></dt> |
| <dd><p>Creates logical expression from a SQL query text.</p> |
| <p>The expression is created and processed against the current schema.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">expr</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">parse_sql_expr</span><span class="p">(</span><span class="s2">"a > 1"</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [2, 3]}</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>expr</strong> – Expression string to be converted to datafusion expression</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>Logical expression .</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.repartition"> |
| <span class="sig-name descname"><span class="pre">repartition</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">num</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.repartition" title="Link to this definition">#</a></dt> |
| <dd><p>Repartition a DataFrame into <code class="docutils literal notranslate"><span class="pre">num</span></code> partitions.</p> |
| <p>The batches allocation uses a round-robin algorithm.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>num</strong> – Number of partitions to repartition the DataFrame into.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>Repartitioned DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.repartition_by_hash"> |
| <span class="sig-name descname"><span class="pre">repartition_by_hash</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">num</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.repartition_by_hash" title="Link to this definition">#</a></dt> |
| <dd><p>Repartition a DataFrame using a hash partitioning scheme.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>exprs</strong> – Expressions or a SQL expression string to evaluate |
| and perform hashing on.</p></li> |
| <li><p><strong>num</strong> – Number of partitions to repartition the DataFrame into.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>Repartitioned DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.schema"> |
| <span class="sig-name descname"><span class="pre">schema</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.Schema</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.schema" title="Link to this definition">#</a></dt> |
| <dd><p>Return the <code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow.Schema</span></code> of this DataFrame.</p> |
| <p>The output schema contains information on the name, data type, and |
| nullability for each column.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Describing schema of the DataFrame</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.select"> |
| <span class="sig-name descname"><span class="pre">select</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.select" title="Link to this definition">#</a></dt> |
| <dd><p>Project arbitrary expressions into a new <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p>String arguments are treated as column names; <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> |
| arguments can reshape, rename, or compute new columns.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>exprs</strong> – Either column names or <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> to select.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after projection. It has one column for each expression.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Select columns by name:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3]}</span> |
| </pre></div> |
| </div> |
| <p>Mix column names, expressions, and aliases. The string <code class="docutils literal notranslate"><span class="pre">"a"</span></code> selects |
| column <code class="docutils literal notranslate"><span class="pre">a</span></code> directly; <code class="docutils literal notranslate"><span class="pre">col("a").alias("alternate_a")</span></code> returns a |
| duplicate under a new name:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"a"</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"alternate_a"</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3], 'b': [10, 20, 30], 'alternate_a': [1, 2, 3]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.select_exprs"> |
| <span class="sig-name descname"><span class="pre">select_exprs</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.select_exprs" title="Link to this definition">#</a></dt> |
| <dd><p>Project arbitrary list of expression strings into a new DataFrame.</p> |
| <p>This method will parse string expressions into logical plan expressions. |
| The output DataFrame has one column for each expression.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>DataFrame only containing the specified columns.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.show"> |
| <span class="sig-name descname"><span class="pre">show</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">num</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">20</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.show" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the DataFrame and print the result to the console.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>num</strong> – Number of lines to show.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.sort"> |
| <span class="sig-name descname"><span class="pre">sort</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion.expr.SortKey</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.sort" title="Link to this definition">#</a></dt> |
| <dd><p>Sort the DataFrame by the specified sorting expressions or column names.</p> |
| <p>Note that any expression can be turned into a sort expression by |
| calling its <code class="docutils literal notranslate"><span class="pre">sort</span></code> method. For ascending-only sorts, the shorter |
| <a class="reference internal" href="#datafusion.dataframe.DataFrame.sort_by" title="datafusion.dataframe.DataFrame.sort_by"><code class="xref py py-meth docutils literal notranslate"><span class="pre">sort_by()</span></code></a> is usually more convenient.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>exprs</strong> – Sort expressions or column names, applied in order.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after sorting.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Sort ascending by a column name:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3], 'b': [20, 30, 10]}</span> |
| </pre></div> |
| </div> |
| <p>Sort descending using <code class="xref py py-meth docutils literal notranslate"><span class="pre">Expr.sort()</span></code>:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">ascending</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [3, 2, 1], 'b': [10, 30, 20]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.sort_by"> |
| <span class="sig-name descname"><span class="pre">sort_by</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.sort_by" title="Link to this definition">#</a></dt> |
| <dd><p>Sort the DataFrame by column expressions in ascending order.</p> |
| <p>This is a convenience method that sorts the DataFrame by the given |
| expressions in ascending order with nulls last. For more control over |
| sort direction and null ordering, use <a class="reference internal" href="#datafusion.dataframe.DataFrame.sort" title="datafusion.dataframe.DataFrame.sort"><code class="xref py py-meth docutils literal notranslate"><span class="pre">sort()</span></code></a> instead.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>exprs</strong> – Expressions or column names to sort by.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after sorting.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Sort by a single column:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">sort_by</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.tail"> |
| <span class="sig-name descname"><span class="pre">tail</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">n</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">5</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.tail" title="Link to this definition">#</a></dt> |
| <dd><p>Return a new <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with a limited number of rows.</p> |
| <p>Be aware this could be potentially expensive since the row size needs to be |
| determined of the dataframe. This is done by collecting it.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>n</strong> – Number of rows to take from the tail of the DataFrame.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after limiting.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.to_arrow_table"> |
| <span class="sig-name descname"><span class="pre">to_arrow_table</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pyarrow.Table</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.to_arrow_table" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and convert it into an Arrow Table.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Arrow Table.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.to_pandas"> |
| <span class="sig-name descname"><span class="pre">to_pandas</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">pandas.DataFrame</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.to_pandas" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and convert it into a Pandas DataFrame.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Pandas DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.to_polars"> |
| <span class="sig-name descname"><span class="pre">to_polars</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">polars.DataFrame</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.to_polars" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and convert it into a Polars DataFrame.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Polars DataFrame.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.to_pydict"> |
| <span class="sig-name descname"><span class="pre">to_pydict</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.to_pydict" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and convert it into a dictionary of lists.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Dictionary of lists.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.to_pylist"> |
| <span class="sig-name descname"><span class="pre">to_pylist</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.to_pylist" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and convert it into a list of dictionaries.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>List of dictionaries.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.transform"> |
| <span class="sig-name descname"><span class="pre">transform</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">func</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">collections.abc.Callable</span><span class="p"><span class="pre">[</span></span><span class="pre">Ellipsis</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.transform" title="Link to this definition">#</a></dt> |
| <dd><p>Apply a function to the current DataFrame which returns another DataFrame.</p> |
| <p>This is useful for chaining together multiple functions.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="k">def</span><span class="w"> </span><span class="nf">add_3</span><span class="p">(</span><span class="n">df</span><span class="p">):</span> |
| <span class="gp">... </span> <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">with_column</span><span class="p">(</span><span class="s2">"modified"</span><span class="p">,</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="k">def</span><span class="w"> </span><span class="nf">within_limit</span><span class="p">(</span><span class="n">df</span><span class="p">:</span> <span class="n">DataFrame</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">DataFrame</span><span class="p">:</span> |
| <span class="gp">... </span> <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o"><</span> <span class="n">lit</span><span class="p">(</span><span class="n">limit</span><span class="p">))</span><span class="o">.</span><span class="n">distinct</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">add_3</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">within_limit</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3], 'modified': [3, 3, 3]}</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>func</strong> – A callable function that takes a DataFrame as it’s first argument</p></li> |
| <li><p><strong>args</strong> – Zero or more arguments to pass to <cite>func</cite></p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>After applying func to the original dataframe.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame">DataFrame</a></p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.union"> |
| <span class="sig-name descname"><span class="pre">union</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.union" title="Link to this definition">#</a></dt> |
| <dd><p>Calculate the union of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <p>The two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> must have exactly the same schema.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>other</strong> – DataFrame to union with.</p></li> |
| <li><p><strong>distinct</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, duplicate rows will be removed.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after union.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Stack rows from both DataFrames, preserving duplicates:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">union</span><span class="p">(</span><span class="n">df2</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 2, 3]}</span> |
| </pre></div> |
| </div> |
| <p>Deduplicate the combined result with <code class="docutils literal notranslate"><span class="pre">distinct=True</span></code>:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">union</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.union_by_name"> |
| <span class="sig-name descname"><span class="pre">union_by_name</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.union_by_name" title="Link to this definition">#</a></dt> |
| <dd><p>Union two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> matching columns by name.</p> |
| <p>Unlike <a class="reference internal" href="#datafusion.dataframe.DataFrame.union" title="datafusion.dataframe.DataFrame.union"><code class="xref py py-meth docutils literal notranslate"><span class="pre">union()</span></code></a> which matches columns by position, this method |
| matches columns by their names, allowing DataFrames with different |
| column orders to be combined.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>other</strong> – DataFrame to union with.</p></li> |
| <li><p><strong>distinct</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, duplicate rows are removed from the result.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame after union by name.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Combine DataFrames with different column orders:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">20</span><span class="p">],</span> <span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">2</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">union_by_name</span><span class="p">(</span><span class="n">df2</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2], 'b': [10, 20]}</span> |
| </pre></div> |
| </div> |
| <p>Union by name with deduplication:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">],</span> <span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">union_by_name</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1], 'b': [10]}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.union_distinct"> |
| <span class="sig-name descname"><span class="pre">union_distinct</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.union_distinct" title="Link to this definition">#</a></dt> |
| <dd><p>Calculate the distinct union of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p> |
| <div class="admonition seealso"> |
| <p class="admonition-title">See also</p> |
| <p><a class="reference internal" href="#datafusion.dataframe.DataFrame.union" title="datafusion.dataframe.DataFrame.union"><code class="xref py py-meth docutils literal notranslate"><span class="pre">union()</span></code></a></p> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.unnest_columns"> |
| <span class="sig-name descname"><span class="pre">unnest_columns</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">columns</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">preserve_nulls</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">recursions</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">int</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.unnest_columns" title="Link to this definition">#</a></dt> |
| <dd><p>Expand columns of arrays into a single row per array element.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>columns</strong> – Column names to perform unnest operation on.</p></li> |
| <li><p><strong>preserve_nulls</strong> – If False, rows with null entries will not be |
| returned.</p></li> |
| <li><p><strong>recursions</strong> – Optional list of <code class="docutils literal notranslate"><span class="pre">(input_column,</span> <span class="pre">output_column,</span> <span class="pre">depth)</span></code> |
| tuples that control how deeply nested columns are unnested. Any |
| column not mentioned here is unnested with depth 1.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A DataFrame with the columns expanded.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Unnest an array column:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">]],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"x"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">unnest_columns</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3], 'b': ['x', 'x', 'y']}</span> |
| </pre></div> |
| </div> |
| <p>With explicit recursion depth:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">unnest_columns</span><span class="p">(</span><span class="s2">"a"</span><span class="p">,</span> <span class="n">recursions</span><span class="o">=</span><span class="p">[(</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"a"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)])</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2, 3], 'b': ['x', 'x', 'y']}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.window"> |
| <span class="sig-name descname"><span class="pre">window</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.window" title="Link to this definition">#</a></dt> |
| <dd><p>Add window function columns to the DataFrame.</p> |
| <p>Applies the given window function expressions and appends the results |
| as new columns.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>exprs</strong> – Window function expressions to evaluate.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with new window function columns appended.</p> |
| </dd> |
| </dl> |
| <p class="rubric">Examples</p> |
| <p>Add a row number within each group:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">datafusion.functions</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">f</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"x"</span><span class="p">,</span> <span class="s2">"x"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">window</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">f</span><span class="o">.</span><span class="n">row_number</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">partition_by</span><span class="o">=</span><span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)],</span> <span class="n">order_by</span><span class="o">=</span><span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)]</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"rn"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="s2">"rn"</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">names</span> |
| <span class="go">True</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.with_column"> |
| <span class="sig-name descname"><span class="pre">with_column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.with_column" title="Link to this definition">#</a></dt> |
| <dd><p>Add an additional column to the DataFrame.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">expr</span></code> must be an <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> constructed with |
| <a class="reference internal" href="../index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.col()</span></code></a> or <a class="reference internal" href="../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.lit()</span></code></a>, or a SQL expression |
| string that will be parsed against the DataFrame schema.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">with_column</span><span class="p">(</span><span class="s2">"b"</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">+</span> <span class="n">lit</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> |
| <span class="go">{'a': [1, 2], 'b': [11, 12]}</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>name</strong> – Name of the column to add.</p></li> |
| <li><p><strong>expr</strong> – Expression to compute the column.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with the new column.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.with_column_renamed"> |
| <span class="sig-name descname"><span class="pre">with_column_renamed</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">old_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">new_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.with_column_renamed" title="Link to this definition">#</a></dt> |
| <dd><p>Rename one column by applying a new projection.</p> |
| <p>This is a no-op if the column to be renamed does not exist.</p> |
| <p>The method supports case sensitive rename with wrapping column name |
| into one the following symbols (” or ‘ or `).</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>old_name</strong> – Old column name.</p></li> |
| <li><p><strong>new_name</strong> – New column name.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with the column renamed.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.with_columns"> |
| <span class="sig-name descname"><span class="pre">with_columns</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Iterable</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">named_exprs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.with_columns" title="Link to this definition">#</a></dt> |
| <dd><p>Add columns to the DataFrame.</p> |
| <p>By passing expressions, iterables of expressions, string SQL expressions, |
| or named expressions. |
| All expressions must be <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> objects created via |
| <a class="reference internal" href="../index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.col()</span></code></a> or <a class="reference internal" href="../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.lit()</span></code></a>, or SQL expression strings. |
| To pass named expressions use the form <code class="docutils literal notranslate"><span class="pre">name=Expr</span></code>.</p> |
| <p>Example usage: The following will add 4 columns labeled <code class="docutils literal notranslate"><span class="pre">a</span></code>, <code class="docutils literal notranslate"><span class="pre">b</span></code>, <code class="docutils literal notranslate"><span class="pre">c</span></code>, |
| and <code class="docutils literal notranslate"><span class="pre">d</span></code>:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span><span class="p">,</span> <span class="n">lit</span> |
| <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">with_columns</span><span class="p">(</span> |
| <span class="n">col</span><span class="p">(</span><span class="s2">"x"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> |
| <span class="p">[</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"b"</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"y"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"c"</span><span class="p">)],</span> |
| <span class="n">d</span><span class="o">=</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span> |
| <span class="p">)</span> |
| |
| <span class="n">Equivalent</span> <span class="n">example</span> <span class="n">using</span> <span class="n">just</span> <span class="n">SQL</span> <span class="n">strings</span><span class="p">:</span> |
| |
| <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">with_columns</span><span class="p">(</span> |
| <span class="s2">"x as a"</span><span class="p">,</span> |
| <span class="p">[</span><span class="s2">"1 as b"</span><span class="p">,</span> <span class="s2">"y as c"</span><span class="p">],</span> |
| <span class="n">d</span><span class="o">=</span><span class="s2">"3"</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>exprs</strong> – Either a single expression, an iterable of expressions to add or |
| SQL expression strings.</p></li> |
| <li><p><strong>named_exprs</strong> – Named expressions in the form of <code class="docutils literal notranslate"><span class="pre">name=expr</span></code></p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>DataFrame with the new columns added.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.write_csv"> |
| <span class="sig-name descname"><span class="pre">write_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">with_header</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.write_csv" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and write the results to a CSV file.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>path</strong> – Path of the CSV file to write.</p></li> |
| <li><p><strong>with_header</strong> – If true, output the CSV header row.</p></li> |
| <li><p><strong>write_options</strong> – Options that impact how the DataFrame is written.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.write_json"> |
| <span class="sig-name descname"><span class="pre">write_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.write_json" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and write the results to a JSON file.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>path</strong> – Path of the JSON file to write.</p></li> |
| <li><p><strong>write_options</strong> – Options that impact how the DataFrame is written.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.write_parquet"> |
| <span class="sig-name descname"><span class="pre">write_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression_level</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.write_parquet" title="Link to this definition">#</a></dt> |
| <dt class="sig sig-object py"> |
| <span class="sig-name descname"><span class="pre">write_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.Compression" title="datafusion.dataframe.Compression"><span class="pre">Compression</span></a></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">Compression.ZSTD</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression_level</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span></dt> |
| <dt class="sig sig-object py"> |
| <span class="sig-name descname"><span class="pre">write_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.ParquetWriterOptions" title="datafusion.dataframe.ParquetWriterOptions"><span class="pre">ParquetWriterOptions</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression_level</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and write the results to a Parquet file.</p> |
| <p>Available compression types are:</p> |
| <ul class="simple"> |
| <li><p>“uncompressed”: No compression.</p></li> |
| <li><p>“snappy”: Snappy compression.</p></li> |
| <li><p>“gzip”: Gzip compression.</p></li> |
| <li><p>“brotli”: Brotli compression.</p></li> |
| <li><p>“lz4”: LZ4 compression.</p></li> |
| <li><p>“lz4_raw”: LZ4_RAW compression.</p></li> |
| <li><p>“zstd”: Zstandard compression.</p></li> |
| </ul> |
| <p>LZO compression is not yet implemented in arrow-rs and is therefore |
| excluded.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>path</strong> – Path of the Parquet file to write.</p></li> |
| <li><p><strong>compression</strong> – Compression type to use. Default is “ZSTD”.</p></li> |
| <li><p><strong>compression_level</strong> – Compression level to use. For ZSTD, the |
| recommended range is 1 to 22, with the default being 4. Higher levels |
| provide better compression but slower speed.</p></li> |
| <li><p><strong>write_options</strong> – Options that impact how the DataFrame is written.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.write_parquet_with_options"> |
| <span class="sig-name descname"><span class="pre">write_parquet_with_options</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.ParquetWriterOptions" title="datafusion.dataframe.ParquetWriterOptions"><span class="pre">ParquetWriterOptions</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.write_parquet_with_options" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and write the results to a Parquet file.</p> |
| <p>Allows advanced writer options to be set with <cite>ParquetWriterOptions</cite>.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>path</strong> – Path of the Parquet file to write.</p></li> |
| <li><p><strong>options</strong> – Sets the writer parquet options (see <cite>ParquetWriterOptions</cite>).</p></li> |
| <li><p><strong>write_options</strong> – Options that impact how the DataFrame is written.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.write_table"> |
| <span class="sig-name descname"><span class="pre">write_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><span class="pre">DataFrameWriteOptions</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.write_table" title="Link to this definition">#</a></dt> |
| <dd><p>Execute the <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> and write the results to a table.</p> |
| <p>The table must be registered with the session to perform this operation. |
| Not all table providers support writing operations. See the individual |
| implementations for details.</p> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.df"> |
| <span class="sig-name descname"><span class="pre">df</span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.df" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrameWriteOptions"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">DataFrameWriteOptions</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">insert_operation</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.InsertOp" title="datafusion.dataframe.InsertOp"><span class="pre">InsertOp</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">single_file_output</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">partition_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">sort_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="../expr/index.html#datafusion.expr.SortExpr" title="datafusion.expr.SortExpr"><span class="pre">datafusion.expr.SortExpr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.SortExpr" title="datafusion.expr.SortExpr"><span class="pre">datafusion.expr.SortExpr</span></a><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.dataframe.DataFrameWriteOptions" title="Link to this definition">#</a></dt> |
| <dd><p>Writer options for DataFrame.</p> |
| <p>There is no guarantee the table provider supports all writer options. |
| See the individual implementation and documentation for details.</p> |
| <p>Instantiate writer options for DataFrame.</p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.DataFrameWriteOptions._raw_write_options"> |
| <span class="sig-name descname"><span class="pre">_raw_write_options</span></span><a class="headerlink" href="#datafusion.dataframe.DataFrameWriteOptions._raw_write_options" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ExplainFormat"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">ExplainFormat</span></span><a class="headerlink" href="#datafusion.dataframe.ExplainFormat" title="Link to this definition">#</a></dt> |
| <dd><p>Bases: <code class="xref py py-obj docutils literal notranslate"><span class="pre">enum.Enum</span></code></p> |
| <p>Output format for explain plans.</p> |
| <p>Controls how the query plan is rendered in <a class="reference internal" href="#datafusion.dataframe.DataFrame.explain" title="datafusion.dataframe.DataFrame.explain"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.explain()</span></code></a>.</p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ExplainFormat.GRAPHVIZ"> |
| <span class="sig-name descname"><span class="pre">GRAPHVIZ</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'graphviz'</span></em><a class="headerlink" href="#datafusion.dataframe.ExplainFormat.GRAPHVIZ" title="Link to this definition">#</a></dt> |
| <dd><p>Graphviz DOT format for graph rendering.</p> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ExplainFormat.INDENT"> |
| <span class="sig-name descname"><span class="pre">INDENT</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'indent'</span></em><a class="headerlink" href="#datafusion.dataframe.ExplainFormat.INDENT" title="Link to this definition">#</a></dt> |
| <dd><p>Default indented text format.</p> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ExplainFormat.PGJSON"> |
| <span class="sig-name descname"><span class="pre">PGJSON</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'pgjson'</span></em><a class="headerlink" href="#datafusion.dataframe.ExplainFormat.PGJSON" title="Link to this definition">#</a></dt> |
| <dd><p>PostgreSQL-compatible JSON format for use with visualization tools.</p> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ExplainFormat.TREE"> |
| <span class="sig-name descname"><span class="pre">TREE</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'tree'</span></em><a class="headerlink" href="#datafusion.dataframe.ExplainFormat.TREE" title="Link to this definition">#</a></dt> |
| <dd><p>Tree-style visual format with box-drawing characters.</p> |
| </dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.InsertOp"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">InsertOp</span></span><a class="headerlink" href="#datafusion.dataframe.InsertOp" title="Link to this definition">#</a></dt> |
| <dd><p>Bases: <code class="xref py py-obj docutils literal notranslate"><span class="pre">enum.Enum</span></code></p> |
| <p>Insert operation mode.</p> |
| <p>These modes are used by the table writing feature to define how record |
| batches should be written to a table.</p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.InsertOp.APPEND"> |
| <span class="sig-name descname"><span class="pre">APPEND</span></span><a class="headerlink" href="#datafusion.dataframe.InsertOp.APPEND" title="Link to this definition">#</a></dt> |
| <dd><p>Appends new rows to the existing table without modifying any existing rows.</p> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.InsertOp.OVERWRITE"> |
| <span class="sig-name descname"><span class="pre">OVERWRITE</span></span><a class="headerlink" href="#datafusion.dataframe.InsertOp.OVERWRITE" title="Link to this definition">#</a></dt> |
| <dd><p>Overwrites all existing rows in the table with the new rows.</p> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.InsertOp.REPLACE"> |
| <span class="sig-name descname"><span class="pre">REPLACE</span></span><a class="headerlink" href="#datafusion.dataframe.InsertOp.REPLACE" title="Link to this definition">#</a></dt> |
| <dd><p>Replace existing rows that collide with the inserted rows.</p> |
| <p>Replacement is typically based on a unique key or primary key.</p> |
| </dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">ParquetColumnOptions</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dictionary_enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">statistics_enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bloom_filter_enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bloom_filter_fpp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">float</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bloom_filter_ndv</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions" title="Link to this definition">#</a></dt> |
| <dd><p>Parquet options for individual columns.</p> |
| <p>Contains the available options that can be applied for an individual Parquet column, |
| replacing the global options in <code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions</span></code>.</p> |
| <p>Initialize the ParquetColumnOptions.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>encoding</strong> – Sets encoding for the column path. Valid values are: <code class="docutils literal notranslate"><span class="pre">plain</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">plain_dictionary</span></code>, <code class="docutils literal notranslate"><span class="pre">rle</span></code>, <code class="docutils literal notranslate"><span class="pre">bit_packed</span></code>, <code class="docutils literal notranslate"><span class="pre">delta_binary_packed</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">delta_length_byte_array</span></code>, <code class="docutils literal notranslate"><span class="pre">delta_byte_array</span></code>, <code class="docutils literal notranslate"><span class="pre">rle_dictionary</span></code>, |
| and <code class="docutils literal notranslate"><span class="pre">byte_stream_split</span></code>. These values are not case-sensitive. If |
| <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the default parquet options</p></li> |
| <li><p><strong>dictionary_enabled</strong> – Sets if dictionary encoding is enabled for the column |
| path. If <cite>None</cite>, uses the default parquet options</p></li> |
| <li><p><strong>compression</strong> – Sets default parquet compression codec for the column path. |
| Valid values are <code class="docutils literal notranslate"><span class="pre">uncompressed</span></code>, <code class="docutils literal notranslate"><span class="pre">snappy</span></code>, <code class="docutils literal notranslate"><span class="pre">gzip(level)</span></code>, <code class="docutils literal notranslate"><span class="pre">lzo</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">brotli(level)</span></code>, <code class="docutils literal notranslate"><span class="pre">lz4</span></code>, <code class="docutils literal notranslate"><span class="pre">zstd(level)</span></code>, and <code class="docutils literal notranslate"><span class="pre">lz4_raw</span></code>. These |
| values are not case-sensitive. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the default parquet |
| options.</p></li> |
| <li><p><strong>statistics_enabled</strong> – Sets if statistics are enabled for the column Valid |
| values are: <code class="docutils literal notranslate"><span class="pre">none</span></code>, <code class="docutils literal notranslate"><span class="pre">chunk</span></code>, and <code class="docutils literal notranslate"><span class="pre">page</span></code> These values are not case |
| sensitive. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the default parquet options.</p></li> |
| <li><p><strong>bloom_filter_enabled</strong> – Sets if bloom filter is enabled for the column path. |
| If <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the default parquet options.</p></li> |
| <li><p><strong>bloom_filter_fpp</strong> – Sets bloom filter false positive probability for the |
| column path. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the default parquet options.</p></li> |
| <li><p><strong>bloom_filter_ndv</strong> – Sets bloom filter number of distinct values. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, |
| uses the default parquet options.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.bloom_filter_enabled"> |
| <span class="sig-name descname"><span class="pre">bloom_filter_enabled</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.bloom_filter_enabled" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.bloom_filter_fpp"> |
| <span class="sig-name descname"><span class="pre">bloom_filter_fpp</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.bloom_filter_fpp" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.bloom_filter_ndv"> |
| <span class="sig-name descname"><span class="pre">bloom_filter_ndv</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.bloom_filter_ndv" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.compression"> |
| <span class="sig-name descname"><span class="pre">compression</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.compression" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.dictionary_enabled"> |
| <span class="sig-name descname"><span class="pre">dictionary_enabled</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.dictionary_enabled" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.encoding"> |
| <span class="sig-name descname"><span class="pre">encoding</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.encoding" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetColumnOptions.statistics_enabled"> |
| <span class="sig-name descname"><span class="pre">statistics_enabled</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetColumnOptions.statistics_enabled" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.dataframe.</span></span><span class="sig-name descname"><span class="pre">ParquetWriterOptions</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data_pagesize_limit</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1024</span> <span class="pre">*</span> <span class="pre">1024</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_batch_size</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1024</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">writer_version</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'1.0'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_arrow_metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'zstd(3)'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression_level</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dictionary_enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dictionary_page_size_limit</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1024</span> <span class="pre">*</span> <span class="pre">1024</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">statistics_enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'page'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_row_group_size</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1024</span> <span class="pre">*</span> <span class="pre">1024</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">created_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'datafusion-python'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">column_index_truncate_length</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">64</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">statistics_truncate_length</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">data_page_row_count_limit</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">20000</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bloom_filter_on_write</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bloom_filter_fpp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">float</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bloom_filter_ndv</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">allow_single_file_parallelism</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">maximum_parallel_row_group_writers</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">maximum_buffered_record_batches_per_stream</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">2</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">column_specific_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#datafusion.dataframe.ParquetColumnOptions" title="datafusion.dataframe.ParquetColumnOptions"><span class="pre">ParquetColumnOptions</span></a><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions" title="Link to this definition">#</a></dt> |
| <dd><p>Advanced parquet writer options.</p> |
| <p>Allows settings the writer options that apply to the entire file. Some options can |
| also be set on a column by column basis, with the field <code class="docutils literal notranslate"><span class="pre">column_specific_options</span></code> |
| (see <code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions</span></code>).</p> |
| <p>Initialize the ParquetWriterOptions.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>data_pagesize_limit</strong> – Sets best effort maximum size of data page in bytes.</p></li> |
| <li><p><strong>write_batch_size</strong> – Sets write_batch_size in bytes.</p></li> |
| <li><p><strong>writer_version</strong> – Sets parquet writer version. Valid values are <code class="docutils literal notranslate"><span class="pre">1.0</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">2.0</span></code>.</p></li> |
| <li><p><strong>skip_arrow_metadata</strong> – Skip encoding the embedded arrow metadata in the |
| KV_meta.</p></li> |
| <li><p><strong>compression</strong> – <p>Compression type to use. Default is <code class="docutils literal notranslate"><span class="pre">zstd(3)</span></code>. |
| Available compression types are</p> |
| <ul> |
| <li><p><code class="docutils literal notranslate"><span class="pre">uncompressed</span></code>: No compression.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">snappy</span></code>: Snappy compression.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">gzip(n)</span></code>: Gzip compression with level n.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">brotli(n)</span></code>: Brotli compression with level n.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">lz4</span></code>: LZ4 compression.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">lz4_raw</span></code>: LZ4_RAW compression.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">zstd(n)</span></code>: Zstandard compression with level n.</p></li> |
| </ul> |
| </p></li> |
| <li><p><strong>compression_level</strong> – Compression level to set.</p></li> |
| <li><p><strong>dictionary_enabled</strong> – Sets if dictionary encoding is enabled. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, |
| uses the default parquet writer setting.</p></li> |
| <li><p><strong>dictionary_page_size_limit</strong> – Sets best effort maximum dictionary page size, |
| in bytes.</p></li> |
| <li><p><strong>statistics_enabled</strong> – Sets if statistics are enabled for any column Valid |
| values are <code class="docutils literal notranslate"><span class="pre">none</span></code>, <code class="docutils literal notranslate"><span class="pre">chunk</span></code>, and <code class="docutils literal notranslate"><span class="pre">page</span></code>. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the |
| default parquet writer setting.</p></li> |
| <li><p><strong>max_row_group_size</strong> – Target maximum number of rows in each row group |
| (defaults to 1M rows). Writing larger row groups requires more memory |
| to write, but can get better compression and be faster to read.</p></li> |
| <li><p><strong>created_by</strong> – Sets “created by” property.</p></li> |
| <li><p><strong>column_index_truncate_length</strong> – Sets column index truncate length.</p></li> |
| <li><p><strong>statistics_truncate_length</strong> – Sets statistics truncate length. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, |
| uses the default parquet writer setting.</p></li> |
| <li><p><strong>data_page_row_count_limit</strong> – Sets best effort maximum number of rows in a data |
| page.</p></li> |
| <li><p><strong>encoding</strong> – Sets default encoding for any column. Valid values are <code class="docutils literal notranslate"><span class="pre">plain</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">plain_dictionary</span></code>, <code class="docutils literal notranslate"><span class="pre">rle</span></code>, <code class="docutils literal notranslate"><span class="pre">bit_packed</span></code>, <code class="docutils literal notranslate"><span class="pre">delta_binary_packed</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">delta_length_byte_array</span></code>, <code class="docutils literal notranslate"><span class="pre">delta_byte_array</span></code>, <code class="docutils literal notranslate"><span class="pre">rle_dictionary</span></code>, |
| and <code class="docutils literal notranslate"><span class="pre">byte_stream_split</span></code>. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, uses the default parquet writer |
| setting.</p></li> |
| <li><p><strong>bloom_filter_on_write</strong> – Write bloom filters for all columns when creating |
| parquet files.</p></li> |
| <li><p><strong>bloom_filter_fpp</strong> – Sets bloom filter false positive probability. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, |
| uses the default parquet writer setting</p></li> |
| <li><p><strong>bloom_filter_ndv</strong> – Sets bloom filter number of distinct values. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, |
| uses the default parquet writer setting.</p></li> |
| <li><p><strong>allow_single_file_parallelism</strong> – Controls whether DataFusion will attempt to |
| speed up writing parquet files by serializing them in parallel. Each |
| column in each row group in each output file are serialized in parallel |
| leveraging a maximum possible core count of |
| <code class="docutils literal notranslate"><span class="pre">n_files</span> <span class="pre">*</span> <span class="pre">n_row_groups</span> <span class="pre">*</span> <span class="pre">n_columns</span></code>.</p></li> |
| <li><p><strong>maximum_parallel_row_group_writers</strong> – By default parallel parquet writer is |
| tuned for minimum memory usage in a streaming execution plan. You may |
| see a performance benefit when writing large parquet files by increasing |
| <code class="docutils literal notranslate"><span class="pre">maximum_parallel_row_group_writers</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">maximum_buffered_record_batches_per_stream</span></code> if your system has idle |
| cores and can tolerate additional memory usage. Boosting these values is |
| likely worthwhile when writing out already in-memory data, such as from |
| a cached data frame.</p></li> |
| <li><p><strong>maximum_buffered_record_batches_per_stream</strong> – See |
| <code class="docutils literal notranslate"><span class="pre">maximum_parallel_row_group_writers</span></code>.</p></li> |
| <li><p><strong>column_specific_options</strong> – Overrides options for specific columns. If a column |
| is not a part of this dictionary, it will use the parameters provided |
| here.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.allow_single_file_parallelism"> |
| <span class="sig-name descname"><span class="pre">allow_single_file_parallelism</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">True</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.allow_single_file_parallelism" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.bloom_filter_fpp"> |
| <span class="sig-name descname"><span class="pre">bloom_filter_fpp</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.bloom_filter_fpp" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.bloom_filter_ndv"> |
| <span class="sig-name descname"><span class="pre">bloom_filter_ndv</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.bloom_filter_ndv" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.bloom_filter_on_write"> |
| <span class="sig-name descname"><span class="pre">bloom_filter_on_write</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">False</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.bloom_filter_on_write" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.column_index_truncate_length"> |
| <span class="sig-name descname"><span class="pre">column_index_truncate_length</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">64</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.column_index_truncate_length" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.column_specific_options"> |
| <span class="sig-name descname"><span class="pre">column_specific_options</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.column_specific_options" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.created_by"> |
| <span class="sig-name descname"><span class="pre">created_by</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'datafusion-python'</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.created_by" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.data_page_row_count_limit"> |
| <span class="sig-name descname"><span class="pre">data_page_row_count_limit</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">20000</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.data_page_row_count_limit" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.data_pagesize_limit"> |
| <span class="sig-name descname"><span class="pre">data_pagesize_limit</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">1048576</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.data_pagesize_limit" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.dictionary_enabled"> |
| <span class="sig-name descname"><span class="pre">dictionary_enabled</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">True</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.dictionary_enabled" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.dictionary_page_size_limit"> |
| <span class="sig-name descname"><span class="pre">dictionary_page_size_limit</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">1048576</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.dictionary_page_size_limit" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.encoding"> |
| <span class="sig-name descname"><span class="pre">encoding</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.encoding" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.max_row_group_size"> |
| <span class="sig-name descname"><span class="pre">max_row_group_size</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">1048576</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.max_row_group_size" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.maximum_buffered_record_batches_per_stream"> |
| <span class="sig-name descname"><span class="pre">maximum_buffered_record_batches_per_stream</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">2</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.maximum_buffered_record_batches_per_stream" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.maximum_parallel_row_group_writers"> |
| <span class="sig-name descname"><span class="pre">maximum_parallel_row_group_writers</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">1</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.maximum_parallel_row_group_writers" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.skip_arrow_metadata"> |
| <span class="sig-name descname"><span class="pre">skip_arrow_metadata</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">False</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.skip_arrow_metadata" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.statistics_enabled"> |
| <span class="sig-name descname"><span class="pre">statistics_enabled</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'page'</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.statistics_enabled" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.statistics_truncate_length"> |
| <span class="sig-name descname"><span class="pre">statistics_truncate_length</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.statistics_truncate_length" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.write_batch_size"> |
| <span class="sig-name descname"><span class="pre">write_batch_size</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">1024</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.write_batch_size" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="datafusion.dataframe.ParquetWriterOptions.writer_version"> |
| <span class="sig-name descname"><span class="pre">writer_version</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'1.0'</span></em><a class="headerlink" href="#datafusion.dataframe.ParquetWriterOptions.writer_version" title="Link to this definition">#</a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="../context/index.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">datafusion.context</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="../dataframe_formatter/index.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">datafusion.dataframe_formatter</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#classes">Classes</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#module-contents">Module Contents</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression"><code class="docutils literal notranslate"><span class="pre">Compression</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.from_str"><code class="docutils literal notranslate"><span class="pre">Compression.from_str()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.get_default_level"><code class="docutils literal notranslate"><span class="pre">Compression.get_default_level()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.BROTLI"><code class="docutils literal notranslate"><span class="pre">Compression.BROTLI</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.GZIP"><code class="docutils literal notranslate"><span class="pre">Compression.GZIP</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.LZ4"><code class="docutils literal notranslate"><span class="pre">Compression.LZ4</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.LZ4_RAW"><code class="docutils literal notranslate"><span class="pre">Compression.LZ4_RAW</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.SNAPPY"><code class="docutils literal notranslate"><span class="pre">Compression.SNAPPY</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.UNCOMPRESSED"><code class="docutils literal notranslate"><span class="pre">Compression.UNCOMPRESSED</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.Compression.ZSTD"><code class="docutils literal notranslate"><span class="pre">Compression.ZSTD</span></code></a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame"><code class="docutils literal notranslate"><span class="pre">DataFrame</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.__aiter__"><code class="docutils literal notranslate"><span class="pre">DataFrame.__aiter__()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.__arrow_c_stream__"><code class="docutils literal notranslate"><span class="pre">DataFrame.__arrow_c_stream__()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.__getitem__"><code class="docutils literal notranslate"><span class="pre">DataFrame.__getitem__()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.__iter__"><code class="docutils literal notranslate"><span class="pre">DataFrame.__iter__()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.__repr__"><code class="docutils literal notranslate"><span class="pre">DataFrame.__repr__()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame._repr_html_"><code class="docutils literal notranslate"><span class="pre">DataFrame._repr_html_()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.aggregate"><code class="docutils literal notranslate"><span class="pre">DataFrame.aggregate()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.alias"><code class="docutils literal notranslate"><span class="pre">DataFrame.alias()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.cache"><code class="docutils literal notranslate"><span class="pre">DataFrame.cache()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.cast"><code class="docutils literal notranslate"><span class="pre">DataFrame.cast()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.col"><code class="docutils literal notranslate"><span class="pre">DataFrame.col()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.collect"><code class="docutils literal notranslate"><span class="pre">DataFrame.collect()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.collect_column"><code class="docutils literal notranslate"><span class="pre">DataFrame.collect_column()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.collect_partitioned"><code class="docutils literal notranslate"><span class="pre">DataFrame.collect_partitioned()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.column"><code class="docutils literal notranslate"><span class="pre">DataFrame.column()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.count"><code class="docutils literal notranslate"><span class="pre">DataFrame.count()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.default_str_repr"><code class="docutils literal notranslate"><span class="pre">DataFrame.default_str_repr()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.describe"><code class="docutils literal notranslate"><span class="pre">DataFrame.describe()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.distinct"><code class="docutils literal notranslate"><span class="pre">DataFrame.distinct()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.distinct_on"><code class="docutils literal notranslate"><span class="pre">DataFrame.distinct_on()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.drop"><code class="docutils literal notranslate"><span class="pre">DataFrame.drop()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.except_all"><code class="docutils literal notranslate"><span class="pre">DataFrame.except_all()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.execute_stream"><code class="docutils literal notranslate"><span class="pre">DataFrame.execute_stream()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.execute_stream_partitioned"><code class="docutils literal notranslate"><span class="pre">DataFrame.execute_stream_partitioned()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.execution_plan"><code class="docutils literal notranslate"><span class="pre">DataFrame.execution_plan()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.explain"><code class="docutils literal notranslate"><span class="pre">DataFrame.explain()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.fill_null"><code class="docutils literal notranslate"><span class="pre">DataFrame.fill_null()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.filter"><code class="docutils literal notranslate"><span class="pre">DataFrame.filter()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.find_qualified_columns"><code class="docutils literal notranslate"><span class="pre">DataFrame.find_qualified_columns()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.head"><code class="docutils literal notranslate"><span class="pre">DataFrame.head()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.intersect"><code class="docutils literal notranslate"><span class="pre">DataFrame.intersect()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.into_view"><code class="docutils literal notranslate"><span class="pre">DataFrame.into_view()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.join"><code class="docutils literal notranslate"><span class="pre">DataFrame.join()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.join_on"><code class="docutils literal notranslate"><span class="pre">DataFrame.join_on()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.limit"><code class="docutils literal notranslate"><span class="pre">DataFrame.limit()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.logical_plan"><code class="docutils literal notranslate"><span class="pre">DataFrame.logical_plan()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.optimized_logical_plan"><code class="docutils literal notranslate"><span class="pre">DataFrame.optimized_logical_plan()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.parse_sql_expr"><code class="docutils literal notranslate"><span class="pre">DataFrame.parse_sql_expr()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.repartition"><code class="docutils literal notranslate"><span class="pre">DataFrame.repartition()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.repartition_by_hash"><code class="docutils literal notranslate"><span class="pre">DataFrame.repartition_by_hash()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.schema"><code class="docutils literal notranslate"><span class="pre">DataFrame.schema()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.select"><code class="docutils literal notranslate"><span class="pre">DataFrame.select()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.select_exprs"><code class="docutils literal notranslate"><span class="pre">DataFrame.select_exprs()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.show"><code class="docutils literal notranslate"><span class="pre">DataFrame.show()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.sort"><code class="docutils literal notranslate"><span class="pre">DataFrame.sort()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.sort_by"><code class="docutils literal notranslate"><span class="pre">DataFrame.sort_by()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.tail"><code class="docutils literal notranslate"><span class="pre">DataFrame.tail()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.to_arrow_table"><code class="docutils literal notranslate"><span class="pre">DataFrame.to_arrow_table()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.to_pandas"><code class="docutils literal notranslate"><span class="pre">DataFrame.to_pandas()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.to_polars"><code class="docutils literal notranslate"><span class="pre">DataFrame.to_polars()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.to_pydict"><code class="docutils literal notranslate"><span class="pre">DataFrame.to_pydict()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.to_pylist"><code class="docutils literal notranslate"><span class="pre">DataFrame.to_pylist()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.transform"><code class="docutils literal notranslate"><span class="pre">DataFrame.transform()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.union"><code class="docutils literal notranslate"><span class="pre">DataFrame.union()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.union_by_name"><code class="docutils literal notranslate"><span class="pre">DataFrame.union_by_name()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.union_distinct"><code class="docutils literal notranslate"><span class="pre">DataFrame.union_distinct()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.unnest_columns"><code class="docutils literal notranslate"><span class="pre">DataFrame.unnest_columns()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.window"><code class="docutils literal notranslate"><span class="pre">DataFrame.window()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.with_column"><code class="docutils literal notranslate"><span class="pre">DataFrame.with_column()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.with_column_renamed"><code class="docutils literal notranslate"><span class="pre">DataFrame.with_column_renamed()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.with_columns"><code class="docutils literal notranslate"><span class="pre">DataFrame.with_columns()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.write_csv"><code class="docutils literal notranslate"><span class="pre">DataFrame.write_csv()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.write_json"><code class="docutils literal notranslate"><span class="pre">DataFrame.write_json()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.write_parquet"><code class="docutils literal notranslate"><span class="pre">DataFrame.write_parquet()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.write_parquet_with_options"><code class="docutils literal notranslate"><span class="pre">DataFrame.write_parquet_with_options()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.write_table"><code class="docutils literal notranslate"><span class="pre">DataFrame.write_table()</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.df"><code class="docutils literal notranslate"><span class="pre">DataFrame.df</span></code></a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrameWriteOptions"><code class="docutils literal notranslate"><span class="pre">DataFrameWriteOptions</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.DataFrameWriteOptions._raw_write_options"><code class="docutils literal notranslate"><span class="pre">DataFrameWriteOptions._raw_write_options</span></code></a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat"><code class="docutils literal notranslate"><span class="pre">ExplainFormat</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.GRAPHVIZ"><code class="docutils literal notranslate"><span class="pre">ExplainFormat.GRAPHVIZ</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.INDENT"><code class="docutils literal notranslate"><span class="pre">ExplainFormat.INDENT</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.PGJSON"><code class="docutils literal notranslate"><span class="pre">ExplainFormat.PGJSON</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.TREE"><code class="docutils literal notranslate"><span class="pre">ExplainFormat.TREE</span></code></a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.InsertOp"><code class="docutils literal notranslate"><span class="pre">InsertOp</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.InsertOp.APPEND"><code class="docutils literal notranslate"><span class="pre">InsertOp.APPEND</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.InsertOp.OVERWRITE"><code class="docutils literal notranslate"><span class="pre">InsertOp.OVERWRITE</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.InsertOp.REPLACE"><code class="docutils literal notranslate"><span class="pre">InsertOp.REPLACE</span></code></a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.bloom_filter_enabled"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.bloom_filter_enabled</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.bloom_filter_fpp"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.bloom_filter_fpp</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.bloom_filter_ndv"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.bloom_filter_ndv</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.compression"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.compression</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.dictionary_enabled"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.dictionary_enabled</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.encoding"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.encoding</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetColumnOptions.statistics_enabled"><code class="docutils literal notranslate"><span class="pre">ParquetColumnOptions.statistics_enabled</span></code></a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions</span></code></a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.allow_single_file_parallelism"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.allow_single_file_parallelism</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.bloom_filter_fpp"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.bloom_filter_fpp</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.bloom_filter_ndv"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.bloom_filter_ndv</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.bloom_filter_on_write"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.bloom_filter_on_write</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.column_index_truncate_length"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.column_index_truncate_length</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.column_specific_options"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.column_specific_options</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.created_by"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.created_by</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.data_page_row_count_limit"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.data_page_row_count_limit</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.data_pagesize_limit"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.data_pagesize_limit</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.dictionary_enabled"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.dictionary_enabled</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.dictionary_page_size_limit"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.dictionary_page_size_limit</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.encoding"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.encoding</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.max_row_group_size"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.max_row_group_size</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.maximum_buffered_record_batches_per_stream"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.maximum_buffered_record_batches_per_stream</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.maximum_parallel_row_group_writers"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.maximum_parallel_row_group_writers</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.skip_arrow_metadata"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.skip_arrow_metadata</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.statistics_enabled"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.statistics_enabled</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.statistics_truncate_length"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.statistics_truncate_length</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.write_batch_size"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.write_batch_size</span></code></a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.dataframe.ParquetWriterOptions.writer_version"><code class="docutils literal notranslate"><span class="pre">ParquetWriterOptions.writer_version</span></code></a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script> |
| <script defer src="../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script> |
| |
| <!-- Based on pydata_sphinx_theme/footer.html --> |
| <footer class="footer mt-5 mt-md-0"> |
| <div class="container"> |
| |
| <div class="footer-item"> |
| <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p> |
| <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> |
| </div> |
| </div> |
| </footer> |
| |
| |
| </body> |
| </html> |
