| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../../../../" data-theme="auto"> |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>datafusion.functions.spark — Apache DataFusion in Python documentation</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto"; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto"; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| <link href="../../../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../../../_static/pygments.css?v=8f2a1f02" /> |
| <link rel="stylesheet" type="text/css" href="../../../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" /> |
| <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css?v=4ae1632d" /> |
| <link rel="stylesheet" type="text/css" href="../../../../_static/theme_overrides.css?v=4af573bc" /> |
| |
| <!-- So that users can add custom icons --> |
| <script src="../../../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" /> |
| <link rel="preload" as="script" href="../../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" /> |
| |
| <script src="../../../../_static/documentation_options.js?v=5929fcd5"></script> |
| <script src="../../../../_static/doctools.js?v=9bcbadda"></script> |
| <script src="../../../../_static/sphinx_highlight.js?v=dc90522c"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'autoapi/datafusion/functions/spark/index';</script> |
| <script src="../../../../_static/toc-toggle.js?v=36375f01"></script> |
| <link rel="icon" href="../../../../_static/favicon.svg"/> |
| <link rel="index" title="Index" href="../../../../genindex.html" /> |
| <link rel="search" title="Search" href="../../../../search.html" /> |
| <link rel="next" title="datafusion.input" href="../../input/index.html" /> |
| <link rel="prev" title="datafusion.functions" href="../index.html" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="" /> |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto"> |
| |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../../../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../../../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../../../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/> |
| <img src="../../../../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../../user-guide/index.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../../contributor-guide/index.html"> |
| Contributor Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../../../index.html"> |
| API Reference |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../../links.html"> |
| Links |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">Rust API docs (docs.rs)</span></a> |
| </li> |
| </ul></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../../user-guide/index.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../../contributor-guide/index.html"> |
| Contributor Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../../../index.html"> |
| API Reference |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../../../links.html"> |
| Links |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">Rust API docs (docs.rs)</span></a> |
| </li> |
| </ul></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"> |
| <ul class="current nav bd-sidenav"> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/introduction.html">Introduction</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/basics.html">Concepts</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/data-sources.html">Data Sources</a></li> |
| |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/views.html">Registering Views</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/basic-info.html">Basic Operations</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/select-and-filter.html">Column Selections</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/expressions.html">Expressions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/joins.html">Joins</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/functions.html">Functions</a></li> |
| |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/aggregations.html">Aggregation</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/windows.html">Window Functions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../../../../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/arrow.html">Arrow</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/avro.html">Avro</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/csv.html">CSV</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/json.html">JSON</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/parquet.html">Parquet</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/table_provider.html">Custom Table Provider</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/configuration.html">Configuration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/distributing-work.html">Distributing work</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/sql.html">SQL</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/upgrade-guides.html">Upgrade Guides</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../contributor-guide/introduction.html">Introduction</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../../../../contributor-guide/ffi.html">Python Extensions</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="../../../index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l2 current active has-children"><a class="reference internal" href="../../index.html">datafusion</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l3"><a class="reference internal" href="../../catalog/index.html">datafusion.catalog</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../context/index.html">datafusion.context</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../dataframe/index.html">datafusion.dataframe</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../expr/index.html">datafusion.expr</a></li> |
| <li class="toctree-l3 current active has-children"><a class="reference internal" href="../index.html">datafusion.functions</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l4 current active"><a class="current reference internal" href="#">datafusion.functions.spark</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="../../input/base/index.html">datafusion.input.base</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="../../input/location/index.html">datafusion.input.location</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../io/index.html">datafusion.io</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../ipc/index.html">datafusion.ipc</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../object_store/index.html">datafusion.object_store</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../options/index.html">datafusion.options</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../plan/index.html">datafusion.plan</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../record_batch/index.html">datafusion.record_batch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../substrait/index.html">datafusion.substrait</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../unparser/index.html">datafusion.unparser</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../../user_defined/index.html">datafusion.user_defined</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../../../../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../../../index.html" class="nav-link">API Reference</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="../../index.html" class="nav-link">datafusion</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">datafusion.functions</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">datafusion.functions.spark</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="module-datafusion.functions.spark"> |
| <span id="datafusion-functions-spark"></span><h1>datafusion.functions.spark<a class="headerlink" href="#module-datafusion.functions.spark" title="Link to this heading">#</a></h1> |
| <p>Spark-compatible function bindings.</p> |
| <p>These functions mirror the semantics of their Apache Spark counterparts |
| exactly. Some override DataFusion built-ins (<code class="docutils literal notranslate"><span class="pre">substring</span></code> is 1-indexed, |
| <code class="docutils literal notranslate"><span class="pre">concat</span></code> propagates NULL, <code class="docutils literal notranslate"><span class="pre">round</span></code> uses HALF_UP rounding, etc.), which is |
| why they live in a separate namespace rather than replacing the defaults.</p> |
| <p>For DataFrame use, import this module and call functions directly. For SQL |
| use, call <code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.SessionContext.enable_spark_functions()</span></code> to |
| register the Spark UDFs by name (overriding any built-ins with matching |
| names) before issuing SQL queries.</p> |
| <section id="functions"> |
| <h2>Functions<a class="headerlink" href="#functions" title="Link to this heading">#</a></h2> |
| <div class="pst-scrollable-table-container"><table class="autosummary longtable table autosummary"> |
| <tbody> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.abs" title="datafusion.functions.spark.abs"><code class="xref py py-obj docutils literal notranslate"><span class="pre">abs</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">abs</span></code>: absolute value.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.add_months" title="datafusion.functions.spark.add_months"><code class="xref py py-obj docutils literal notranslate"><span class="pre">add_months</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">add_months</span></code>: date + N months.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.array" title="datafusion.functions.spark.array"><code class="xref py py-obj docutils literal notranslate"><span class="pre">array</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">array</span></code>: builds an array from the given elements.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.array_contains" title="datafusion.functions.spark.array_contains"><code class="xref py py-obj docutils literal notranslate"><span class="pre">array_contains</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">array_contains</span></code>: true if the array contains the element.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.array_repeat" title="datafusion.functions.spark.array_repeat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">array_repeat</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">array_repeat</span></code>: array of <code class="docutils literal notranslate"><span class="pre">element</span></code> repeated <code class="docutils literal notranslate"><span class="pre">count</span></code> times.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.ascii" title="datafusion.functions.spark.ascii"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ascii</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">ascii</span></code>: code point of the first character.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.avg" title="datafusion.functions.spark.avg"><code class="xref py py-obj docutils literal notranslate"><span class="pre">avg</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">avg</span></code>: returns the mean of a numeric column.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.base64" title="datafusion.functions.spark.base64"><code class="xref py py-obj docutils literal notranslate"><span class="pre">base64</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">base64</span></code>: encode binary as a base64 string.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bin" title="datafusion.functions.spark.bin"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bin</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">bin</span></code>: binary string representation of a long.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.bit_count" title="datafusion.functions.spark.bit_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bit_count</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_count</span></code>: number of bits set in the integer's binary form.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bit_get" title="datafusion.functions.spark.bit_get"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bit_get</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_get</span></code>: returns the bit (0 or 1) at <code class="docutils literal notranslate"><span class="pre">pos</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitmap_bit_position" title="datafusion.functions.spark.bitmap_bit_position"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitmap_bit_position</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bit_position</span></code>: bit position for a child expression.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitmap_bucket_number" title="datafusion.functions.spark.bitmap_bucket_number"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitmap_bucket_number</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bucket_number</span></code>: bucket number for a child expression.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitmap_count" title="datafusion.functions.spark.bitmap_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitmap_count</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_count</span></code>: number of set bits in a bitmap.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitwise_not" title="datafusion.functions.spark.bitwise_not"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitwise_not</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">~</span></code>: bitwise NOT.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.ceil" title="datafusion.functions.spark.ceil"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ceil</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">ceil</span></code>: smallest integer ≥ arg.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.char" title="datafusion.functions.spark.char"><code class="xref py py-obj docutils literal notranslate"><span class="pre">char</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">char</span></code>: ASCII character for a code point (mod 256).</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.collect_list" title="datafusion.functions.spark.collect_list"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collect_list</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_list</span></code>: collect values into an array (preserves dups).</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.collect_set" title="datafusion.functions.spark.collect_set"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collect_set</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_set</span></code>: collect distinct values into an array.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.concat" title="datafusion.functions.spark.concat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">concat</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">concat</span></code>: concatenates strings; NULL if any input is NULL.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.crc32" title="datafusion.functions.spark.crc32"><code class="xref py py-obj docutils literal notranslate"><span class="pre">crc32</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">crc32</span></code>: cyclic redundancy check value as a bigint.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.csc" title="datafusion.functions.spark.csc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">csc</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">csc</span></code>: cosecant.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_add" title="datafusion.functions.spark.date_add"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_add</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_add</span></code>: date + N days.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_diff" title="datafusion.functions.spark.date_diff"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_diff</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_diff</span></code>: number of days from <code class="docutils literal notranslate"><span class="pre">start_date</span></code> to <code class="docutils literal notranslate"><span class="pre">end_date</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_part" title="datafusion.functions.spark.date_part"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_part</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_part</span></code>: extract <code class="docutils literal notranslate"><span class="pre">field</span></code> from a date/time/timestamp.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_sub" title="datafusion.functions.spark.date_sub"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_sub</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_sub</span></code>: date - N days.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_trunc" title="datafusion.functions.spark.date_trunc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_trunc</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_trunc</span></code>: truncate timestamp to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.elt" title="datafusion.functions.spark.elt"><code class="xref py py-obj docutils literal notranslate"><span class="pre">elt</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">elt</span></code>: returns the n-th input (1-indexed).</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.expm1" title="datafusion.functions.spark.expm1"><code class="xref py py-obj docutils literal notranslate"><span class="pre">expm1</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">expm1</span></code>: exp(arg) - 1.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.factorial" title="datafusion.functions.spark.factorial"><code class="xref py py-obj docutils literal notranslate"><span class="pre">factorial</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">factorial</span></code>: n! for n in [0..20], else NULL.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.floor" title="datafusion.functions.spark.floor"><code class="xref py py-obj docutils literal notranslate"><span class="pre">floor</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">floor</span></code>: largest integer ≤ arg.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.format_string" title="datafusion.functions.spark.format_string"><code class="xref py py-obj docutils literal notranslate"><span class="pre">format_string</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">format_string</span></code>: printf-style format string.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.from_utc_timestamp" title="datafusion.functions.spark.from_utc_timestamp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_utc_timestamp</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">from_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as UTC, convert to <code class="docutils literal notranslate"><span class="pre">tz</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.hex" title="datafusion.functions.spark.hex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hex</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">hex</span></code>: hexadecimal representation.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.hour" title="datafusion.functions.spark.hour"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hour</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">hour</span></code>: extract hour component of a timestamp.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.if_" title="datafusion.functions.spark.if_"><code class="xref py py-obj docutils literal notranslate"><span class="pre">if_</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">if</span></code>: returns <code class="docutils literal notranslate"><span class="pre">if_true</span></code> when <code class="docutils literal notranslate"><span class="pre">condition</span></code> is true, else <code class="docutils literal notranslate"><span class="pre">if_false</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.ilike" title="datafusion.functions.spark.ilike"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ilike</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">ilike</span></code>: case-insensitive pattern match.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.is_valid_utf8" title="datafusion.functions.spark.is_valid_utf8"><code class="xref py py-obj docutils literal notranslate"><span class="pre">is_valid_utf8</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">is_valid_utf8</span></code>: true if the string is valid UTF-8.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.json_tuple" title="datafusion.functions.spark.json_tuple"><code class="xref py py-obj docutils literal notranslate"><span class="pre">json_tuple</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">json_tuple</span></code>: extract top-level fields from a JSON string.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.last_day" title="datafusion.functions.spark.last_day"><code class="xref py py-obj docutils literal notranslate"><span class="pre">last_day</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">last_day</span></code>: last day of the month containing the date.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.length" title="datafusion.functions.spark.length"><code class="xref py py-obj docutils literal notranslate"><span class="pre">length</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">length</span></code>: character length of a string, or byte length of binary.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.like" title="datafusion.functions.spark.like"><code class="xref py py-obj docutils literal notranslate"><span class="pre">like</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">like</span></code>: case-sensitive pattern match.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.luhn_check" title="datafusion.functions.spark.luhn_check"><code class="xref py py-obj docutils literal notranslate"><span class="pre">luhn_check</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">luhn_check</span></code>: true if the digit string passes the Luhn check.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.make_dt_interval" title="datafusion.functions.spark.make_dt_interval"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_dt_interval</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">make_dt_interval</span></code>: day-time interval from components.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.make_interval" title="datafusion.functions.spark.make_interval"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_interval</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">make_interval</span></code>: interval from year/month/week/day/hour/min/sec parts.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.make_valid_utf8" title="datafusion.functions.spark.make_valid_utf8"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_valid_utf8</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">make_valid_utf8</span></code>: replace invalid UTF-8 bytes with U+FFFD.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.map_from_arrays" title="datafusion.functions.spark.map_from_arrays"><code class="xref py py-obj docutils literal notranslate"><span class="pre">map_from_arrays</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_arrays</span></code>: build a map from parallel key/value arrays.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.map_from_entries" title="datafusion.functions.spark.map_from_entries"><code class="xref py py-obj docutils literal notranslate"><span class="pre">map_from_entries</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_entries</span></code>: build a map from an array of key/value structs.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.minute" title="datafusion.functions.spark.minute"><code class="xref py py-obj docutils literal notranslate"><span class="pre">minute</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">minute</span></code>: extract minute component of a timestamp.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.modulus" title="datafusion.functions.spark.modulus"><code class="xref py py-obj docutils literal notranslate"><span class="pre">modulus</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">mod</span></code>: remainder of <code class="docutils literal notranslate"><span class="pre">dividend</span> <span class="pre">/</span> <span class="pre">divisor</span></code> (sign follows dividend).</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.negative" title="datafusion.functions.spark.negative"><code class="xref py py-obj docutils literal notranslate"><span class="pre">negative</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">negative</span></code>: unary minus.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.next_day" title="datafusion.functions.spark.next_day"><code class="xref py py-obj docutils literal notranslate"><span class="pre">next_day</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">next_day</span></code>: first date after <code class="docutils literal notranslate"><span class="pre">start_date</span></code> named <code class="docutils literal notranslate"><span class="pre">day_of_week</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.parse_url" title="datafusion.functions.spark.parse_url"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_url</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">parse_url</span></code>: extract a part from a URL; errors on invalid URLs.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.pmod" title="datafusion.functions.spark.pmod"><code class="xref py py-obj docutils literal notranslate"><span class="pre">pmod</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">pmod</span></code>: positive remainder of division.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.rint" title="datafusion.functions.spark.rint"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rint</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">rint</span></code>: round to nearest mathematical integer (as double).</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.round" title="datafusion.functions.spark.round"><code class="xref py py-obj docutils literal notranslate"><span class="pre">round</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">round</span></code>: round to <code class="docutils literal notranslate"><span class="pre">scale</span></code> decimal places, HALF_UP rounding.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.sec" title="datafusion.functions.spark.sec"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sec</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">sec</span></code>: secant.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.second" title="datafusion.functions.spark.second"><code class="xref py py-obj docutils literal notranslate"><span class="pre">second</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">second</span></code>: extract second component of a timestamp.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.sha1" title="datafusion.functions.spark.sha1"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha1</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">sha1</span></code>: SHA-1 hash as a hex string.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.sha2" title="datafusion.functions.spark.sha2"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha2</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">sha2</span></code>: SHA-2 family hash (224, 256, 384, 512). Bit length 0 = 256.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.shiftleft" title="datafusion.functions.spark.shiftleft"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shiftleft</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftleft</span></code>: <code class="docutils literal notranslate"><span class="pre">value</span></code> shifted left by <code class="docutils literal notranslate"><span class="pre">shift</span></code> bits.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.shiftright" title="datafusion.functions.spark.shiftright"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shiftright</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftright</span></code>: arithmetic right shift.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.shiftrightunsigned" title="datafusion.functions.spark.shiftrightunsigned"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shiftrightunsigned</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftrightunsigned</span></code>: logical (unsigned) right shift.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.shuffle" title="datafusion.functions.spark.shuffle"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shuffle</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">shuffle</span></code>: returns a random permutation of the input array.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.size" title="datafusion.functions.spark.size"><code class="xref py py-obj docutils literal notranslate"><span class="pre">size</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">size</span></code>: length of an array or map.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.slice" title="datafusion.functions.spark.slice"><code class="xref py py-obj docutils literal notranslate"><span class="pre">slice</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">slice</span></code>: subset of the array from 1-indexed <code class="docutils literal notranslate"><span class="pre">start</span></code> with <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.soundex" title="datafusion.functions.spark.soundex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">soundex</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">soundex</span></code>: Soundex phonetic code.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.space" title="datafusion.functions.spark.space"><code class="xref py py-obj docutils literal notranslate"><span class="pre">space</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">space</span></code>: string of n spaces.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.spark_cast" title="datafusion.functions.spark.spark_cast"><code class="xref py py-obj docutils literal notranslate"><span class="pre">spark_cast</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">cast</span></code>: cast <code class="docutils literal notranslate"><span class="pre">arg</span></code> to the type named by <code class="docutils literal notranslate"><span class="pre">type_str</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.str_to_map" title="datafusion.functions.spark.str_to_map"><code class="xref py py-obj docutils literal notranslate"><span class="pre">str_to_map</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">str_to_map</span></code>: split text into key/value pairs using delimiters.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.substring" title="datafusion.functions.spark.substring"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substring</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">substring</span></code>: 1-indexed substring starting at <code class="docutils literal notranslate"><span class="pre">pos</span></code> of given <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.time_trunc" title="datafusion.functions.spark.time_trunc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">time_trunc</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">time_trunc</span></code>: truncate time value to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.to_utc_timestamp" title="datafusion.functions.spark.to_utc_timestamp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_utc_timestamp</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">to_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as <code class="docutils literal notranslate"><span class="pre">tz</span></code>, convert to UTC.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.trunc" title="datafusion.functions.spark.trunc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">trunc</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">trunc</span></code>: truncate date to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.try_parse_url" title="datafusion.functions.spark.try_parse_url"><code class="xref py py-obj docutils literal notranslate"><span class="pre">try_parse_url</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">try_parse_url</span></code>: like <code class="docutils literal notranslate"><span class="pre">parse_url</span></code> but returns NULL on invalid URLs.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.try_sum" title="datafusion.functions.spark.try_sum"><code class="xref py py-obj docutils literal notranslate"><span class="pre">try_sum</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">try_sum</span></code>: sum that returns NULL on overflow.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.try_url_decode" title="datafusion.functions.spark.try_url_decode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">try_url_decode</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">try_url_decode</span></code>: like <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>; returns NULL on invalid input.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.unbase64" title="datafusion.functions.spark.unbase64"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unbase64</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">unbase64</span></code>: decode a base64 string to binary.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.unhex" title="datafusion.functions.spark.unhex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unhex</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">unhex</span></code>: convert hexadecimal string to binary.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_date" title="datafusion.functions.spark.unix_date"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_date</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_date</span></code>: days since 1970-01-01 for <code class="docutils literal notranslate"><span class="pre">dt</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_micros" title="datafusion.functions.spark.unix_micros"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_micros</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_micros</span></code>: microseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_millis" title="datafusion.functions.spark.unix_millis"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_millis</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_millis</span></code>: milliseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_seconds" title="datafusion.functions.spark.unix_seconds"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_seconds</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_seconds</span></code>: seconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.url_decode" title="datafusion.functions.spark.url_decode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">url_decode</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>: decode an application/x-www-form-urlencoded string.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.url_encode" title="datafusion.functions.spark.url_encode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">url_encode</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">url_encode</span></code>: encode a string in application/x-www-form-urlencoded.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.width_bucket" title="datafusion.functions.spark.width_bucket"><code class="xref py py-obj docutils literal notranslate"><span class="pre">width_bucket</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">width_bucket</span></code>: bucket number for <code class="docutils literal notranslate"><span class="pre">value</span></code> in equi-width histogram.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.xxhash64" title="datafusion.functions.spark.xxhash64"><code class="xref py py-obj docutils literal notranslate"><span class="pre">xxhash64</span></code></a>(→ datafusion.expr.Expr)</p></td> |
| <td><p>Spark <code class="docutils literal notranslate"><span class="pre">xxhash64</span></code>: 64-bit xxHash of the arguments.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| </section> |
| <section id="module-contents"> |
| <h2>Module Contents<a class="headerlink" href="#module-contents" title="Link to this heading">#</a></h2> |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.abs"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">abs</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.abs" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">abs</span></code>: absolute value.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">5</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.add_months"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">add_months</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">months</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.add_months" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">add_months</span></code>: date + N months.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">months</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">add_months</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.date(2020, 3, 15)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.array"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">array</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.array" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">array</span></code>: builds an array from the given elements.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">[1, 2, 3]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.array_contains"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">array_contains</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.array_contains" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">array_contains</span></code>: true if the array contains the element.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">value</span></code> accepts a native Python literal or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array_contains</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">)),</span> <span class="mi">1</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">True</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.array_repeat"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">array_repeat</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">count</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.array_repeat" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">array_repeat</span></code>: array of <code class="docutils literal notranslate"><span class="pre">element</span></code> repeated <code class="docutils literal notranslate"><span class="pre">count</span></code> times.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">count</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array_repeat</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">['a', 'a', 'a']</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.ascii"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">ascii</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.ascii" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">ascii</span></code>: code point of the first character.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ascii</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"A"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">65</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.avg"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">avg</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">filter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">order_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">datafusion.expr.SortKey</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">datafusion.expr.SortKey</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">null_treatment</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion.common.NullTreatment</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.avg" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">avg</span></code>: returns the mean of a numeric column.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">avg</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)])</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2.0</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.base64"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">base64</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.base64" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">base64</span></code>: encode binary as a base64 string.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">base64</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="sa">b</span><span class="s2">"hi"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'aGk='</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bin"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bin</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bin" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bin</span></code>: binary string representation of a long.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bin</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">7</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'111'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bit_count"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bit_count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bit_count" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_count</span></code>: number of bits set in the integer’s binary form.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bit_count</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">7</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">3</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bit_get"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bit_get</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">pos</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bit_get" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_get</span></code>: returns the bit (0 or 1) at <code class="docutils literal notranslate"><span class="pre">pos</span></code>.</p> |
| <p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">pos</span></code> is treated as a column name (matching pyspark), |
| not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal position.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bit_get</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">5</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bitmap_bit_position"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitmap_bit_position</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bitmap_bit_position" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bit_position</span></code>: bit position for a child expression.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitmap_bit_position</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">15</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">14</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bitmap_bucket_number"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitmap_bucket_number</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bitmap_bucket_number" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bucket_number</span></code>: bucket number for a child expression.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitmap_bucket_number</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">15</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bitmap_count"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitmap_count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bitmap_count" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_count</span></code>: number of set bits in a bitmap.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitmap_count</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="sa">b</span><span class="s2">"</span><span class="se">\xff</span><span class="s2">"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">8</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.bitwise_not"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitwise_not</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.bitwise_not" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">~</span></code>: bitwise NOT.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitwise_not</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">-1</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.ceil"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">ceil</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.ceil" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">ceil</span></code>: smallest integer ≥ arg.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ceil</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">1.2</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.char"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">char</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.char" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">char</span></code>: ASCII character for a code point (mod 256).</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">char</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">65</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'A'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.collect_list"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">collect_list</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">filter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">order_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">datafusion.expr.SortKey</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">datafusion.expr.SortKey</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">null_treatment</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion.common.NullTreatment</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.collect_list" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_list</span></code>: collect values into an array (preserves dups).</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)])</span> |
| <span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> |
| <span class="go">[1, 2, 2]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.collect_set"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">collect_set</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">filter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">order_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">datafusion.expr.SortKey</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">datafusion.expr.SortKey</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">null_treatment</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion.common.NullTreatment</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.collect_set" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_set</span></code>: collect distinct values into an array.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">collect_set</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)])</span> |
| <span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> |
| <span class="go">[1, 2, 3]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.concat"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">concat</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.concat" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">concat</span></code>: concatenates strings; NULL if any input is NULL.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'ab'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.crc32"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">crc32</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.crc32" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">crc32</span></code>: cyclic redundancy check value as a bigint.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"ABC"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">crc32</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"s"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2743272264</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.csc"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">csc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.csc" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">csc</span></code>: cosecant.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">csc</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">1.5708</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s1">'v'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><span class="s2">"</span> |
| <span class="go">'1.0000'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.date_add"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_add</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">days</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.date_add" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_add</span></code>: date + N days.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">days</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_add</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.date(2020, 1, 20)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.date_diff"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_diff</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">end</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.date_diff" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_diff</span></code>: number of days from <code class="docutils literal notranslate"><span class="pre">start_date</span></code> to <code class="docutils literal notranslate"><span class="pre">end_date</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">end</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">20</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_diff</span><span class="p">(</span><span class="n">end</span><span class="p">,</span> <span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">5</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.date_part"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_part</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">field</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">source</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.date_part" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_part</span></code>: extract <code class="docutils literal notranslate"><span class="pre">field</span></code> from a date/time/timestamp.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">field</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_part</span><span class="p">(</span><span class="s2">"year"</span><span class="p">,</span> <span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2020</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.date_sub"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_sub</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">days</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.date_sub" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_sub</span></code>: date - N days.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">days</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_sub</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.date(2020, 1, 10)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.date_trunc"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_trunc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timestamp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.date_trunc" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_trunc</span></code>: truncate timestamp to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">format</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_trunc</span><span class="p">(</span><span class="s2">"month"</span><span class="p">,</span> <span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.datetime(2020, 1, 1, 0, 0)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.elt"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">elt</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">inputs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.elt" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">elt</span></code>: returns the n-th input (1-indexed).</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">elt</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'b'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.expm1"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">expm1</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.expm1" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">expm1</span></code>: exp(arg) - 1.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">expm1</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">0.0</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">0.0</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.factorial"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">factorial</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.factorial" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">factorial</span></code>: n! for n in [0..20], else NULL.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">factorial</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">()))</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">120</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.floor"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">floor</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.floor" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">floor</span></code>: largest integer ≤ arg.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">floor</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">1.8</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.format_string"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">format_string</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.format_string" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">format_string</span></code>: printf-style format string.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">format</span></code> is the printf-style template (a plain <code class="docutils literal notranslate"><span class="pre">str</span></code> is auto-promoted |
| to a literal expression); remaining args are values to substitute.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">format_string</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="s2">"</span><span class="si">%d</span><span class="s2">-</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">42</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hi"</span><span class="p">)</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'42-hi'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.from_utc_timestamp"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">from_utc_timestamp</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">timestamp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">tz</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.from_utc_timestamp" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">from_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as UTC, convert to <code class="docutils literal notranslate"><span class="pre">tz</span></code>.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">tz</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">from_utc_timestamp</span><span class="p">(</span><span class="n">ts</span><span class="p">,</span> <span class="s2">"UTC"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.datetime(2020, 1, 15, 14, 30, 45)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.hex"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">hex</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.hex" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">hex</span></code>: hexadecimal representation.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">hex</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">255</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'FF'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.hour"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">hour</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.hour" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">hour</span></code>: extract hour component of a timestamp.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">hour</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">14</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.if_"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">if_</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">condition</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">if_true</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Any</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">if_false</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.if_" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">if</span></code>: returns <code class="docutils literal notranslate"><span class="pre">if_true</span></code> when <code class="docutils literal notranslate"><span class="pre">condition</span></code> is true, else <code class="docutils literal notranslate"><span class="pre">if_false</span></code>.</p> |
| <p>Exposed as <code class="docutils literal notranslate"><span class="pre">if_</span></code> because <code class="docutils literal notranslate"><span class="pre">if</span></code> is a Python keyword. <code class="docutils literal notranslate"><span class="pre">if_true</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">if_false</span></code> accept native Python literals or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">if_</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">></span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="s2">"big"</span><span class="p">,</span> <span class="s2">"small"</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'big'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.ilike"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">ilike</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">pattern</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">ilike.str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">escapeChar</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">ilike.str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.ilike" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">ilike</span></code>: case-insensitive pattern match.</p> |
| <p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">pattern</span></code> is treated as a column name (matching pyspark), |
| not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal pattern. |
| <code class="docutils literal notranslate"><span class="pre">escapeChar</span></code> is accepted for pyspark parity but is not yet wired through |
| the Rust binding; passing a non-<code class="docutils literal notranslate"><span class="pre">None</span></code> value raises <code class="docutils literal notranslate"><span class="pre">NotImplementedError</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ilike</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"HELLO"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"h%"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">True</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.is_valid_utf8"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">is_valid_utf8</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.is_valid_utf8" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">is_valid_utf8</span></code>: true if the string is valid UTF-8.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">is_valid_utf8</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">True</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.json_tuple"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">json_tuple</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">fields</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.json_tuple" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">json_tuple</span></code>: extract top-level fields from a JSON string.</p> |
| <p>Each field name accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">json_tuple</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s1">'{"a":1,"b":"x"}'</span><span class="p">),</span> <span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">{'c0': '1', 'c1': 'x'}</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.last_day"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">last_day</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.last_day" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">last_day</span></code>: last day of the month containing the date.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">last_day</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.date(2020, 1, 31)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.length"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">length</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.length" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">length</span></code>: character length of a string, or byte length of binary.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">length</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">5</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.like"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">like</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">pattern</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">like.str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">escapeChar</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">like.str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.like" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">like</span></code>: case-sensitive pattern match.</p> |
| <p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">pattern</span></code> is treated as a column name (matching pyspark), |
| not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal pattern. |
| <code class="docutils literal notranslate"><span class="pre">escapeChar</span></code> is accepted for pyspark parity but is not yet wired through |
| the Rust binding; passing a non-<code class="docutils literal notranslate"><span class="pre">None</span></code> value raises <code class="docutils literal notranslate"><span class="pre">NotImplementedError</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">like</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"h%"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">True</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.luhn_check"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">luhn_check</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.luhn_check" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">luhn_check</span></code>: true if the digit string passes the Luhn check.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">luhn_check</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"4111111111111111"</span><span class="p">)</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">True</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.make_dt_interval"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">make_dt_interval</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">days</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">hours</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">mins</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">secs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">float</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.make_dt_interval" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">make_dt_interval</span></code>: day-time interval from components.</p> |
| <p>All parts are optional; omitted parts default to zero, matching pyspark. |
| Integer parts accept a native <code class="docutils literal notranslate"><span class="pre">int</span></code> and <code class="docutils literal notranslate"><span class="pre">secs</span></code> accepts a <code class="docutils literal notranslate"><span class="pre">float</span></code>, |
| or any part may be an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_dt_interval</span><span class="p">()</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.timedelta(0)</span> |
| </pre></div> |
| </div> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_dt_interval</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">hours</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mins</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">secs</span><span class="o">=</span><span class="mf">4.5</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.timedelta(days=1, seconds=7384, microseconds=500000)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.make_interval"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">make_interval</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">years</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">months</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">weeks</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">days</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">hours</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">mins</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">secs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">float</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.make_interval" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">make_interval</span></code>: interval from year/month/week/day/hour/min/sec parts.</p> |
| <p>All parts are optional; omitted parts default to zero, matching pyspark. |
| Integer parts accept a native <code class="docutils literal notranslate"><span class="pre">int</span></code> and <code class="docutils literal notranslate"><span class="pre">secs</span></code> accepts a <code class="docutils literal notranslate"><span class="pre">float</span></code>, |
| or any part may be an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_interval</span><span class="p">()</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="o">.</span><span class="n">months</span> |
| <span class="go">0</span> |
| </pre></div> |
| </div> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_interval</span><span class="p">(</span><span class="n">years</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="o">.</span><span class="n">months</span> |
| <span class="go">12</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.make_valid_utf8"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">make_valid_utf8</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.make_valid_utf8" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">make_valid_utf8</span></code>: replace invalid UTF-8 bytes with U+FFFD.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_valid_utf8</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'hello'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.map_from_arrays"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">map_from_arrays</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col1</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">col2</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.map_from_arrays" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_arrays</span></code>: build a map from parallel key/value arrays.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">keys</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"b"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">vals</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">map_from_arrays</span><span class="p">(</span><span class="n">keys</span><span class="p">,</span> <span class="n">vals</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">[('a', 1), ('b', 2)]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.map_from_entries"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">map_from_entries</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.map_from_entries" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_entries</span></code>: build a map from an array of key/value structs.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">col</span></code> must be an array whose elements are two-field structs; the first |
| field becomes the map key and the second the value.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">entry_type</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">list_</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">([(</span><span class="s2">"key"</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">()),</span> <span class="p">(</span><span class="s2">"value"</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())]))</span> |
| <span class="gp">>>> </span><span class="n">entries</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[[{</span><span class="s2">"key"</span><span class="p">:</span> <span class="s2">"a"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"key"</span><span class="p">:</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="mi">2</span><span class="p">}]],</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">entry_type</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_arrow</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">([</span><span class="n">entries</span><span class="p">],</span> <span class="n">names</span><span class="o">=</span><span class="p">[</span><span class="s2">"e"</span><span class="p">]))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">map_from_entries</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"e"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">[('a', 1), ('b', 2)]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.minute"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">minute</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.minute" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">minute</span></code>: extract minute component of a timestamp.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">minute</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">30</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.modulus"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">modulus</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dividend</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">float</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">divisor</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">float</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.modulus" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">mod</span></code>: remainder of <code class="docutils literal notranslate"><span class="pre">dividend</span> <span class="pre">/</span> <span class="pre">divisor</span></code> (sign follows dividend).</p> |
| <p><code class="docutils literal notranslate"><span class="pre">dividend</span></code> and <code class="docutils literal notranslate"><span class="pre">divisor</span></code> accept native numbers or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">modulus</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.negative"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">negative</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.negative" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">negative</span></code>: unary minus.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">negative</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">-3</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.next_day"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">next_day</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">date</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">dayOfWeek</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.next_day" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">next_day</span></code>: first date after <code class="docutils literal notranslate"><span class="pre">start_date</span></code> named <code class="docutils literal notranslate"><span class="pre">day_of_week</span></code>.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">dayOfWeek</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">next_day</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="s2">"Mon"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.date(2020, 1, 20)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.parse_url"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">parse_url</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">partToExtract</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">key</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.parse_url" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">parse_url</span></code>: extract a part from a URL; errors on invalid URLs.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">partToExtract</span></code> is one of <code class="docutils literal notranslate"><span class="pre">"HOST"</span></code>, <code class="docutils literal notranslate"><span class="pre">"PATH"</span></code>, <code class="docutils literal notranslate"><span class="pre">"QUERY"</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">"REF"</span></code>, <code class="docutils literal notranslate"><span class="pre">"PROTOCOL"</span></code>, <code class="docutils literal notranslate"><span class="pre">"FILE"</span></code>, <code class="docutils literal notranslate"><span class="pre">"AUTHORITY"</span></code>, <code class="docutils literal notranslate"><span class="pre">"USERINFO"</span></code>. |
| Pass <code class="docutils literal notranslate"><span class="pre">key</span></code> only with <code class="docutils literal notranslate"><span class="pre">"QUERY"</span></code> to extract a single parameter. Bare |
| <code class="docutils literal notranslate"><span class="pre">str</span></code> values for <code class="docutils literal notranslate"><span class="pre">partToExtract</span></code>/<code class="docutils literal notranslate"><span class="pre">key</span></code> are treated as column names |
| (matching pyspark); pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">parse_url</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"http://example.com/path?q=1"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"HOST"</span><span class="p">)</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'example.com'</span> |
| </pre></div> |
| </div> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">parse_url</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"http://example.com/path?q=1"</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"QUERY"</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="n">key</span><span class="o">=</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"q"</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'1'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.pmod"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">pmod</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dividend</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">float</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">divisor</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">float</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.pmod" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">pmod</span></code>: positive remainder of division.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">dividend</span></code> and <code class="docutils literal notranslate"><span class="pre">divisor</span></code> accept native numbers or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">pmod</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.rint"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">rint</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.rint" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">rint</span></code>: round to nearest mathematical integer (as double).</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">rint</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">2.5</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2.0</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.round"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">round</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">scale</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.round" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">round</span></code>: round to <code class="docutils literal notranslate"><span class="pre">scale</span></code> decimal places, HALF_UP rounding.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">scale</span></code> defaults to zero when omitted, matching pyspark, and accepts a |
| native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">2.5</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">3.0</span> |
| </pre></div> |
| </div> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">2.345</span><span class="p">),</span> <span class="n">scale</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2.35</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.sec"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">sec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.sec" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">sec</span></code>: secant.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sec</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">0.0</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1.0</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.second"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">second</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.second" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">second</span></code>: extract second component of a timestamp.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">second</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">45</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.sha1"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">sha1</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.sha1" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">sha1</span></code>: SHA-1 hash as a hex string.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"hello"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"s"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.sha2"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">sha2</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">numBits</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.sha2" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">sha2</span></code>: SHA-2 family hash (224, 256, 384, 512). Bit length 0 = 256.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"hello"</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sha2</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"s"</span><span class="p">),</span> <span class="mi">256</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.shiftleft"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shiftleft</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">numBits</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.shiftleft" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftleft</span></code>: <code class="docutils literal notranslate"><span class="pre">value</span></code> shifted left by <code class="docutils literal notranslate"><span class="pre">shift</span></code> bits.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shiftleft</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">8</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.shiftright"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shiftright</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">numBits</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.shiftright" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftright</span></code>: arithmetic right shift.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shiftright</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">8</span><span class="p">),</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.shiftrightunsigned"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shiftrightunsigned</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">numBits</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.shiftrightunsigned" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftrightunsigned</span></code>: logical (unsigned) right shift.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shiftrightunsigned</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">8</span><span class="p">),</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">2</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.shuffle"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shuffle</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">seed</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.shuffle" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shuffle</span></code>: returns a random permutation of the input array.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">seed</span></code> is accepted for pyspark parity but is not yet wired through the |
| Rust binding; passing a non-<code class="docutils literal notranslate"><span class="pre">None</span></code> value raises <code class="docutils literal notranslate"><span class="pre">NotImplementedError</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">))</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span> |
| <span class="go">[1, 2, 3]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.size"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">size</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.size" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">size</span></code>: length of an array or map.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">size</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">))</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">3</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.slice"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">slice</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">length</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.slice" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">slice</span></code>: subset of the array from 1-indexed <code class="docutils literal notranslate"><span class="pre">start</span></code> with <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p> |
| <p>Negative <code class="docutils literal notranslate"><span class="pre">start</span></code> counts from the end. <code class="docutils literal notranslate"><span class="pre">start</span></code> and <code class="docutils literal notranslate"><span class="pre">length</span></code> accept |
| native <code class="docutils literal notranslate"><span class="pre">int</span></code> values or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">slice</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">4</span><span class="p">)),</span> |
| <span class="gp">... </span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">[2, 3]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.soundex"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">soundex</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.soundex" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">soundex</span></code>: Soundex phonetic code.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">soundex</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"Robert"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'R163'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.space"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">space</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.space" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">space</span></code>: string of n spaces.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">col</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">space</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">' '</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.spark_cast"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">spark_cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">arg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">type_str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.spark_cast" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">cast</span></code>: cast <code class="docutils literal notranslate"><span class="pre">arg</span></code> to the type named by <code class="docutils literal notranslate"><span class="pre">type_str</span></code>.</p> |
| <p>Uses Spark cast semantics (e.g. overflow returns NULL, not error). |
| <code class="docutils literal notranslate"><span class="pre">type_str</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p>Currently only supports casting numeric values to <code class="docutils literal notranslate"><span class="pre">"timestamp"</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">spark_cast</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1579098645</span><span class="p">),</span> <span class="s2">"timestamp"</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> |
| <span class="go">'2020-01-15T14:30:45+00:00'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.str_to_map"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">str_to_map</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">text</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">pairDelim</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">keyValueDelim</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.str_to_map" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">str_to_map</span></code>: split text into key/value pairs using delimiters.</p> |
| <p>Delimiters default to <code class="docutils literal notranslate"><span class="pre">","</span></code> and <code class="docutils literal notranslate"><span class="pre">":"</span></code> when omitted, matching pyspark. |
| Parameter names match <code class="docutils literal notranslate"><span class="pre">pyspark.sql.functions.str_to_map</span></code>; pyspark types |
| the delimiters as column-or-name, so a bare <code class="docutils literal notranslate"><span class="pre">str</span></code> is treated as a column |
| name. Pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal delimiter.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">str_to_map</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a:1,b:2"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">[('a', '1'), ('b', '2')]</span> |
| </pre></div> |
| </div> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">str_to_map</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a=1;b=2"</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="n">pairDelim</span><span class="o">=</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">";"</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="n">keyValueDelim</span><span class="o">=</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"="</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">[('a', '1'), ('b', '2')]</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.substring"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">substring</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">pos</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">len</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.substring" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">substring</span></code>: 1-indexed substring starting at <code class="docutils literal notranslate"><span class="pre">pos</span></code> of given <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p> |
| <p>Negative <code class="docutils literal notranslate"><span class="pre">pos</span></code> counts from the end. <code class="docutils literal notranslate"><span class="pre">pos</span></code> and <code class="docutils literal notranslate"><span class="pre">len</span></code> accept native |
| <code class="docutils literal notranslate"><span class="pre">int</span></code> values or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">substring</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'hel'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.time_trunc"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">time_trunc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">unit</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">time</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.time_trunc" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">time_trunc</span></code>: truncate time value to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p> |
| <p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">unit</span></code> is treated as a column name (matching pyspark), |
| not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal unit.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">time</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">t</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">time</span><span class="p">(</span><span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">time64</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">time_trunc</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hour"</span><span class="p">),</span> <span class="n">t</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.time(14, 0)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.to_utc_timestamp"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">to_utc_timestamp</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">timestamp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">tz</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.to_utc_timestamp" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">to_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as <code class="docutils literal notranslate"><span class="pre">tz</span></code>, convert to UTC.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">tz</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">to_utc_timestamp</span><span class="p">(</span><span class="n">ts</span><span class="p">,</span> <span class="s2">"UTC"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.datetime(2020, 1, 15, 14, 30, 45)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.trunc"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">trunc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">date</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.trunc" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">trunc</span></code>: truncate date to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">format</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">trunc</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="s2">"YEAR"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">datetime.date(2020, 1, 1)</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.try_parse_url"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">try_parse_url</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">partToExtract</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">key</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.try_parse_url" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">try_parse_url</span></code>: like <code class="docutils literal notranslate"><span class="pre">parse_url</span></code> but returns NULL on invalid URLs.</p> |
| <p>Bare <code class="docutils literal notranslate"><span class="pre">str</span></code> values for <code class="docutils literal notranslate"><span class="pre">partToExtract</span></code>/<code class="docutils literal notranslate"><span class="pre">key</span></code> are treated as column |
| names (matching pyspark); pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">try_parse_url</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"http://example.com/"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"HOST"</span><span class="p">)</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'example.com'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.try_sum"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">try_sum</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinct</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">filter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">order_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">datafusion.expr.SortKey</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">datafusion.expr.SortKey</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">null_treatment</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion.common.NullTreatment</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.try_sum" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">try_sum</span></code>: sum that returns NULL on overflow.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">try_sum</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)])</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">6</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.try_url_decode"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">try_url_decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.try_url_decode" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">try_url_decode</span></code>: like <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>; returns NULL on invalid input.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">try_url_decode</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a%20b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'a b'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.unbase64"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unbase64</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.unbase64" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unbase64</span></code>: decode a base64 string to binary.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unbase64</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"aGk="</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">b'hi'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.unhex"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unhex</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.unhex" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unhex</span></code>: convert hexadecimal string to binary.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unhex</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"FF"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">b'\xff'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.unix_date"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_date</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.unix_date" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_date</span></code>: days since 1970-01-01 for <code class="docutils literal notranslate"><span class="pre">dt</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">date32</span><span class="p">()))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_date</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">18276</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.unix_micros"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_micros</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.unix_micros" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_micros</span></code>: microseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_micros</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1579098645000000</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.unix_millis"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_millis</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.unix_millis" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_millis</span></code>: milliseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_millis</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1579098645000</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.unix_seconds"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_seconds</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.unix_seconds" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_seconds</span></code>: seconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span> |
| <span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span> |
| <span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_seconds</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">1579098645</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.url_decode"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">url_decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.url_decode" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>: decode an application/x-www-form-urlencoded string.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">url_decode</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a%20b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'a b'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.url_encode"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">url_encode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.url_encode" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">url_encode</span></code>: encode a string in application/x-www-form-urlencoded.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">url_encode</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">'a+b'</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.width_bucket"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">width_bucket</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">v</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">min</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">max</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">numBucket</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">int</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.width_bucket" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">width_bucket</span></code>: bucket number for <code class="docutils literal notranslate"><span class="pre">value</span></code> in equi-width histogram.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">numBucket</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">width_bucket</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">5.0</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">0.0</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">10.0</span><span class="p">),</span> <span class="mi">5</span> |
| <span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">3</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="datafusion.functions.spark.xxhash64"> |
| <span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">xxhash64</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.Expr</span></a></span></span><a class="headerlink" href="#datafusion.functions.spark.xxhash64" title="Link to this definition">#</a></dt> |
| <dd><p>Spark <code class="docutils literal notranslate"><span class="pre">xxhash64</span></code>: 64-bit xxHash of the arguments.</p> |
| <p class="rubric">Examples</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span> |
| <span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span> |
| <span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">xxhash64</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span> |
| <span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span> |
| <span class="go">-4367754540140381902</span> |
| </pre></div> |
| </div> |
| </dd></dl> |
| |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="../index.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">datafusion.functions</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="../../input/index.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">datafusion.input</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#functions">Functions</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#module-contents">Module Contents</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.abs"><code class="docutils literal notranslate"><span class="pre">abs()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.add_months"><code class="docutils literal notranslate"><span class="pre">add_months()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.array"><code class="docutils literal notranslate"><span class="pre">array()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.array_contains"><code class="docutils literal notranslate"><span class="pre">array_contains()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.array_repeat"><code class="docutils literal notranslate"><span class="pre">array_repeat()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.ascii"><code class="docutils literal notranslate"><span class="pre">ascii()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.avg"><code class="docutils literal notranslate"><span class="pre">avg()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.base64"><code class="docutils literal notranslate"><span class="pre">base64()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bin"><code class="docutils literal notranslate"><span class="pre">bin()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bit_count"><code class="docutils literal notranslate"><span class="pre">bit_count()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bit_get"><code class="docutils literal notranslate"><span class="pre">bit_get()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitmap_bit_position"><code class="docutils literal notranslate"><span class="pre">bitmap_bit_position()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitmap_bucket_number"><code class="docutils literal notranslate"><span class="pre">bitmap_bucket_number()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitmap_count"><code class="docutils literal notranslate"><span class="pre">bitmap_count()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitwise_not"><code class="docutils literal notranslate"><span class="pre">bitwise_not()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.ceil"><code class="docutils literal notranslate"><span class="pre">ceil()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.char"><code class="docutils literal notranslate"><span class="pre">char()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.collect_list"><code class="docutils literal notranslate"><span class="pre">collect_list()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.collect_set"><code class="docutils literal notranslate"><span class="pre">collect_set()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.concat"><code class="docutils literal notranslate"><span class="pre">concat()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.crc32"><code class="docutils literal notranslate"><span class="pre">crc32()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.csc"><code class="docutils literal notranslate"><span class="pre">csc()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_add"><code class="docutils literal notranslate"><span class="pre">date_add()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_diff"><code class="docutils literal notranslate"><span class="pre">date_diff()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_part"><code class="docutils literal notranslate"><span class="pre">date_part()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_sub"><code class="docutils literal notranslate"><span class="pre">date_sub()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_trunc"><code class="docutils literal notranslate"><span class="pre">date_trunc()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.elt"><code class="docutils literal notranslate"><span class="pre">elt()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.expm1"><code class="docutils literal notranslate"><span class="pre">expm1()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.factorial"><code class="docutils literal notranslate"><span class="pre">factorial()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.floor"><code class="docutils literal notranslate"><span class="pre">floor()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.format_string"><code class="docutils literal notranslate"><span class="pre">format_string()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.from_utc_timestamp"><code class="docutils literal notranslate"><span class="pre">from_utc_timestamp()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.hex"><code class="docutils literal notranslate"><span class="pre">hex()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.hour"><code class="docutils literal notranslate"><span class="pre">hour()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.if_"><code class="docutils literal notranslate"><span class="pre">if_()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.ilike"><code class="docutils literal notranslate"><span class="pre">ilike()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.is_valid_utf8"><code class="docutils literal notranslate"><span class="pre">is_valid_utf8()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.json_tuple"><code class="docutils literal notranslate"><span class="pre">json_tuple()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.last_day"><code class="docutils literal notranslate"><span class="pre">last_day()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.length"><code class="docutils literal notranslate"><span class="pre">length()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.like"><code class="docutils literal notranslate"><span class="pre">like()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.luhn_check"><code class="docutils literal notranslate"><span class="pre">luhn_check()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.make_dt_interval"><code class="docutils literal notranslate"><span class="pre">make_dt_interval()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.make_interval"><code class="docutils literal notranslate"><span class="pre">make_interval()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.make_valid_utf8"><code class="docutils literal notranslate"><span class="pre">make_valid_utf8()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.map_from_arrays"><code class="docutils literal notranslate"><span class="pre">map_from_arrays()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.map_from_entries"><code class="docutils literal notranslate"><span class="pre">map_from_entries()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.minute"><code class="docutils literal notranslate"><span class="pre">minute()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.modulus"><code class="docutils literal notranslate"><span class="pre">modulus()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.negative"><code class="docutils literal notranslate"><span class="pre">negative()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.next_day"><code class="docutils literal notranslate"><span class="pre">next_day()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.parse_url"><code class="docutils literal notranslate"><span class="pre">parse_url()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.pmod"><code class="docutils literal notranslate"><span class="pre">pmod()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.rint"><code class="docutils literal notranslate"><span class="pre">rint()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.round"><code class="docutils literal notranslate"><span class="pre">round()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.sec"><code class="docutils literal notranslate"><span class="pre">sec()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.second"><code class="docutils literal notranslate"><span class="pre">second()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.sha1"><code class="docutils literal notranslate"><span class="pre">sha1()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.sha2"><code class="docutils literal notranslate"><span class="pre">sha2()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shiftleft"><code class="docutils literal notranslate"><span class="pre">shiftleft()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shiftright"><code class="docutils literal notranslate"><span class="pre">shiftright()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shiftrightunsigned"><code class="docutils literal notranslate"><span class="pre">shiftrightunsigned()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shuffle"><code class="docutils literal notranslate"><span class="pre">shuffle()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.size"><code class="docutils literal notranslate"><span class="pre">size()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.slice"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.soundex"><code class="docutils literal notranslate"><span class="pre">soundex()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.space"><code class="docutils literal notranslate"><span class="pre">space()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.spark_cast"><code class="docutils literal notranslate"><span class="pre">spark_cast()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.str_to_map"><code class="docutils literal notranslate"><span class="pre">str_to_map()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.substring"><code class="docutils literal notranslate"><span class="pre">substring()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.time_trunc"><code class="docutils literal notranslate"><span class="pre">time_trunc()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.to_utc_timestamp"><code class="docutils literal notranslate"><span class="pre">to_utc_timestamp()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.trunc"><code class="docutils literal notranslate"><span class="pre">trunc()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.try_parse_url"><code class="docutils literal notranslate"><span class="pre">try_parse_url()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.try_sum"><code class="docutils literal notranslate"><span class="pre">try_sum()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.try_url_decode"><code class="docutils literal notranslate"><span class="pre">try_url_decode()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unbase64"><code class="docutils literal notranslate"><span class="pre">unbase64()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unhex"><code class="docutils literal notranslate"><span class="pre">unhex()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_date"><code class="docutils literal notranslate"><span class="pre">unix_date()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_micros"><code class="docutils literal notranslate"><span class="pre">unix_micros()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_millis"><code class="docutils literal notranslate"><span class="pre">unix_millis()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_seconds"><code class="docutils literal notranslate"><span class="pre">unix_seconds()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.url_decode"><code class="docutils literal notranslate"><span class="pre">url_decode()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.url_encode"><code class="docutils literal notranslate"><span class="pre">url_encode()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.width_bucket"><code class="docutils literal notranslate"><span class="pre">width_bucket()</span></code></a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.xxhash64"><code class="docutils literal notranslate"><span class="pre">xxhash64()</span></code></a></li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script> |
| <script defer src="../../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script> |
| |
| <!-- Based on pydata_sphinx_theme/footer.html --> |
| <footer class="footer mt-5 mt-md-0"> |
| <div class="container"> |
| |
| <div class="footer-item"> |
| <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p> |
| <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> |
| </div> |
| </div> |
| </footer> |
| |
| |
| </body> |
| </html> |
