| |
| <!doctype html> |
| <html lang="en" class="no-js"> |
| <head> |
| |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| |
| |
| |
| <link rel="canonical" href="https://py.iceberg.apache.org/reference/pyiceberg/table/"> |
| |
| |
| <link rel="prev" href="../serializers/"> |
| |
| |
| <link rel="next" href="inspect/"> |
| |
| |
| <link rel="icon" href="../../../assets/images/iceberg-logo-icon.png"> |
| <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.21"> |
| |
| |
| |
| <title>table - PyIceberg</title> |
| |
| |
| |
| <link rel="stylesheet" href="../../../assets/stylesheets/main.2a3383ac.min.css"> |
| |
| |
| <link rel="stylesheet" href="../../../assets/stylesheets/palette.06af60db.min.css"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Lato:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback"> |
| <style>:root{--md-text-font:"Lato";--md-code-font:"Roboto Mono"}</style> |
| |
| |
| |
| <link rel="stylesheet" href="../../../assets/_mkdocstrings.css"> |
| |
| <script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script> |
| |
| |
| |
| |
| |
| |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| _paq.push(["setDoNotTrack", true]); |
| _paq.push(["disableCookies"]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '82']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo --> |
| |
| </head> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"> |
| |
| |
| <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off"> |
| <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off"> |
| <label class="md-overlay" for="__drawer"></label> |
| <div data-md-component="skip"> |
| |
| |
| <a href="#pyiceberg.table" class="md-skip"> |
| Skip to content |
| </a> |
| |
| </div> |
| <div data-md-component="announce"> |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| <header class="md-header md-header--shadow md-header--lifted" data-md-component="header"> |
| <nav class="md-header__inner md-grid" aria-label="Header"> |
| <a href="../../.." title="PyIceberg" class="md-header__button md-logo" aria-label="PyIceberg" data-md-component="logo"> |
| |
| <img src="../../../assets/images/iceberg-logo-icon.png" alt="logo"> |
| |
| </a> |
| <label class="md-header__button md-icon" for="__drawer"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg> |
| </label> |
| <div class="md-header__title" data-md-component="header-title"> |
| <div class="md-header__ellipsis"> |
| <div class="md-header__topic"> |
| <span class="md-ellipsis"> |
| PyIceberg |
| </span> |
| </div> |
| <div class="md-header__topic" data-md-component="header-topic"> |
| <span class="md-ellipsis"> |
| |
| table |
| |
| </span> |
| </div> |
| </div> |
| </div> |
| |
| |
| <form class="md-header__option" data-md-component="palette"> |
| |
| |
| |
| |
| <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0"> |
| |
| <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden> |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg> |
| </label> |
| |
| |
| |
| |
| |
| <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_1"> |
| |
| <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden> |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg> |
| </label> |
| |
| |
| </form> |
| |
| |
| |
| <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script> |
| |
| |
| |
| |
| |
| <label class="md-header__button md-icon" for="__search"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg> |
| </label> |
| <div class="md-search" data-md-component="search" role="dialog"> |
| <label class="md-search__overlay" for="__search"></label> |
| <div class="md-search__inner" role="search"> |
| <form class="md-search__form" name="search"> |
| <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required> |
| <label class="md-search__icon md-icon" for="__search"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg> |
| </label> |
| <nav class="md-search__options" aria-label="Search"> |
| |
| <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg> |
| </button> |
| </nav> |
| |
| </form> |
| <div class="md-search__output"> |
| <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix> |
| <div class="md-search-result" data-md-component="search-result"> |
| <div class="md-search-result__meta"> |
| Initializing search |
| </div> |
| <ol class="md-search-result__list" role="presentation"></ol> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div class="md-header__source"> |
| <a href="https://github.com/apache/iceberg-python" title="Go to repository" class="md-source" data-md-component="source"> |
| <div class="md-source__icon md-icon"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.0.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg> |
| </div> |
| <div class="md-source__repository"> |
| apache/iceberg-python |
| </div> |
| </a> |
| </div> |
| |
| </nav> |
| |
| |
| |
| <nav class="md-tabs" aria-label="Tabs" data-md-component="tabs"> |
| <div class="md-grid"> |
| <ul class="md-tabs__list"> |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../.." class="md-tabs__link"> |
| |
| |
| |
| |
| |
| Getting started |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../../configuration/" class="md-tabs__link"> |
| |
| |
| |
| |
| |
| Configuration |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../../cli/" class="md-tabs__link"> |
| |
| |
| |
| |
| |
| CLI |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../../api/" class="md-tabs__link"> |
| |
| |
| |
| |
| |
| API |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../../contributing/" class="md-tabs__link"> |
| |
| |
| |
| |
| |
| Contributing |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../../community/" class="md-tabs__link"> |
| |
| |
| |
| |
| |
| Community |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item"> |
| <a href="../../../verify-release/" class="md-tabs__link"> |
| |
| |
| |
| Releases |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-tabs__item md-tabs__item--active"> |
| <a href="../" class="md-tabs__link"> |
| |
| |
| |
| Code Reference |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </div> |
| </nav> |
| |
| |
| </header> |
| |
| <div class="md-container" data-md-component="container"> |
| |
| |
| |
| |
| <main class="md-main" data-md-component="main"> |
| <div class="md-main__inner md-grid"> |
| |
| |
| |
| <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" > |
| <div class="md-sidebar__scrollwrap"> |
| <div class="md-sidebar__inner"> |
| |
| |
| |
| |
| |
| |
| <nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0"> |
| <label class="md-nav__title" for="__drawer"> |
| <a href="../../.." title="PyIceberg" class="md-nav__button md-logo" aria-label="PyIceberg" data-md-component="logo"> |
| |
| <img src="../../../assets/images/iceberg-logo-icon.png" alt="logo"> |
| |
| </a> |
| PyIceberg |
| </label> |
| |
| <div class="md-nav__source"> |
| <a href="https://github.com/apache/iceberg-python" title="Go to repository" class="md-source" data-md-component="source"> |
| <div class="md-source__icon md-icon"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.0.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg> |
| </div> |
| <div class="md-source__repository"> |
| apache/iceberg-python |
| </div> |
| </a> |
| </div> |
| |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../.." class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Getting started |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../configuration/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Configuration |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../cli/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| CLI |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../../../api/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| API |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_4" id="__nav_4_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_4"> |
| <span class="md-nav__icon md-icon"></span> |
| API |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../row-filter-syntax/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Row Filter Syntax |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../expression-dsl/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Expression DSL |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../contributing/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Contributing |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../community/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Community |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" > |
| |
| |
| <label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Releases |
| |
| </span> |
| |
| |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_7"> |
| <span class="md-nav__icon md-icon"></span> |
| Releases |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../verify-release/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Verify a release |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../how-to-release/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| How to release |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="https://github.com/apache/iceberg-python/releases" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Release Notes |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../../../nightly-build/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Nightly Build |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" checked> |
| |
| |
| <label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex=""> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Code Reference |
| |
| </span> |
| |
| |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="true"> |
| <label class="md-nav__title" for="__nav_8"> |
| <span class="md-nav__icon md-icon"></span> |
| Code Reference |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--active md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1" checked> |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| pyiceberg |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1" id="__nav_8_1_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_8_1_label" aria-expanded="true"> |
| <label class="md-nav__title" for="__nav_8_1"> |
| <span class="md-nav__icon md-icon"></span> |
| pyiceberg |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_1" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../avro/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| avro |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_1" id="__nav_8_1_1_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_1_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_1"> |
| <span class="md-nav__icon md-icon"></span> |
| avro |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_1_1" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../avro/codecs/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| codecs |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_1_1" id="__nav_8_1_1_1_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="4" aria-labelledby="__nav_8_1_1_1_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_1_1"> |
| <span class="md-nav__icon md-icon"></span> |
| codecs |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/codecs/bzip2/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| bzip2 |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/codecs/codec/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| codec |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/codecs/deflate/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| deflate |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/codecs/snappy_codec/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| snappy_codec |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/codecs/zstandard_codec/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| zstandard_codec |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/decoder/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| decoder |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/encoder/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| encoder |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/file/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| file |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/reader/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| reader |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/resolver/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| resolver |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../avro/writer/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| writer |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_2" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../catalog/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| catalog |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_2" id="__nav_8_1_2_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_2_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_2"> |
| <span class="md-nav__icon md-icon"></span> |
| catalog |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/bigquery_metastore/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| bigquery_metastore |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/dynamodb/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| dynamodb |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/glue/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| glue |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/hive/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| hive |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/memory/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| memory |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/noop/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| noop |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_2_7" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../catalog/rest/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| rest |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_2_7" id="__nav_8_1_2_7_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="4" aria-labelledby="__nav_8_1_2_7_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_2_7"> |
| <span class="md-nav__icon md-icon"></span> |
| rest |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/rest/auth/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| auth |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/rest/response/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| response |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../catalog/sql/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| sql |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_3" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../cli/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| cli |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_3" id="__nav_8_1_3_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_3_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_3"> |
| <span class="md-nav__icon md-icon"></span> |
| cli |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../cli/console/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| console |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../cli/output/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| output |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../conversions/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| conversions |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../exceptions/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| exceptions |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_6" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../expressions/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| expressions |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_6" id="__nav_8_1_6_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_6_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_6"> |
| <span class="md-nav__icon md-icon"></span> |
| expressions |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../expressions/literals/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| literals |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../expressions/parser/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| parser |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../expressions/visitors/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| visitors |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_7" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../io/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| io |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_7" id="__nav_8_1_7_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_7_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_7"> |
| <span class="md-nav__icon md-icon"></span> |
| io |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../io/fsspec/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| fsspec |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../io/pyarrow/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| pyarrow |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../manifest/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| manifest |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../partitioning/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| partitioning |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../schema/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| schema |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../serializers/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| serializers |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--active md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_12" checked> |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="./" class="md-nav__link md-nav__link--active"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| table |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link md-nav__link--active" for="__nav_8_1_12" id="__nav_8_1_12_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_12_label" aria-expanded="true"> |
| <label class="md-nav__title" for="__nav_8_1_12"> |
| <span class="md-nav__icon md-icon"></span> |
| table |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="inspect/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| inspect |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="locations/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| locations |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="maintenance/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| maintenance |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="metadata/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| metadata |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="name_mapping/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| name_mapping |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="puffin/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| puffin |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="refs/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| refs |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="snapshots/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| snapshots |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="sorting/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| sorting |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="statistics/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| statistics |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_12_11" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="update/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| update |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_12_11" id="__nav_8_1_12_11_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="4" aria-labelledby="__nav_8_1_12_11_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_12_11"> |
| <span class="md-nav__icon md-icon"></span> |
| update |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="update/schema/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| schema |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="update/snapshot/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| snapshot |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="update/sorting/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| sorting |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="update/spec/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| spec |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="update/statistics/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| statistics |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="update/validate/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| validate |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="upsert_util/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| upsert_util |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../transforms/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| transforms |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../typedef/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| typedef |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../types/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| types |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_16" > |
| |
| |
| <div class="md-nav__link md-nav__container"> |
| <a href="../utils/" class="md-nav__link "> |
| |
| |
| |
| <span class="md-ellipsis"> |
| utils |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| <label class="md-nav__link " for="__nav_8_1_16" id="__nav_8_1_16_label" tabindex="0"> |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| </div> |
| |
| <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_16_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_8_1_16"> |
| <span class="md-nav__icon md-icon"></span> |
| utils |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/bin_packing/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| bin_packing |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/concurrent/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| concurrent |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/config/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| config |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/datetime/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| datetime |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/decimal/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| decimal |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/deprecated/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| deprecated |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/lazydict/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| lazydict |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/parsing/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| parsing |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/properties/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| properties |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/schema_conversion/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| schema_conversion |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/singleton/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| singleton |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="../utils/truncate/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| truncate |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| </ul> |
| </nav> |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" > |
| <div class="md-sidebar__scrollwrap"> |
| <div class="md-sidebar__inner"> |
| |
| |
| <nav class="md-nav md-nav--secondary" aria-label="Table of contents"> |
| |
| |
| |
| |
| <label class="md-nav__title" for="__toc"> |
| <span class="md-nav__icon md-icon"></span> |
| Table of contents |
| </label> |
| <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| table |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.CommitTableRequest" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| CommitTableRequest |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.CommitTableResponse" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| CommitTableResponse |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.CreateTableTransaction" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| CreateTableTransaction |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="CreateTableTransaction"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.CreateTableTransaction.commit_transaction" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| commit_transaction |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| DataScan |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="DataScan"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.plan_files" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| plan_files |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.to_arrow" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_arrow |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.to_arrow_batch_reader" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_arrow_batch_reader |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.to_duckdb" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_duckdb |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.to_pandas" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_pandas |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.to_polars" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_polars |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.DataScan.to_ray" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_ray |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.FileScanTask" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| FileScanTask |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Namespace" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Namespace |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.StaticTable" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| StaticTable |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="StaticTable"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.StaticTable.refresh" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| refresh |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Table |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="Table"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.inspect" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| inspect |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.maintenance" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| maintenance |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.properties" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| properties |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.__datafusion_table_provider__" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| __datafusion_table_provider__ |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.__eq__" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| __eq__ |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.__repr__" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| __repr__ |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.add_files" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| add_files |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.append" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| append |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.current_snapshot" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| current_snapshot |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.delete" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| delete |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.dynamic_partition_overwrite" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| dynamic_partition_overwrite |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.history" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| history |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.last_partition_id" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| last_partition_id |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.location" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| location |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.location_provider" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| location_provider |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.manage_snapshots" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| manage_snapshots |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.name" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| name |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.name_mapping" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| name_mapping |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.overwrite" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| overwrite |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.refresh" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| refresh |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.refs" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| refs |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.scan" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| scan |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.schema" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| schema |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.schemas" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| schemas |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.snapshot_as_of_timestamp" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| snapshot_as_of_timestamp |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.snapshot_by_id" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| snapshot_by_id |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.snapshot_by_name" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| snapshot_by_name |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.sort_order" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| sort_order |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.sort_orders" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| sort_orders |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.spec" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| spec |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.specs" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| specs |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.to_bodo" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_bodo |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.to_daft" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_daft |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.to_polars" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| to_polars |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.transaction" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| transaction |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.update_schema" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_schema |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.update_sort_order" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_sort_order |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.update_statistics" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_statistics |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Table.upsert" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| upsert |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.TableIdentifier" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| TableIdentifier |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.TableScan" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| TableScan |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="TableScan"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.TableScan.update" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Transaction |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="Transaction"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.__enter__" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| __enter__ |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.__exit__" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| __exit__ |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.__init__" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| __init__ |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.add_files" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| add_files |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.append" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| append |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.commit_transaction" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| commit_transaction |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.delete" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| delete |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.dynamic_partition_overwrite" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| dynamic_partition_overwrite |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.overwrite" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| overwrite |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.remove_properties" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| remove_properties |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.set_properties" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| set_properties |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.update_location" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_location |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.update_schema" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_schema |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.update_snapshot" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_snapshot |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.update_sort_order" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_sort_order |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.update_spec" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_spec |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.update_statistics" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| update_statistics |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.upgrade_table_version" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| upgrade_table_version |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.Transaction.upsert" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| upsert |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.UpsertResult" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| UpsertResult |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#pyiceberg.table.WriteTask" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| WriteTask |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| |
| </nav> |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div class="md-content" data-md-component="content"> |
| <article class="md-content__inner md-typeset"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <h1>table</h1> |
| |
| <div class="doc doc-object doc-module"> |
| |
| |
| |
| <a id="pyiceberg.table"></a> |
| <div class="doc doc-contents first"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.CommitTableRequest" class="doc doc-heading"> |
| <code>CommitTableRequest</code> |
| |
| |
| <a href="#pyiceberg.table.CommitTableRequest" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="IcebergBaseModel (pyiceberg.typedef.IcebergBaseModel)" href="../typedef/#pyiceberg.typedef.IcebergBaseModel">IcebergBaseModel</a></code></p> |
| |
| |
| <p>A pydantic BaseModel for a table commit request.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1047">1047</a></span> |
| <span class="normal"><a href="#__codelineno-0-1048">1048</a></span> |
| <span class="normal"><a href="#__codelineno-0-1049">1049</a></span> |
| <span class="normal"><a href="#__codelineno-0-1050">1050</a></span> |
| <span class="normal"><a href="#__codelineno-0-1051">1051</a></span> |
| <span class="normal"><a href="#__codelineno-0-1052">1052</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1047" name="__codelineno-0-1047"></a><span class="k">class</span><span class="w"> </span><span class="nc">CommitTableRequest</span><span class="p">(</span><span class="n">IcebergBaseModel</span><span class="p">):</span> |
| <a id="__codelineno-0-1048" name="__codelineno-0-1048"></a><span class="w"> </span><span class="sd">"""A pydantic BaseModel for a table commit request."""</span> |
| <a id="__codelineno-0-1049" name="__codelineno-0-1049"></a> |
| <a id="__codelineno-0-1050" name="__codelineno-0-1050"></a> <span class="n">identifier</span><span class="p">:</span> <span class="n">TableIdentifier</span> <span class="o">=</span> <span class="n">Field</span><span class="p">()</span> |
| <a id="__codelineno-0-1051" name="__codelineno-0-1051"></a> <span class="n">requirements</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableRequirement</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">tuple</span><span class="p">)</span> |
| <a id="__codelineno-0-1052" name="__codelineno-0-1052"></a> <span class="n">updates</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableUpdate</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">tuple</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.CommitTableResponse" class="doc doc-heading"> |
| <code>CommitTableResponse</code> |
| |
| |
| <a href="#pyiceberg.table.CommitTableResponse" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="IcebergBaseModel (pyiceberg.typedef.IcebergBaseModel)" href="../typedef/#pyiceberg.typedef.IcebergBaseModel">IcebergBaseModel</a></code></p> |
| |
| |
| <p>A pydantic BaseModel for a table commit response.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1055">1055</a></span> |
| <span class="normal"><a href="#__codelineno-0-1056">1056</a></span> |
| <span class="normal"><a href="#__codelineno-0-1057">1057</a></span> |
| <span class="normal"><a href="#__codelineno-0-1058">1058</a></span> |
| <span class="normal"><a href="#__codelineno-0-1059">1059</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1055" name="__codelineno-0-1055"></a><span class="k">class</span><span class="w"> </span><span class="nc">CommitTableResponse</span><span class="p">(</span><span class="n">IcebergBaseModel</span><span class="p">):</span> |
| <a id="__codelineno-0-1056" name="__codelineno-0-1056"></a><span class="w"> </span><span class="sd">"""A pydantic BaseModel for a table commit response."""</span> |
| <a id="__codelineno-0-1057" name="__codelineno-0-1057"></a> |
| <a id="__codelineno-0-1058" name="__codelineno-0-1058"></a> <span class="n">metadata</span><span class="p">:</span> <span class="n">TableMetadata</span> |
| <a id="__codelineno-0-1059" name="__codelineno-0-1059"></a> <span class="n">metadata_location</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">alias</span><span class="o">=</span><span class="s2">"metadata-location"</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.CreateTableTransaction" class="doc doc-heading"> |
| <code>CreateTableTransaction</code> |
| |
| |
| <a href="#pyiceberg.table.CreateTableTransaction" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="Transaction (pyiceberg.table.Transaction)" href="#pyiceberg.table.Transaction">Transaction</a></code></p> |
| |
| |
| <p>A transaction that involves the creation of a new table.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-973"> 973</a></span> |
| <span class="normal"><a href="#__codelineno-0-974"> 974</a></span> |
| <span class="normal"><a href="#__codelineno-0-975"> 975</a></span> |
| <span class="normal"><a href="#__codelineno-0-976"> 976</a></span> |
| <span class="normal"><a href="#__codelineno-0-977"> 977</a></span> |
| <span class="normal"><a href="#__codelineno-0-978"> 978</a></span> |
| <span class="normal"><a href="#__codelineno-0-979"> 979</a></span> |
| <span class="normal"><a href="#__codelineno-0-980"> 980</a></span> |
| <span class="normal"><a href="#__codelineno-0-981"> 981</a></span> |
| <span class="normal"><a href="#__codelineno-0-982"> 982</a></span> |
| <span class="normal"><a href="#__codelineno-0-983"> 983</a></span> |
| <span class="normal"><a href="#__codelineno-0-984"> 984</a></span> |
| <span class="normal"><a href="#__codelineno-0-985"> 985</a></span> |
| <span class="normal"><a href="#__codelineno-0-986"> 986</a></span> |
| <span class="normal"><a href="#__codelineno-0-987"> 987</a></span> |
| <span class="normal"><a href="#__codelineno-0-988"> 988</a></span> |
| <span class="normal"><a href="#__codelineno-0-989"> 989</a></span> |
| <span class="normal"><a href="#__codelineno-0-990"> 990</a></span> |
| <span class="normal"><a href="#__codelineno-0-991"> 991</a></span> |
| <span class="normal"><a href="#__codelineno-0-992"> 992</a></span> |
| <span class="normal"><a href="#__codelineno-0-993"> 993</a></span> |
| <span class="normal"><a href="#__codelineno-0-994"> 994</a></span> |
| <span class="normal"><a href="#__codelineno-0-995"> 995</a></span> |
| <span class="normal"><a href="#__codelineno-0-996"> 996</a></span> |
| <span class="normal"><a href="#__codelineno-0-997"> 997</a></span> |
| <span class="normal"><a href="#__codelineno-0-998"> 998</a></span> |
| <span class="normal"><a href="#__codelineno-0-999"> 999</a></span> |
| <span class="normal"><a href="#__codelineno-0-1000">1000</a></span> |
| <span class="normal"><a href="#__codelineno-0-1001">1001</a></span> |
| <span class="normal"><a href="#__codelineno-0-1002">1002</a></span> |
| <span class="normal"><a href="#__codelineno-0-1003">1003</a></span> |
| <span class="normal"><a href="#__codelineno-0-1004">1004</a></span> |
| <span class="normal"><a href="#__codelineno-0-1005">1005</a></span> |
| <span class="normal"><a href="#__codelineno-0-1006">1006</a></span> |
| <span class="normal"><a href="#__codelineno-0-1007">1007</a></span> |
| <span class="normal"><a href="#__codelineno-0-1008">1008</a></span> |
| <span class="normal"><a href="#__codelineno-0-1009">1009</a></span> |
| <span class="normal"><a href="#__codelineno-0-1010">1010</a></span> |
| <span class="normal"><a href="#__codelineno-0-1011">1011</a></span> |
| <span class="normal"><a href="#__codelineno-0-1012">1012</a></span> |
| <span class="normal"><a href="#__codelineno-0-1013">1013</a></span> |
| <span class="normal"><a href="#__codelineno-0-1014">1014</a></span> |
| <span class="normal"><a href="#__codelineno-0-1015">1015</a></span> |
| <span class="normal"><a href="#__codelineno-0-1016">1016</a></span> |
| <span class="normal"><a href="#__codelineno-0-1017">1017</a></span> |
| <span class="normal"><a href="#__codelineno-0-1018">1018</a></span> |
| <span class="normal"><a href="#__codelineno-0-1019">1019</a></span> |
| <span class="normal"><a href="#__codelineno-0-1020">1020</a></span> |
| <span class="normal"><a href="#__codelineno-0-1021">1021</a></span> |
| <span class="normal"><a href="#__codelineno-0-1022">1022</a></span> |
| <span class="normal"><a href="#__codelineno-0-1023">1023</a></span> |
| <span class="normal"><a href="#__codelineno-0-1024">1024</a></span> |
| <span class="normal"><a href="#__codelineno-0-1025">1025</a></span> |
| <span class="normal"><a href="#__codelineno-0-1026">1026</a></span> |
| <span class="normal"><a href="#__codelineno-0-1027">1027</a></span> |
| <span class="normal"><a href="#__codelineno-0-1028">1028</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-973" name="__codelineno-0-973"></a><span class="k">class</span><span class="w"> </span><span class="nc">CreateTableTransaction</span><span class="p">(</span><span class="n">Transaction</span><span class="p">):</span> |
| <a id="__codelineno-0-974" name="__codelineno-0-974"></a><span class="w"> </span><span class="sd">"""A transaction that involves the creation of a new table."""</span> |
| <a id="__codelineno-0-975" name="__codelineno-0-975"></a> |
| <a id="__codelineno-0-976" name="__codelineno-0-976"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_initial_changes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_metadata</span><span class="p">:</span> <span class="n">TableMetadata</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-977" name="__codelineno-0-977"></a><span class="w"> </span><span class="sd">"""Set the initial changes that can reconstruct the initial table metadata when creating the CreateTableTransaction."""</span> |
| <a id="__codelineno-0-978" name="__codelineno-0-978"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span> |
| <a id="__codelineno-0-979" name="__codelineno-0-979"></a> <span class="n">AssignUUIDUpdate</span><span class="p">(</span><span class="n">uuid</span><span class="o">=</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">table_uuid</span><span class="p">),</span> |
| <a id="__codelineno-0-980" name="__codelineno-0-980"></a> <span class="n">UpgradeFormatVersionUpdate</span><span class="p">(</span><span class="n">format_version</span><span class="o">=</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">),</span> |
| <a id="__codelineno-0-981" name="__codelineno-0-981"></a> <span class="p">)</span> |
| <a id="__codelineno-0-982" name="__codelineno-0-982"></a> |
| <a id="__codelineno-0-983" name="__codelineno-0-983"></a> <span class="n">schema</span><span class="p">:</span> <span class="n">Schema</span> <span class="o">=</span> <span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span> |
| <a id="__codelineno-0-984" name="__codelineno-0-984"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span> |
| <a id="__codelineno-0-985" name="__codelineno-0-985"></a> <span class="n">AddSchemaUpdate</span><span class="p">(</span><span class="n">schema_</span><span class="o">=</span><span class="n">schema</span><span class="p">),</span> |
| <a id="__codelineno-0-986" name="__codelineno-0-986"></a> <span class="n">SetCurrentSchemaUpdate</span><span class="p">(</span><span class="n">schema_id</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span> |
| <a id="__codelineno-0-987" name="__codelineno-0-987"></a> <span class="p">)</span> |
| <a id="__codelineno-0-988" name="__codelineno-0-988"></a> |
| <a id="__codelineno-0-989" name="__codelineno-0-989"></a> <span class="n">spec</span><span class="p">:</span> <span class="n">PartitionSpec</span> <span class="o">=</span> <span class="n">table_metadata</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span> |
| <a id="__codelineno-0-990" name="__codelineno-0-990"></a> <span class="k">if</span> <span class="n">spec</span><span class="o">.</span><span class="n">is_unpartitioned</span><span class="p">():</span> |
| <a id="__codelineno-0-991" name="__codelineno-0-991"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span><span class="n">AddPartitionSpecUpdate</span><span class="p">(</span><span class="n">spec</span><span class="o">=</span><span class="n">UNPARTITIONED_PARTITION_SPEC</span><span class="p">),)</span> |
| <a id="__codelineno-0-992" name="__codelineno-0-992"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-993" name="__codelineno-0-993"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span><span class="n">AddPartitionSpecUpdate</span><span class="p">(</span><span class="n">spec</span><span class="o">=</span><span class="n">spec</span><span class="p">),)</span> |
| <a id="__codelineno-0-994" name="__codelineno-0-994"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span><span class="n">SetDefaultSpecUpdate</span><span class="p">(</span><span class="n">spec_id</span><span class="o">=-</span><span class="mi">1</span><span class="p">),)</span> |
| <a id="__codelineno-0-995" name="__codelineno-0-995"></a> |
| <a id="__codelineno-0-996" name="__codelineno-0-996"></a> <span class="n">sort_order</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">SortOrder</span><span class="p">]</span> <span class="o">=</span> <span class="n">table_metadata</span><span class="o">.</span><span class="n">sort_order_by_id</span><span class="p">(</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">default_sort_order_id</span><span class="p">)</span> |
| <a id="__codelineno-0-997" name="__codelineno-0-997"></a> <span class="k">if</span> <span class="n">sort_order</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">sort_order</span><span class="o">.</span><span class="n">is_unsorted</span><span class="p">:</span> |
| <a id="__codelineno-0-998" name="__codelineno-0-998"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span><span class="n">AddSortOrderUpdate</span><span class="p">(</span><span class="n">sort_order</span><span class="o">=</span><span class="n">UNSORTED_SORT_ORDER</span><span class="p">),)</span> |
| <a id="__codelineno-0-999" name="__codelineno-0-999"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-1000" name="__codelineno-0-1000"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span><span class="n">AddSortOrderUpdate</span><span class="p">(</span><span class="n">sort_order</span><span class="o">=</span><span class="n">sort_order</span><span class="p">),)</span> |
| <a id="__codelineno-0-1001" name="__codelineno-0-1001"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span><span class="n">SetDefaultSortOrderUpdate</span><span class="p">(</span><span class="n">sort_order_id</span><span class="o">=-</span><span class="mi">1</span><span class="p">),)</span> |
| <a id="__codelineno-0-1002" name="__codelineno-0-1002"></a> |
| <a id="__codelineno-0-1003" name="__codelineno-0-1003"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="p">(</span> |
| <a id="__codelineno-0-1004" name="__codelineno-0-1004"></a> <span class="n">SetLocationUpdate</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">location</span><span class="p">),</span> |
| <a id="__codelineno-0-1005" name="__codelineno-0-1005"></a> <span class="n">SetPropertiesUpdate</span><span class="p">(</span><span class="n">updates</span><span class="o">=</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">properties</span><span class="p">),</span> |
| <a id="__codelineno-0-1006" name="__codelineno-0-1006"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1007" name="__codelineno-0-1007"></a> |
| <a id="__codelineno-0-1008" name="__codelineno-0-1008"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">:</span> <span class="n">StagedTable</span><span class="p">):</span> |
| <a id="__codelineno-0-1009" name="__codelineno-0-1009"></a> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
| <a id="__codelineno-0-1010" name="__codelineno-0-1010"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_initial_changes</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">metadata</span><span class="p">)</span> |
| <a id="__codelineno-0-1011" name="__codelineno-0-1011"></a> |
| <a id="__codelineno-0-1012" name="__codelineno-0-1012"></a> <span class="k">def</span><span class="w"> </span><span class="nf">commit_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1013" name="__codelineno-0-1013"></a><span class="w"> </span><span class="sd">"""Commit the changes to the catalog.</span> |
| <a id="__codelineno-0-1014" name="__codelineno-0-1014"></a> |
| <a id="__codelineno-0-1015" name="__codelineno-0-1015"></a><span class="sd"> In the case of a CreateTableTransaction, the only requirement is AssertCreate.</span> |
| <a id="__codelineno-0-1016" name="__codelineno-0-1016"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1017" name="__codelineno-0-1017"></a><span class="sd"> The table with the updates applied.</span> |
| <a id="__codelineno-0-1018" name="__codelineno-0-1018"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1019" name="__codelineno-0-1019"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-1020" name="__codelineno-0-1020"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">_do_commit</span><span class="p">(</span> <span class="c1"># pylint: disable=W0212</span> |
| <a id="__codelineno-0-1021" name="__codelineno-0-1021"></a> <span class="n">updates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">,</span> |
| <a id="__codelineno-0-1022" name="__codelineno-0-1022"></a> <span class="n">requirements</span><span class="o">=</span><span class="p">(</span><span class="n">AssertCreate</span><span class="p">(),),</span> |
| <a id="__codelineno-0-1023" name="__codelineno-0-1023"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1024" name="__codelineno-0-1024"></a> |
| <a id="__codelineno-0-1025" name="__codelineno-0-1025"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-1026" name="__codelineno-0-1026"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-1027" name="__codelineno-0-1027"></a> |
| <a id="__codelineno-0-1028" name="__codelineno-0-1028"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.CreateTableTransaction.commit_transaction" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">commit_transaction</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.CreateTableTransaction.commit_transaction" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Commit the changes to the catalog.</p> |
| <p>In the case of a CreateTableTransaction, the only requirement is AssertCreate. |
| Returns: |
| The table with the updates applied.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1012">1012</a></span> |
| <span class="normal"><a href="#__codelineno-0-1013">1013</a></span> |
| <span class="normal"><a href="#__codelineno-0-1014">1014</a></span> |
| <span class="normal"><a href="#__codelineno-0-1015">1015</a></span> |
| <span class="normal"><a href="#__codelineno-0-1016">1016</a></span> |
| <span class="normal"><a href="#__codelineno-0-1017">1017</a></span> |
| <span class="normal"><a href="#__codelineno-0-1018">1018</a></span> |
| <span class="normal"><a href="#__codelineno-0-1019">1019</a></span> |
| <span class="normal"><a href="#__codelineno-0-1020">1020</a></span> |
| <span class="normal"><a href="#__codelineno-0-1021">1021</a></span> |
| <span class="normal"><a href="#__codelineno-0-1022">1022</a></span> |
| <span class="normal"><a href="#__codelineno-0-1023">1023</a></span> |
| <span class="normal"><a href="#__codelineno-0-1024">1024</a></span> |
| <span class="normal"><a href="#__codelineno-0-1025">1025</a></span> |
| <span class="normal"><a href="#__codelineno-0-1026">1026</a></span> |
| <span class="normal"><a href="#__codelineno-0-1027">1027</a></span> |
| <span class="normal"><a href="#__codelineno-0-1028">1028</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1012" name="__codelineno-0-1012"></a><span class="k">def</span><span class="w"> </span><span class="nf">commit_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1013" name="__codelineno-0-1013"></a><span class="w"> </span><span class="sd">"""Commit the changes to the catalog.</span> |
| <a id="__codelineno-0-1014" name="__codelineno-0-1014"></a> |
| <a id="__codelineno-0-1015" name="__codelineno-0-1015"></a><span class="sd"> In the case of a CreateTableTransaction, the only requirement is AssertCreate.</span> |
| <a id="__codelineno-0-1016" name="__codelineno-0-1016"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1017" name="__codelineno-0-1017"></a><span class="sd"> The table with the updates applied.</span> |
| <a id="__codelineno-0-1018" name="__codelineno-0-1018"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1019" name="__codelineno-0-1019"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-1020" name="__codelineno-0-1020"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">_do_commit</span><span class="p">(</span> <span class="c1"># pylint: disable=W0212</span> |
| <a id="__codelineno-0-1021" name="__codelineno-0-1021"></a> <span class="n">updates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">,</span> |
| <a id="__codelineno-0-1022" name="__codelineno-0-1022"></a> <span class="n">requirements</span><span class="o">=</span><span class="p">(</span><span class="n">AssertCreate</span><span class="p">(),),</span> |
| <a id="__codelineno-0-1023" name="__codelineno-0-1023"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1024" name="__codelineno-0-1024"></a> |
| <a id="__codelineno-0-1025" name="__codelineno-0-1025"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-1026" name="__codelineno-0-1026"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-1027" name="__codelineno-0-1027"></a> |
| <a id="__codelineno-0-1028" name="__codelineno-0-1028"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.DataScan" class="doc doc-heading"> |
| <code>DataScan</code> |
| |
| |
| <a href="#pyiceberg.table.DataScan" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="TableScan (pyiceberg.table.TableScan)" href="#pyiceberg.table.TableScan">TableScan</a></code></p> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1873">1873</a></span> |
| <span class="normal"><a href="#__codelineno-0-1874">1874</a></span> |
| <span class="normal"><a href="#__codelineno-0-1875">1875</a></span> |
| <span class="normal"><a href="#__codelineno-0-1876">1876</a></span> |
| <span class="normal"><a href="#__codelineno-0-1877">1877</a></span> |
| <span class="normal"><a href="#__codelineno-0-1878">1878</a></span> |
| <span class="normal"><a href="#__codelineno-0-1879">1879</a></span> |
| <span class="normal"><a href="#__codelineno-0-1880">1880</a></span> |
| <span class="normal"><a href="#__codelineno-0-1881">1881</a></span> |
| <span class="normal"><a href="#__codelineno-0-1882">1882</a></span> |
| <span class="normal"><a href="#__codelineno-0-1883">1883</a></span> |
| <span class="normal"><a href="#__codelineno-0-1884">1884</a></span> |
| <span class="normal"><a href="#__codelineno-0-1885">1885</a></span> |
| <span class="normal"><a href="#__codelineno-0-1886">1886</a></span> |
| <span class="normal"><a href="#__codelineno-0-1887">1887</a></span> |
| <span class="normal"><a href="#__codelineno-0-1888">1888</a></span> |
| <span class="normal"><a href="#__codelineno-0-1889">1889</a></span> |
| <span class="normal"><a href="#__codelineno-0-1890">1890</a></span> |
| <span class="normal"><a href="#__codelineno-0-1891">1891</a></span> |
| <span class="normal"><a href="#__codelineno-0-1892">1892</a></span> |
| <span class="normal"><a href="#__codelineno-0-1893">1893</a></span> |
| <span class="normal"><a href="#__codelineno-0-1894">1894</a></span> |
| <span class="normal"><a href="#__codelineno-0-1895">1895</a></span> |
| <span class="normal"><a href="#__codelineno-0-1896">1896</a></span> |
| <span class="normal"><a href="#__codelineno-0-1897">1897</a></span> |
| <span class="normal"><a href="#__codelineno-0-1898">1898</a></span> |
| <span class="normal"><a href="#__codelineno-0-1899">1899</a></span> |
| <span class="normal"><a href="#__codelineno-0-1900">1900</a></span> |
| <span class="normal"><a href="#__codelineno-0-1901">1901</a></span> |
| <span class="normal"><a href="#__codelineno-0-1902">1902</a></span> |
| <span class="normal"><a href="#__codelineno-0-1903">1903</a></span> |
| <span class="normal"><a href="#__codelineno-0-1904">1904</a></span> |
| <span class="normal"><a href="#__codelineno-0-1905">1905</a></span> |
| <span class="normal"><a href="#__codelineno-0-1906">1906</a></span> |
| <span class="normal"><a href="#__codelineno-0-1907">1907</a></span> |
| <span class="normal"><a href="#__codelineno-0-1908">1908</a></span> |
| <span class="normal"><a href="#__codelineno-0-1909">1909</a></span> |
| <span class="normal"><a href="#__codelineno-0-1910">1910</a></span> |
| <span class="normal"><a href="#__codelineno-0-1911">1911</a></span> |
| <span class="normal"><a href="#__codelineno-0-1912">1912</a></span> |
| <span class="normal"><a href="#__codelineno-0-1913">1913</a></span> |
| <span class="normal"><a href="#__codelineno-0-1914">1914</a></span> |
| <span class="normal"><a href="#__codelineno-0-1915">1915</a></span> |
| <span class="normal"><a href="#__codelineno-0-1916">1916</a></span> |
| <span class="normal"><a href="#__codelineno-0-1917">1917</a></span> |
| <span class="normal"><a href="#__codelineno-0-1918">1918</a></span> |
| <span class="normal"><a href="#__codelineno-0-1919">1919</a></span> |
| <span class="normal"><a href="#__codelineno-0-1920">1920</a></span> |
| <span class="normal"><a href="#__codelineno-0-1921">1921</a></span> |
| <span class="normal"><a href="#__codelineno-0-1922">1922</a></span> |
| <span class="normal"><a href="#__codelineno-0-1923">1923</a></span> |
| <span class="normal"><a href="#__codelineno-0-1924">1924</a></span> |
| <span class="normal"><a href="#__codelineno-0-1925">1925</a></span> |
| <span class="normal"><a href="#__codelineno-0-1926">1926</a></span> |
| <span class="normal"><a href="#__codelineno-0-1927">1927</a></span> |
| <span class="normal"><a href="#__codelineno-0-1928">1928</a></span> |
| <span class="normal"><a href="#__codelineno-0-1929">1929</a></span> |
| <span class="normal"><a href="#__codelineno-0-1930">1930</a></span> |
| <span class="normal"><a href="#__codelineno-0-1931">1931</a></span> |
| <span class="normal"><a href="#__codelineno-0-1932">1932</a></span> |
| <span class="normal"><a href="#__codelineno-0-1933">1933</a></span> |
| <span class="normal"><a href="#__codelineno-0-1934">1934</a></span> |
| <span class="normal"><a href="#__codelineno-0-1935">1935</a></span> |
| <span class="normal"><a href="#__codelineno-0-1936">1936</a></span> |
| <span class="normal"><a href="#__codelineno-0-1937">1937</a></span> |
| <span class="normal"><a href="#__codelineno-0-1938">1938</a></span> |
| <span class="normal"><a href="#__codelineno-0-1939">1939</a></span> |
| <span class="normal"><a href="#__codelineno-0-1940">1940</a></span> |
| <span class="normal"><a href="#__codelineno-0-1941">1941</a></span> |
| <span class="normal"><a href="#__codelineno-0-1942">1942</a></span> |
| <span class="normal"><a href="#__codelineno-0-1943">1943</a></span> |
| <span class="normal"><a href="#__codelineno-0-1944">1944</a></span> |
| <span class="normal"><a href="#__codelineno-0-1945">1945</a></span> |
| <span class="normal"><a href="#__codelineno-0-1946">1946</a></span> |
| <span class="normal"><a href="#__codelineno-0-1947">1947</a></span> |
| <span class="normal"><a href="#__codelineno-0-1948">1948</a></span> |
| <span class="normal"><a href="#__codelineno-0-1949">1949</a></span> |
| <span class="normal"><a href="#__codelineno-0-1950">1950</a></span> |
| <span class="normal"><a href="#__codelineno-0-1951">1951</a></span> |
| <span class="normal"><a href="#__codelineno-0-1952">1952</a></span> |
| <span class="normal"><a href="#__codelineno-0-1953">1953</a></span> |
| <span class="normal"><a href="#__codelineno-0-1954">1954</a></span> |
| <span class="normal"><a href="#__codelineno-0-1955">1955</a></span> |
| <span class="normal"><a href="#__codelineno-0-1956">1956</a></span> |
| <span class="normal"><a href="#__codelineno-0-1957">1957</a></span> |
| <span class="normal"><a href="#__codelineno-0-1958">1958</a></span> |
| <span class="normal"><a href="#__codelineno-0-1959">1959</a></span> |
| <span class="normal"><a href="#__codelineno-0-1960">1960</a></span> |
| <span class="normal"><a href="#__codelineno-0-1961">1961</a></span> |
| <span class="normal"><a href="#__codelineno-0-1962">1962</a></span> |
| <span class="normal"><a href="#__codelineno-0-1963">1963</a></span> |
| <span class="normal"><a href="#__codelineno-0-1964">1964</a></span> |
| <span class="normal"><a href="#__codelineno-0-1965">1965</a></span> |
| <span class="normal"><a href="#__codelineno-0-1966">1966</a></span> |
| <span class="normal"><a href="#__codelineno-0-1967">1967</a></span> |
| <span class="normal"><a href="#__codelineno-0-1968">1968</a></span> |
| <span class="normal"><a href="#__codelineno-0-1969">1969</a></span> |
| <span class="normal"><a href="#__codelineno-0-1970">1970</a></span> |
| <span class="normal"><a href="#__codelineno-0-1971">1971</a></span> |
| <span class="normal"><a href="#__codelineno-0-1972">1972</a></span> |
| <span class="normal"><a href="#__codelineno-0-1973">1973</a></span> |
| <span class="normal"><a href="#__codelineno-0-1974">1974</a></span> |
| <span class="normal"><a href="#__codelineno-0-1975">1975</a></span> |
| <span class="normal"><a href="#__codelineno-0-1976">1976</a></span> |
| <span class="normal"><a href="#__codelineno-0-1977">1977</a></span> |
| <span class="normal"><a href="#__codelineno-0-1978">1978</a></span> |
| <span class="normal"><a href="#__codelineno-0-1979">1979</a></span> |
| <span class="normal"><a href="#__codelineno-0-1980">1980</a></span> |
| <span class="normal"><a href="#__codelineno-0-1981">1981</a></span> |
| <span class="normal"><a href="#__codelineno-0-1982">1982</a></span> |
| <span class="normal"><a href="#__codelineno-0-1983">1983</a></span> |
| <span class="normal"><a href="#__codelineno-0-1984">1984</a></span> |
| <span class="normal"><a href="#__codelineno-0-1985">1985</a></span> |
| <span class="normal"><a href="#__codelineno-0-1986">1986</a></span> |
| <span class="normal"><a href="#__codelineno-0-1987">1987</a></span> |
| <span class="normal"><a href="#__codelineno-0-1988">1988</a></span> |
| <span class="normal"><a href="#__codelineno-0-1989">1989</a></span> |
| <span class="normal"><a href="#__codelineno-0-1990">1990</a></span> |
| <span class="normal"><a href="#__codelineno-0-1991">1991</a></span> |
| <span class="normal"><a href="#__codelineno-0-1992">1992</a></span> |
| <span class="normal"><a href="#__codelineno-0-1993">1993</a></span> |
| <span class="normal"><a href="#__codelineno-0-1994">1994</a></span> |
| <span class="normal"><a href="#__codelineno-0-1995">1995</a></span> |
| <span class="normal"><a href="#__codelineno-0-1996">1996</a></span> |
| <span class="normal"><a href="#__codelineno-0-1997">1997</a></span> |
| <span class="normal"><a href="#__codelineno-0-1998">1998</a></span> |
| <span class="normal"><a href="#__codelineno-0-1999">1999</a></span> |
| <span class="normal"><a href="#__codelineno-0-2000">2000</a></span> |
| <span class="normal"><a href="#__codelineno-0-2001">2001</a></span> |
| <span class="normal"><a href="#__codelineno-0-2002">2002</a></span> |
| <span class="normal"><a href="#__codelineno-0-2003">2003</a></span> |
| <span class="normal"><a href="#__codelineno-0-2004">2004</a></span> |
| <span class="normal"><a href="#__codelineno-0-2005">2005</a></span> |
| <span class="normal"><a href="#__codelineno-0-2006">2006</a></span> |
| <span class="normal"><a href="#__codelineno-0-2007">2007</a></span> |
| <span class="normal"><a href="#__codelineno-0-2008">2008</a></span> |
| <span class="normal"><a href="#__codelineno-0-2009">2009</a></span> |
| <span class="normal"><a href="#__codelineno-0-2010">2010</a></span> |
| <span class="normal"><a href="#__codelineno-0-2011">2011</a></span> |
| <span class="normal"><a href="#__codelineno-0-2012">2012</a></span> |
| <span class="normal"><a href="#__codelineno-0-2013">2013</a></span> |
| <span class="normal"><a href="#__codelineno-0-2014">2014</a></span> |
| <span class="normal"><a href="#__codelineno-0-2015">2015</a></span> |
| <span class="normal"><a href="#__codelineno-0-2016">2016</a></span> |
| <span class="normal"><a href="#__codelineno-0-2017">2017</a></span> |
| <span class="normal"><a href="#__codelineno-0-2018">2018</a></span> |
| <span class="normal"><a href="#__codelineno-0-2019">2019</a></span> |
| <span class="normal"><a href="#__codelineno-0-2020">2020</a></span> |
| <span class="normal"><a href="#__codelineno-0-2021">2021</a></span> |
| <span class="normal"><a href="#__codelineno-0-2022">2022</a></span> |
| <span class="normal"><a href="#__codelineno-0-2023">2023</a></span> |
| <span class="normal"><a href="#__codelineno-0-2024">2024</a></span> |
| <span class="normal"><a href="#__codelineno-0-2025">2025</a></span> |
| <span class="normal"><a href="#__codelineno-0-2026">2026</a></span> |
| <span class="normal"><a href="#__codelineno-0-2027">2027</a></span> |
| <span class="normal"><a href="#__codelineno-0-2028">2028</a></span> |
| <span class="normal"><a href="#__codelineno-0-2029">2029</a></span> |
| <span class="normal"><a href="#__codelineno-0-2030">2030</a></span> |
| <span class="normal"><a href="#__codelineno-0-2031">2031</a></span> |
| <span class="normal"><a href="#__codelineno-0-2032">2032</a></span> |
| <span class="normal"><a href="#__codelineno-0-2033">2033</a></span> |
| <span class="normal"><a href="#__codelineno-0-2034">2034</a></span> |
| <span class="normal"><a href="#__codelineno-0-2035">2035</a></span> |
| <span class="normal"><a href="#__codelineno-0-2036">2036</a></span> |
| <span class="normal"><a href="#__codelineno-0-2037">2037</a></span> |
| <span class="normal"><a href="#__codelineno-0-2038">2038</a></span> |
| <span class="normal"><a href="#__codelineno-0-2039">2039</a></span> |
| <span class="normal"><a href="#__codelineno-0-2040">2040</a></span> |
| <span class="normal"><a href="#__codelineno-0-2041">2041</a></span> |
| <span class="normal"><a href="#__codelineno-0-2042">2042</a></span> |
| <span class="normal"><a href="#__codelineno-0-2043">2043</a></span> |
| <span class="normal"><a href="#__codelineno-0-2044">2044</a></span> |
| <span class="normal"><a href="#__codelineno-0-2045">2045</a></span> |
| <span class="normal"><a href="#__codelineno-0-2046">2046</a></span> |
| <span class="normal"><a href="#__codelineno-0-2047">2047</a></span> |
| <span class="normal"><a href="#__codelineno-0-2048">2048</a></span> |
| <span class="normal"><a href="#__codelineno-0-2049">2049</a></span> |
| <span class="normal"><a href="#__codelineno-0-2050">2050</a></span> |
| <span class="normal"><a href="#__codelineno-0-2051">2051</a></span> |
| <span class="normal"><a href="#__codelineno-0-2052">2052</a></span> |
| <span class="normal"><a href="#__codelineno-0-2053">2053</a></span> |
| <span class="normal"><a href="#__codelineno-0-2054">2054</a></span> |
| <span class="normal"><a href="#__codelineno-0-2055">2055</a></span> |
| <span class="normal"><a href="#__codelineno-0-2056">2056</a></span> |
| <span class="normal"><a href="#__codelineno-0-2057">2057</a></span> |
| <span class="normal"><a href="#__codelineno-0-2058">2058</a></span> |
| <span class="normal"><a href="#__codelineno-0-2059">2059</a></span> |
| <span class="normal"><a href="#__codelineno-0-2060">2060</a></span> |
| <span class="normal"><a href="#__codelineno-0-2061">2061</a></span> |
| <span class="normal"><a href="#__codelineno-0-2062">2062</a></span> |
| <span class="normal"><a href="#__codelineno-0-2063">2063</a></span> |
| <span class="normal"><a href="#__codelineno-0-2064">2064</a></span> |
| <span class="normal"><a href="#__codelineno-0-2065">2065</a></span> |
| <span class="normal"><a href="#__codelineno-0-2066">2066</a></span> |
| <span class="normal"><a href="#__codelineno-0-2067">2067</a></span> |
| <span class="normal"><a href="#__codelineno-0-2068">2068</a></span> |
| <span class="normal"><a href="#__codelineno-0-2069">2069</a></span> |
| <span class="normal"><a href="#__codelineno-0-2070">2070</a></span> |
| <span class="normal"><a href="#__codelineno-0-2071">2071</a></span> |
| <span class="normal"><a href="#__codelineno-0-2072">2072</a></span> |
| <span class="normal"><a href="#__codelineno-0-2073">2073</a></span> |
| <span class="normal"><a href="#__codelineno-0-2074">2074</a></span> |
| <span class="normal"><a href="#__codelineno-0-2075">2075</a></span> |
| <span class="normal"><a href="#__codelineno-0-2076">2076</a></span> |
| <span class="normal"><a href="#__codelineno-0-2077">2077</a></span> |
| <span class="normal"><a href="#__codelineno-0-2078">2078</a></span> |
| <span class="normal"><a href="#__codelineno-0-2079">2079</a></span> |
| <span class="normal"><a href="#__codelineno-0-2080">2080</a></span> |
| <span class="normal"><a href="#__codelineno-0-2081">2081</a></span> |
| <span class="normal"><a href="#__codelineno-0-2082">2082</a></span> |
| <span class="normal"><a href="#__codelineno-0-2083">2083</a></span> |
| <span class="normal"><a href="#__codelineno-0-2084">2084</a></span> |
| <span class="normal"><a href="#__codelineno-0-2085">2085</a></span> |
| <span class="normal"><a href="#__codelineno-0-2086">2086</a></span> |
| <span class="normal"><a href="#__codelineno-0-2087">2087</a></span> |
| <span class="normal"><a href="#__codelineno-0-2088">2088</a></span> |
| <span class="normal"><a href="#__codelineno-0-2089">2089</a></span> |
| <span class="normal"><a href="#__codelineno-0-2090">2090</a></span> |
| <span class="normal"><a href="#__codelineno-0-2091">2091</a></span> |
| <span class="normal"><a href="#__codelineno-0-2092">2092</a></span> |
| <span class="normal"><a href="#__codelineno-0-2093">2093</a></span> |
| <span class="normal"><a href="#__codelineno-0-2094">2094</a></span> |
| <span class="normal"><a href="#__codelineno-0-2095">2095</a></span> |
| <span class="normal"><a href="#__codelineno-0-2096">2096</a></span> |
| <span class="normal"><a href="#__codelineno-0-2097">2097</a></span> |
| <span class="normal"><a href="#__codelineno-0-2098">2098</a></span> |
| <span class="normal"><a href="#__codelineno-0-2099">2099</a></span> |
| <span class="normal"><a href="#__codelineno-0-2100">2100</a></span> |
| <span class="normal"><a href="#__codelineno-0-2101">2101</a></span> |
| <span class="normal"><a href="#__codelineno-0-2102">2102</a></span> |
| <span class="normal"><a href="#__codelineno-0-2103">2103</a></span> |
| <span class="normal"><a href="#__codelineno-0-2104">2104</a></span> |
| <span class="normal"><a href="#__codelineno-0-2105">2105</a></span> |
| <span class="normal"><a href="#__codelineno-0-2106">2106</a></span> |
| <span class="normal"><a href="#__codelineno-0-2107">2107</a></span> |
| <span class="normal"><a href="#__codelineno-0-2108">2108</a></span> |
| <span class="normal"><a href="#__codelineno-0-2109">2109</a></span> |
| <span class="normal"><a href="#__codelineno-0-2110">2110</a></span> |
| <span class="normal"><a href="#__codelineno-0-2111">2111</a></span> |
| <span class="normal"><a href="#__codelineno-0-2112">2112</a></span> |
| <span class="normal"><a href="#__codelineno-0-2113">2113</a></span> |
| <span class="normal"><a href="#__codelineno-0-2114">2114</a></span> |
| <span class="normal"><a href="#__codelineno-0-2115">2115</a></span> |
| <span class="normal"><a href="#__codelineno-0-2116">2116</a></span> |
| <span class="normal"><a href="#__codelineno-0-2117">2117</a></span> |
| <span class="normal"><a href="#__codelineno-0-2118">2118</a></span> |
| <span class="normal"><a href="#__codelineno-0-2119">2119</a></span> |
| <span class="normal"><a href="#__codelineno-0-2120">2120</a></span> |
| <span class="normal"><a href="#__codelineno-0-2121">2121</a></span> |
| <span class="normal"><a href="#__codelineno-0-2122">2122</a></span> |
| <span class="normal"><a href="#__codelineno-0-2123">2123</a></span> |
| <span class="normal"><a href="#__codelineno-0-2124">2124</a></span> |
| <span class="normal"><a href="#__codelineno-0-2125">2125</a></span> |
| <span class="normal"><a href="#__codelineno-0-2126">2126</a></span> |
| <span class="normal"><a href="#__codelineno-0-2127">2127</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1873" name="__codelineno-0-1873"></a><span class="k">class</span><span class="w"> </span><span class="nc">DataScan</span><span class="p">(</span><span class="n">TableScan</span><span class="p">):</span> |
| <a id="__codelineno-0-1874" name="__codelineno-0-1874"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_build_partition_projection</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">spec_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">BooleanExpression</span><span class="p">:</span> |
| <a id="__codelineno-0-1875" name="__codelineno-0-1875"></a> <span class="n">project</span> <span class="o">=</span> <span class="n">inclusive_projection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">specs</span><span class="p">()[</span><span class="n">spec_id</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-1876" name="__codelineno-0-1876"></a> <span class="k">return</span> <span class="n">project</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-1877" name="__codelineno-0-1877"></a> |
| <a id="__codelineno-0-1878" name="__codelineno-0-1878"></a> <span class="nd">@cached_property</span> |
| <a id="__codelineno-0-1879" name="__codelineno-0-1879"></a> <span class="k">def</span><span class="w"> </span><span class="nf">partition_filters</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">KeyDefaultDict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">]:</span> |
| <a id="__codelineno-0-1880" name="__codelineno-0-1880"></a> <span class="k">return</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_partition_projection</span><span class="p">)</span> |
| <a id="__codelineno-0-1881" name="__codelineno-0-1881"></a> |
| <a id="__codelineno-0-1882" name="__codelineno-0-1882"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_build_manifest_evaluator</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">spec_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="n">ManifestFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]:</span> |
| <a id="__codelineno-0-1883" name="__codelineno-0-1883"></a> <span class="n">spec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">specs</span><span class="p">()[</span><span class="n">spec_id</span><span class="p">]</span> |
| <a id="__codelineno-0-1884" name="__codelineno-0-1884"></a> <span class="k">return</span> <span class="n">manifest_evaluator</span><span class="p">(</span><span class="n">spec</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition_filters</span><span class="p">[</span><span class="n">spec_id</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-1885" name="__codelineno-0-1885"></a> |
| <a id="__codelineno-0-1886" name="__codelineno-0-1886"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_build_partition_evaluator</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">spec_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]:</span> |
| <a id="__codelineno-0-1887" name="__codelineno-0-1887"></a> <span class="n">spec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">specs</span><span class="p">()[</span><span class="n">spec_id</span><span class="p">]</span> |
| <a id="__codelineno-0-1888" name="__codelineno-0-1888"></a> <span class="n">partition_type</span> <span class="o">=</span> <span class="n">spec</span><span class="o">.</span><span class="n">partition_type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">())</span> |
| <a id="__codelineno-0-1889" name="__codelineno-0-1889"></a> <span class="n">partition_schema</span> <span class="o">=</span> <span class="n">Schema</span><span class="p">(</span><span class="o">*</span><span class="n">partition_type</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span> |
| <a id="__codelineno-0-1890" name="__codelineno-0-1890"></a> <span class="n">partition_expr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition_filters</span><span class="p">[</span><span class="n">spec_id</span><span class="p">]</span> |
| <a id="__codelineno-0-1891" name="__codelineno-0-1891"></a> |
| <a id="__codelineno-0-1892" name="__codelineno-0-1892"></a> <span class="c1"># The lambda created here is run in multiple threads.</span> |
| <a id="__codelineno-0-1893" name="__codelineno-0-1893"></a> <span class="c1"># So we avoid creating _EvaluatorExpression methods bound to a single</span> |
| <a id="__codelineno-0-1894" name="__codelineno-0-1894"></a> <span class="c1"># shared instance across multiple threads.</span> |
| <a id="__codelineno-0-1895" name="__codelineno-0-1895"></a> <span class="k">return</span> <span class="k">lambda</span> <span class="n">data_file</span><span class="p">:</span> <span class="n">expression_evaluator</span><span class="p">(</span><span class="n">partition_schema</span><span class="p">,</span> <span class="n">partition_expr</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">)(</span><span class="n">data_file</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span> |
| <a id="__codelineno-0-1896" name="__codelineno-0-1896"></a> |
| <a id="__codelineno-0-1897" name="__codelineno-0-1897"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_build_metrics_evaluator</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]:</span> |
| <a id="__codelineno-0-1898" name="__codelineno-0-1898"></a> <span class="n">schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span> |
| <a id="__codelineno-0-1899" name="__codelineno-0-1899"></a> <span class="n">include_empty_files</span> <span class="o">=</span> <span class="n">strtobool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"include_empty_files"</span><span class="p">,</span> <span class="s2">"false"</span><span class="p">))</span> |
| <a id="__codelineno-0-1900" name="__codelineno-0-1900"></a> |
| <a id="__codelineno-0-1901" name="__codelineno-0-1901"></a> <span class="c1"># The lambda created here is run in multiple threads.</span> |
| <a id="__codelineno-0-1902" name="__codelineno-0-1902"></a> <span class="c1"># So we avoid creating _InclusiveMetricsEvaluator methods bound to a single</span> |
| <a id="__codelineno-0-1903" name="__codelineno-0-1903"></a> <span class="c1"># shared instance across multiple threads.</span> |
| <a id="__codelineno-0-1904" name="__codelineno-0-1904"></a> <span class="k">return</span> <span class="k">lambda</span> <span class="n">data_file</span><span class="p">:</span> <span class="n">_InclusiveMetricsEvaluator</span><span class="p">(</span> |
| <a id="__codelineno-0-1905" name="__codelineno-0-1905"></a> <span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-1906" name="__codelineno-0-1906"></a> <span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-1907" name="__codelineno-0-1907"></a> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1908" name="__codelineno-0-1908"></a> <span class="n">include_empty_files</span><span class="p">,</span> |
| <a id="__codelineno-0-1909" name="__codelineno-0-1909"></a> <span class="p">)</span><span class="o">.</span><span class="n">eval</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-1910" name="__codelineno-0-1910"></a> |
| <a id="__codelineno-0-1911" name="__codelineno-0-1911"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_build_residual_evaluator</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">spec_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="n">ResidualEvaluator</span><span class="p">]:</span> |
| <a id="__codelineno-0-1912" name="__codelineno-0-1912"></a> <span class="n">spec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">specs</span><span class="p">()[</span><span class="n">spec_id</span><span class="p">]</span> |
| <a id="__codelineno-0-1913" name="__codelineno-0-1913"></a> |
| <a id="__codelineno-0-1914" name="__codelineno-0-1914"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.expressions.visitors</span><span class="w"> </span><span class="kn">import</span> <span class="n">residual_evaluator_of</span> |
| <a id="__codelineno-0-1915" name="__codelineno-0-1915"></a> |
| <a id="__codelineno-0-1916" name="__codelineno-0-1916"></a> <span class="c1"># The lambda created here is run in multiple threads.</span> |
| <a id="__codelineno-0-1917" name="__codelineno-0-1917"></a> <span class="c1"># So we avoid creating _EvaluatorExpression methods bound to a single</span> |
| <a id="__codelineno-0-1918" name="__codelineno-0-1918"></a> <span class="c1"># shared instance across multiple threads.</span> |
| <a id="__codelineno-0-1919" name="__codelineno-0-1919"></a> <span class="k">return</span> <span class="k">lambda</span> <span class="n">datafile</span><span class="p">:</span> <span class="p">(</span> |
| <a id="__codelineno-0-1920" name="__codelineno-0-1920"></a> <span class="n">residual_evaluator_of</span><span class="p">(</span> |
| <a id="__codelineno-0-1921" name="__codelineno-0-1921"></a> <span class="n">spec</span><span class="o">=</span><span class="n">spec</span><span class="p">,</span> |
| <a id="__codelineno-0-1922" name="__codelineno-0-1922"></a> <span class="n">expr</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-1923" name="__codelineno-0-1923"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1924" name="__codelineno-0-1924"></a> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-1925" name="__codelineno-0-1925"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1926" name="__codelineno-0-1926"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1927" name="__codelineno-0-1927"></a> |
| <a id="__codelineno-0-1928" name="__codelineno-0-1928"></a> <span class="nd">@staticmethod</span> |
| <a id="__codelineno-0-1929" name="__codelineno-0-1929"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_check_sequence_number</span><span class="p">(</span><span class="n">min_sequence_number</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">manifest</span><span class="p">:</span> <span class="n">ManifestFile</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <a id="__codelineno-0-1930" name="__codelineno-0-1930"></a><span class="w"> </span><span class="sd">"""Ensure that no manifests are loaded that contain deletes that are older than the data.</span> |
| <a id="__codelineno-0-1931" name="__codelineno-0-1931"></a> |
| <a id="__codelineno-0-1932" name="__codelineno-0-1932"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1933" name="__codelineno-0-1933"></a><span class="sd"> min_sequence_number (int): The minimal sequence number.</span> |
| <a id="__codelineno-0-1934" name="__codelineno-0-1934"></a><span class="sd"> manifest (ManifestFile): A ManifestFile that can be either data or deletes.</span> |
| <a id="__codelineno-0-1935" name="__codelineno-0-1935"></a> |
| <a id="__codelineno-0-1936" name="__codelineno-0-1936"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1937" name="__codelineno-0-1937"></a><span class="sd"> Boolean indicating if it is either a data file, or a relevant delete file.</span> |
| <a id="__codelineno-0-1938" name="__codelineno-0-1938"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1939" name="__codelineno-0-1939"></a> <span class="k">return</span> <span class="n">manifest</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">ManifestContent</span><span class="o">.</span><span class="n">DATA</span> <span class="ow">or</span> <span class="p">(</span> |
| <a id="__codelineno-0-1940" name="__codelineno-0-1940"></a> <span class="c1"># Not interested in deletes that are older than the data</span> |
| <a id="__codelineno-0-1941" name="__codelineno-0-1941"></a> <span class="n">manifest</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">ManifestContent</span><span class="o">.</span><span class="n">DELETES</span> |
| <a id="__codelineno-0-1942" name="__codelineno-0-1942"></a> <span class="ow">and</span> <span class="p">(</span><span class="n">manifest</span><span class="o">.</span><span class="n">sequence_number</span> <span class="ow">or</span> <span class="n">INITIAL_SEQUENCE_NUMBER</span><span class="p">)</span> <span class="o">>=</span> <span class="n">min_sequence_number</span> |
| <a id="__codelineno-0-1943" name="__codelineno-0-1943"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1944" name="__codelineno-0-1944"></a> |
| <a id="__codelineno-0-1945" name="__codelineno-0-1945"></a> <span class="k">def</span><span class="w"> </span><span class="nf">plan_files</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">FileScanTask</span><span class="p">]:</span> |
| <a id="__codelineno-0-1946" name="__codelineno-0-1946"></a><span class="w"> </span><span class="sd">"""Plans the relevant files by filtering on the PartitionSpecs.</span> |
| <a id="__codelineno-0-1947" name="__codelineno-0-1947"></a> |
| <a id="__codelineno-0-1948" name="__codelineno-0-1948"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1949" name="__codelineno-0-1949"></a><span class="sd"> List of FileScanTasks that contain both data and delete files.</span> |
| <a id="__codelineno-0-1950" name="__codelineno-0-1950"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1951" name="__codelineno-0-1951"></a> <span class="n">snapshot</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot</span><span class="p">()</span> |
| <a id="__codelineno-0-1952" name="__codelineno-0-1952"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">snapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-1953" name="__codelineno-0-1953"></a> <span class="k">return</span> <span class="nb">iter</span><span class="p">([])</span> |
| <a id="__codelineno-0-1954" name="__codelineno-0-1954"></a> |
| <a id="__codelineno-0-1955" name="__codelineno-0-1955"></a> <span class="c1"># step 1: filter manifests using partition summaries</span> |
| <a id="__codelineno-0-1956" name="__codelineno-0-1956"></a> <span class="c1"># the filter depends on the partition spec used to write the manifest file, so create a cache of filters for each spec id</span> |
| <a id="__codelineno-0-1957" name="__codelineno-0-1957"></a> |
| <a id="__codelineno-0-1958" name="__codelineno-0-1958"></a> <span class="n">manifest_evaluators</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">ManifestFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]]</span> <span class="o">=</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_manifest_evaluator</span><span class="p">)</span> |
| <a id="__codelineno-0-1959" name="__codelineno-0-1959"></a> |
| <a id="__codelineno-0-1960" name="__codelineno-0-1960"></a> <span class="n">residual_evaluators</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="n">ResidualEvaluator</span><span class="p">]]</span> <span class="o">=</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_residual_evaluator</span><span class="p">)</span> |
| <a id="__codelineno-0-1961" name="__codelineno-0-1961"></a> |
| <a id="__codelineno-0-1962" name="__codelineno-0-1962"></a> <span class="n">manifests</span> <span class="o">=</span> <span class="p">[</span> |
| <a id="__codelineno-0-1963" name="__codelineno-0-1963"></a> <span class="n">manifest_file</span> |
| <a id="__codelineno-0-1964" name="__codelineno-0-1964"></a> <span class="k">for</span> <span class="n">manifest_file</span> <span class="ow">in</span> <span class="n">snapshot</span><span class="o">.</span><span class="n">manifests</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">)</span> |
| <a id="__codelineno-0-1965" name="__codelineno-0-1965"></a> <span class="k">if</span> <span class="n">manifest_evaluators</span><span class="p">[</span><span class="n">manifest_file</span><span class="o">.</span><span class="n">partition_spec_id</span><span class="p">](</span><span class="n">manifest_file</span><span class="p">)</span> |
| <a id="__codelineno-0-1966" name="__codelineno-0-1966"></a> <span class="p">]</span> |
| <a id="__codelineno-0-1967" name="__codelineno-0-1967"></a> |
| <a id="__codelineno-0-1968" name="__codelineno-0-1968"></a> <span class="c1"># step 2: filter the data files in each manifest</span> |
| <a id="__codelineno-0-1969" name="__codelineno-0-1969"></a> <span class="c1"># this filter depends on the partition spec used to write the manifest file</span> |
| <a id="__codelineno-0-1970" name="__codelineno-0-1970"></a> |
| <a id="__codelineno-0-1971" name="__codelineno-0-1971"></a> <span class="n">partition_evaluators</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]]</span> <span class="o">=</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_partition_evaluator</span><span class="p">)</span> |
| <a id="__codelineno-0-1972" name="__codelineno-0-1972"></a> |
| <a id="__codelineno-0-1973" name="__codelineno-0-1973"></a> <span class="n">min_sequence_number</span> <span class="o">=</span> <span class="n">_min_sequence_number</span><span class="p">(</span><span class="n">manifests</span><span class="p">)</span> |
| <a id="__codelineno-0-1974" name="__codelineno-0-1974"></a> |
| <a id="__codelineno-0-1975" name="__codelineno-0-1975"></a> <span class="n">data_entries</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">ManifestEntry</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-1976" name="__codelineno-0-1976"></a> <span class="n">positional_delete_entries</span> <span class="o">=</span> <span class="n">SortedList</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">entry</span><span class="p">:</span> <span class="n">entry</span><span class="o">.</span><span class="n">sequence_number</span> <span class="ow">or</span> <span class="n">INITIAL_SEQUENCE_NUMBER</span><span class="p">)</span> |
| <a id="__codelineno-0-1977" name="__codelineno-0-1977"></a> |
| <a id="__codelineno-0-1978" name="__codelineno-0-1978"></a> <span class="n">executor</span> <span class="o">=</span> <span class="n">ExecutorFactory</span><span class="o">.</span><span class="n">get_or_create</span><span class="p">()</span> |
| <a id="__codelineno-0-1979" name="__codelineno-0-1979"></a> <span class="k">for</span> <span class="n">manifest_entry</span> <span class="ow">in</span> <span class="n">chain</span><span class="p">(</span> |
| <a id="__codelineno-0-1980" name="__codelineno-0-1980"></a> <span class="o">*</span><span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span> |
| <a id="__codelineno-0-1981" name="__codelineno-0-1981"></a> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">_open_manifest</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">),</span> |
| <a id="__codelineno-0-1982" name="__codelineno-0-1982"></a> <span class="p">[</span> |
| <a id="__codelineno-0-1983" name="__codelineno-0-1983"></a> <span class="p">(</span> |
| <a id="__codelineno-0-1984" name="__codelineno-0-1984"></a> <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-1985" name="__codelineno-0-1985"></a> <span class="n">manifest</span><span class="p">,</span> |
| <a id="__codelineno-0-1986" name="__codelineno-0-1986"></a> <span class="n">partition_evaluators</span><span class="p">[</span><span class="n">manifest</span><span class="o">.</span><span class="n">partition_spec_id</span><span class="p">],</span> |
| <a id="__codelineno-0-1987" name="__codelineno-0-1987"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_build_metrics_evaluator</span><span class="p">(),</span> |
| <a id="__codelineno-0-1988" name="__codelineno-0-1988"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1989" name="__codelineno-0-1989"></a> <span class="k">for</span> <span class="n">manifest</span> <span class="ow">in</span> <span class="n">manifests</span> |
| <a id="__codelineno-0-1990" name="__codelineno-0-1990"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_sequence_number</span><span class="p">(</span><span class="n">min_sequence_number</span><span class="p">,</span> <span class="n">manifest</span><span class="p">)</span> |
| <a id="__codelineno-0-1991" name="__codelineno-0-1991"></a> <span class="p">],</span> |
| <a id="__codelineno-0-1992" name="__codelineno-0-1992"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1993" name="__codelineno-0-1993"></a> <span class="p">):</span> |
| <a id="__codelineno-0-1994" name="__codelineno-0-1994"></a> <span class="n">data_file</span> <span class="o">=</span> <span class="n">manifest_entry</span><span class="o">.</span><span class="n">data_file</span> |
| <a id="__codelineno-0-1995" name="__codelineno-0-1995"></a> <span class="k">if</span> <span class="n">data_file</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">DataFileContent</span><span class="o">.</span><span class="n">DATA</span><span class="p">:</span> |
| <a id="__codelineno-0-1996" name="__codelineno-0-1996"></a> <span class="n">data_entries</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">manifest_entry</span><span class="p">)</span> |
| <a id="__codelineno-0-1997" name="__codelineno-0-1997"></a> <span class="k">elif</span> <span class="n">data_file</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">DataFileContent</span><span class="o">.</span><span class="n">POSITION_DELETES</span><span class="p">:</span> |
| <a id="__codelineno-0-1998" name="__codelineno-0-1998"></a> <span class="n">positional_delete_entries</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">manifest_entry</span><span class="p">)</span> |
| <a id="__codelineno-0-1999" name="__codelineno-0-1999"></a> <span class="k">elif</span> <span class="n">data_file</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">DataFileContent</span><span class="o">.</span><span class="n">EQUALITY_DELETES</span><span class="p">:</span> |
| <a id="__codelineno-0-2000" name="__codelineno-0-2000"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"PyIceberg does not yet support equality deletes: https://github.com/apache/iceberg/issues/6568"</span><span class="p">)</span> |
| <a id="__codelineno-0-2001" name="__codelineno-0-2001"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-2002" name="__codelineno-0-2002"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unknown DataFileContent (</span><span class="si">{</span><span class="n">data_file</span><span class="o">.</span><span class="n">content</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">manifest_entry</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-2003" name="__codelineno-0-2003"></a> |
| <a id="__codelineno-0-2004" name="__codelineno-0-2004"></a> <span class="k">return</span> <span class="p">[</span> |
| <a id="__codelineno-0-2005" name="__codelineno-0-2005"></a> <span class="n">FileScanTask</span><span class="p">(</span> |
| <a id="__codelineno-0-2006" name="__codelineno-0-2006"></a> <span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="p">,</span> |
| <a id="__codelineno-0-2007" name="__codelineno-0-2007"></a> <span class="n">delete_files</span><span class="o">=</span><span class="n">_match_deletes_to_data_file</span><span class="p">(</span> |
| <a id="__codelineno-0-2008" name="__codelineno-0-2008"></a> <span class="n">data_entry</span><span class="p">,</span> |
| <a id="__codelineno-0-2009" name="__codelineno-0-2009"></a> <span class="n">positional_delete_entries</span><span class="p">,</span> |
| <a id="__codelineno-0-2010" name="__codelineno-0-2010"></a> <span class="p">),</span> |
| <a id="__codelineno-0-2011" name="__codelineno-0-2011"></a> <span class="n">residual</span><span class="o">=</span><span class="n">residual_evaluators</span><span class="p">[</span><span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="o">.</span><span class="n">spec_id</span><span class="p">](</span><span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="p">)</span><span class="o">.</span><span class="n">residual_for</span><span class="p">(</span> |
| <a id="__codelineno-0-2012" name="__codelineno-0-2012"></a> <span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="o">.</span><span class="n">partition</span> |
| <a id="__codelineno-0-2013" name="__codelineno-0-2013"></a> <span class="p">),</span> |
| <a id="__codelineno-0-2014" name="__codelineno-0-2014"></a> <span class="p">)</span> |
| <a id="__codelineno-0-2015" name="__codelineno-0-2015"></a> <span class="k">for</span> <span class="n">data_entry</span> <span class="ow">in</span> <span class="n">data_entries</span> |
| <a id="__codelineno-0-2016" name="__codelineno-0-2016"></a> <span class="p">]</span> |
| <a id="__codelineno-0-2017" name="__codelineno-0-2017"></a> |
| <a id="__codelineno-0-2018" name="__codelineno-0-2018"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_arrow</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-2019" name="__codelineno-0-2019"></a><span class="w"> </span><span class="sd">"""Read an Arrow table eagerly from this DataScan.</span> |
| <a id="__codelineno-0-2020" name="__codelineno-0-2020"></a> |
| <a id="__codelineno-0-2021" name="__codelineno-0-2021"></a><span class="sd"> All rows will be loaded into memory at once.</span> |
| <a id="__codelineno-0-2022" name="__codelineno-0-2022"></a> |
| <a id="__codelineno-0-2023" name="__codelineno-0-2023"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2024" name="__codelineno-0-2024"></a><span class="sd"> pa.Table: Materialized Arrow Table from the Iceberg table's DataScan</span> |
| <a id="__codelineno-0-2025" name="__codelineno-0-2025"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2026" name="__codelineno-0-2026"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">ArrowScan</span> |
| <a id="__codelineno-0-2027" name="__codelineno-0-2027"></a> |
| <a id="__codelineno-0-2028" name="__codelineno-0-2028"></a> <span class="k">return</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-2029" name="__codelineno-0-2029"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit</span> |
| <a id="__codelineno-0-2030" name="__codelineno-0-2030"></a> <span class="p">)</span><span class="o">.</span><span class="n">to_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plan_files</span><span class="p">())</span> |
| <a id="__codelineno-0-2031" name="__codelineno-0-2031"></a> |
| <a id="__codelineno-0-2032" name="__codelineno-0-2032"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_arrow_batch_reader</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchReader</span><span class="p">:</span> |
| <a id="__codelineno-0-2033" name="__codelineno-0-2033"></a><span class="w"> </span><span class="sd">"""Return an Arrow RecordBatchReader from this DataScan.</span> |
| <a id="__codelineno-0-2034" name="__codelineno-0-2034"></a> |
| <a id="__codelineno-0-2035" name="__codelineno-0-2035"></a><span class="sd"> For large results, using a RecordBatchReader requires less memory than</span> |
| <a id="__codelineno-0-2036" name="__codelineno-0-2036"></a><span class="sd"> loading an Arrow Table for the same DataScan, because a RecordBatch</span> |
| <a id="__codelineno-0-2037" name="__codelineno-0-2037"></a><span class="sd"> is read one at a time.</span> |
| <a id="__codelineno-0-2038" name="__codelineno-0-2038"></a> |
| <a id="__codelineno-0-2039" name="__codelineno-0-2039"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2040" name="__codelineno-0-2040"></a><span class="sd"> pa.RecordBatchReader: Arrow RecordBatchReader from the Iceberg table's DataScan</span> |
| <a id="__codelineno-0-2041" name="__codelineno-0-2041"></a><span class="sd"> which can be used to read a stream of record batches one by one.</span> |
| <a id="__codelineno-0-2042" name="__codelineno-0-2042"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2043" name="__codelineno-0-2043"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-2044" name="__codelineno-0-2044"></a> |
| <a id="__codelineno-0-2045" name="__codelineno-0-2045"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">ArrowScan</span><span class="p">,</span> <span class="n">schema_to_pyarrow</span> |
| <a id="__codelineno-0-2046" name="__codelineno-0-2046"></a> |
| <a id="__codelineno-0-2047" name="__codelineno-0-2047"></a> <span class="n">target_schema</span> <span class="o">=</span> <span class="n">schema_to_pyarrow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">())</span> |
| <a id="__codelineno-0-2048" name="__codelineno-0-2048"></a> <span class="n">batches</span> <span class="o">=</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-2049" name="__codelineno-0-2049"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit</span> |
| <a id="__codelineno-0-2050" name="__codelineno-0-2050"></a> <span class="p">)</span><span class="o">.</span><span class="n">to_record_batches</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plan_files</span><span class="p">())</span> |
| <a id="__codelineno-0-2051" name="__codelineno-0-2051"></a> |
| <a id="__codelineno-0-2052" name="__codelineno-0-2052"></a> <span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchReader</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span> |
| <a id="__codelineno-0-2053" name="__codelineno-0-2053"></a> <span class="n">target_schema</span><span class="p">,</span> |
| <a id="__codelineno-0-2054" name="__codelineno-0-2054"></a> <span class="n">batches</span><span class="p">,</span> |
| <a id="__codelineno-0-2055" name="__codelineno-0-2055"></a> <span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">target_schema</span><span class="p">)</span> |
| <a id="__codelineno-0-2056" name="__codelineno-0-2056"></a> |
| <a id="__codelineno-0-2057" name="__codelineno-0-2057"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_pandas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-2058" name="__codelineno-0-2058"></a><span class="w"> </span><span class="sd">"""Read a Pandas DataFrame eagerly from this Iceberg table.</span> |
| <a id="__codelineno-0-2059" name="__codelineno-0-2059"></a> |
| <a id="__codelineno-0-2060" name="__codelineno-0-2060"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2061" name="__codelineno-0-2061"></a><span class="sd"> pd.DataFrame: Materialized Pandas Dataframe from the Iceberg table</span> |
| <a id="__codelineno-0-2062" name="__codelineno-0-2062"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2063" name="__codelineno-0-2063"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">()</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> |
| <a id="__codelineno-0-2064" name="__codelineno-0-2064"></a> |
| <a id="__codelineno-0-2065" name="__codelineno-0-2065"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_duckdb</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">connection</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DuckDBPyConnection</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">DuckDBPyConnection</span><span class="p">:</span> |
| <a id="__codelineno-0-2066" name="__codelineno-0-2066"></a><span class="w"> </span><span class="sd">"""Shorthand for loading the Iceberg Table in DuckDB.</span> |
| <a id="__codelineno-0-2067" name="__codelineno-0-2067"></a> |
| <a id="__codelineno-0-2068" name="__codelineno-0-2068"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2069" name="__codelineno-0-2069"></a><span class="sd"> DuckDBPyConnection: In memory DuckDB connection with the Iceberg table.</span> |
| <a id="__codelineno-0-2070" name="__codelineno-0-2070"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2071" name="__codelineno-0-2071"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">duckdb</span> |
| <a id="__codelineno-0-2072" name="__codelineno-0-2072"></a> |
| <a id="__codelineno-0-2073" name="__codelineno-0-2073"></a> <span class="n">con</span> <span class="o">=</span> <span class="n">connection</span> <span class="ow">or</span> <span class="n">duckdb</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">database</span><span class="o">=</span><span class="s2">":memory:"</span><span class="p">)</span> |
| <a id="__codelineno-0-2074" name="__codelineno-0-2074"></a> <span class="n">con</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">table_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span> |
| <a id="__codelineno-0-2075" name="__codelineno-0-2075"></a> |
| <a id="__codelineno-0-2076" name="__codelineno-0-2076"></a> <span class="k">return</span> <span class="n">con</span> |
| <a id="__codelineno-0-2077" name="__codelineno-0-2077"></a> |
| <a id="__codelineno-0-2078" name="__codelineno-0-2078"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_ray</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ray</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">Dataset</span><span class="p">:</span> |
| <a id="__codelineno-0-2079" name="__codelineno-0-2079"></a><span class="w"> </span><span class="sd">"""Read a Ray Dataset eagerly from this Iceberg table.</span> |
| <a id="__codelineno-0-2080" name="__codelineno-0-2080"></a> |
| <a id="__codelineno-0-2081" name="__codelineno-0-2081"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2082" name="__codelineno-0-2082"></a><span class="sd"> ray.data.dataset.Dataset: Materialized Ray Dataset from the Iceberg table</span> |
| <a id="__codelineno-0-2083" name="__codelineno-0-2083"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2084" name="__codelineno-0-2084"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">ray</span> |
| <a id="__codelineno-0-2085" name="__codelineno-0-2085"></a> |
| <a id="__codelineno-0-2086" name="__codelineno-0-2086"></a> <span class="k">return</span> <span class="n">ray</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">from_arrow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span> |
| <a id="__codelineno-0-2087" name="__codelineno-0-2087"></a> |
| <a id="__codelineno-0-2088" name="__codelineno-0-2088"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_polars</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pl</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-2089" name="__codelineno-0-2089"></a><span class="w"> </span><span class="sd">"""Read a Polars DataFrame from this Iceberg table.</span> |
| <a id="__codelineno-0-2090" name="__codelineno-0-2090"></a> |
| <a id="__codelineno-0-2091" name="__codelineno-0-2091"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2092" name="__codelineno-0-2092"></a><span class="sd"> pl.DataFrame: Materialized Polars Dataframe from the Iceberg table</span> |
| <a id="__codelineno-0-2093" name="__codelineno-0-2093"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2094" name="__codelineno-0-2094"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">polars</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pl</span> |
| <a id="__codelineno-0-2095" name="__codelineno-0-2095"></a> |
| <a id="__codelineno-0-2096" name="__codelineno-0-2096"></a> <span class="n">result</span> <span class="o">=</span> <span class="n">pl</span><span class="o">.</span><span class="n">from_arrow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span> |
| <a id="__codelineno-0-2097" name="__codelineno-0-2097"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">pl</span><span class="o">.</span><span class="n">Series</span><span class="p">):</span> |
| <a id="__codelineno-0-2098" name="__codelineno-0-2098"></a> <span class="n">result</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">to_frame</span><span class="p">()</span> |
| <a id="__codelineno-0-2099" name="__codelineno-0-2099"></a> |
| <a id="__codelineno-0-2100" name="__codelineno-0-2100"></a> <span class="k">return</span> <span class="n">result</span> |
| <a id="__codelineno-0-2101" name="__codelineno-0-2101"></a> |
| <a id="__codelineno-0-2102" name="__codelineno-0-2102"></a> <span class="k">def</span><span class="w"> </span><span class="nf">count</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> |
| <a id="__codelineno-0-2103" name="__codelineno-0-2103"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">ArrowScan</span> |
| <a id="__codelineno-0-2104" name="__codelineno-0-2104"></a> |
| <a id="__codelineno-0-2105" name="__codelineno-0-2105"></a> <span class="c1"># Usage: Calculates the total number of records in a Scan that haven't had positional deletes.</span> |
| <a id="__codelineno-0-2106" name="__codelineno-0-2106"></a> <span class="n">res</span> <span class="o">=</span> <span class="mi">0</span> |
| <a id="__codelineno-0-2107" name="__codelineno-0-2107"></a> <span class="c1"># every task is a FileScanTask</span> |
| <a id="__codelineno-0-2108" name="__codelineno-0-2108"></a> <span class="n">tasks</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">plan_files</span><span class="p">()</span> |
| <a id="__codelineno-0-2109" name="__codelineno-0-2109"></a> |
| <a id="__codelineno-0-2110" name="__codelineno-0-2110"></a> <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">tasks</span><span class="p">:</span> |
| <a id="__codelineno-0-2111" name="__codelineno-0-2111"></a> <span class="c1"># task.residual is a Boolean Expression if the filter condition is fully satisfied by the</span> |
| <a id="__codelineno-0-2112" name="__codelineno-0-2112"></a> <span class="c1"># partition value and task.delete_files represents that positional delete haven't been merged yet</span> |
| <a id="__codelineno-0-2113" name="__codelineno-0-2113"></a> <span class="c1"># hence those files have to read as a pyarrow table applying the filter and deletes</span> |
| <a id="__codelineno-0-2114" name="__codelineno-0-2114"></a> <span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">residual</span> <span class="o">==</span> <span class="n">AlwaysTrue</span><span class="p">()</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">delete_files</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-2115" name="__codelineno-0-2115"></a> <span class="c1"># Every File has a metadata stat that stores the file record count</span> |
| <a id="__codelineno-0-2116" name="__codelineno-0-2116"></a> <span class="n">res</span> <span class="o">+=</span> <span class="n">task</span><span class="o">.</span><span class="n">file</span><span class="o">.</span><span class="n">record_count</span> |
| <a id="__codelineno-0-2117" name="__codelineno-0-2117"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-2118" name="__codelineno-0-2118"></a> <span class="n">arrow_scan</span> <span class="o">=</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-2119" name="__codelineno-0-2119"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-2120" name="__codelineno-0-2120"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-2121" name="__codelineno-0-2121"></a> <span class="n">projected_schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">(),</span> |
| <a id="__codelineno-0-2122" name="__codelineno-0-2122"></a> <span class="n">row_filter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-2123" name="__codelineno-0-2123"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-2124" name="__codelineno-0-2124"></a> <span class="p">)</span> |
| <a id="__codelineno-0-2125" name="__codelineno-0-2125"></a> <span class="n">tbl</span> <span class="o">=</span> <span class="n">arrow_scan</span><span class="o">.</span><span class="n">to_table</span><span class="p">([</span><span class="n">task</span><span class="p">])</span> |
| <a id="__codelineno-0-2126" name="__codelineno-0-2126"></a> <span class="n">res</span> <span class="o">+=</span> <span class="nb">len</span><span class="p">(</span><span class="n">tbl</span><span class="p">)</span> |
| <a id="__codelineno-0-2127" name="__codelineno-0-2127"></a> <span class="k">return</span> <span class="n">res</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.plan_files" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">plan_files</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.plan_files" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Plans the relevant files by filtering on the PartitionSpecs.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="typing.Iterable">Iterable</span>[<a class="autorefs autorefs-internal" title="FileScanTask |
| |
| |
| |
| dataclass |
| (pyiceberg.table.FileScanTask)" href="#pyiceberg.table.FileScanTask">FileScanTask</a>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>List of FileScanTasks that contain both data and delete files.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1945">1945</a></span> |
| <span class="normal"><a href="#__codelineno-0-1946">1946</a></span> |
| <span class="normal"><a href="#__codelineno-0-1947">1947</a></span> |
| <span class="normal"><a href="#__codelineno-0-1948">1948</a></span> |
| <span class="normal"><a href="#__codelineno-0-1949">1949</a></span> |
| <span class="normal"><a href="#__codelineno-0-1950">1950</a></span> |
| <span class="normal"><a href="#__codelineno-0-1951">1951</a></span> |
| <span class="normal"><a href="#__codelineno-0-1952">1952</a></span> |
| <span class="normal"><a href="#__codelineno-0-1953">1953</a></span> |
| <span class="normal"><a href="#__codelineno-0-1954">1954</a></span> |
| <span class="normal"><a href="#__codelineno-0-1955">1955</a></span> |
| <span class="normal"><a href="#__codelineno-0-1956">1956</a></span> |
| <span class="normal"><a href="#__codelineno-0-1957">1957</a></span> |
| <span class="normal"><a href="#__codelineno-0-1958">1958</a></span> |
| <span class="normal"><a href="#__codelineno-0-1959">1959</a></span> |
| <span class="normal"><a href="#__codelineno-0-1960">1960</a></span> |
| <span class="normal"><a href="#__codelineno-0-1961">1961</a></span> |
| <span class="normal"><a href="#__codelineno-0-1962">1962</a></span> |
| <span class="normal"><a href="#__codelineno-0-1963">1963</a></span> |
| <span class="normal"><a href="#__codelineno-0-1964">1964</a></span> |
| <span class="normal"><a href="#__codelineno-0-1965">1965</a></span> |
| <span class="normal"><a href="#__codelineno-0-1966">1966</a></span> |
| <span class="normal"><a href="#__codelineno-0-1967">1967</a></span> |
| <span class="normal"><a href="#__codelineno-0-1968">1968</a></span> |
| <span class="normal"><a href="#__codelineno-0-1969">1969</a></span> |
| <span class="normal"><a href="#__codelineno-0-1970">1970</a></span> |
| <span class="normal"><a href="#__codelineno-0-1971">1971</a></span> |
| <span class="normal"><a href="#__codelineno-0-1972">1972</a></span> |
| <span class="normal"><a href="#__codelineno-0-1973">1973</a></span> |
| <span class="normal"><a href="#__codelineno-0-1974">1974</a></span> |
| <span class="normal"><a href="#__codelineno-0-1975">1975</a></span> |
| <span class="normal"><a href="#__codelineno-0-1976">1976</a></span> |
| <span class="normal"><a href="#__codelineno-0-1977">1977</a></span> |
| <span class="normal"><a href="#__codelineno-0-1978">1978</a></span> |
| <span class="normal"><a href="#__codelineno-0-1979">1979</a></span> |
| <span class="normal"><a href="#__codelineno-0-1980">1980</a></span> |
| <span class="normal"><a href="#__codelineno-0-1981">1981</a></span> |
| <span class="normal"><a href="#__codelineno-0-1982">1982</a></span> |
| <span class="normal"><a href="#__codelineno-0-1983">1983</a></span> |
| <span class="normal"><a href="#__codelineno-0-1984">1984</a></span> |
| <span class="normal"><a href="#__codelineno-0-1985">1985</a></span> |
| <span class="normal"><a href="#__codelineno-0-1986">1986</a></span> |
| <span class="normal"><a href="#__codelineno-0-1987">1987</a></span> |
| <span class="normal"><a href="#__codelineno-0-1988">1988</a></span> |
| <span class="normal"><a href="#__codelineno-0-1989">1989</a></span> |
| <span class="normal"><a href="#__codelineno-0-1990">1990</a></span> |
| <span class="normal"><a href="#__codelineno-0-1991">1991</a></span> |
| <span class="normal"><a href="#__codelineno-0-1992">1992</a></span> |
| <span class="normal"><a href="#__codelineno-0-1993">1993</a></span> |
| <span class="normal"><a href="#__codelineno-0-1994">1994</a></span> |
| <span class="normal"><a href="#__codelineno-0-1995">1995</a></span> |
| <span class="normal"><a href="#__codelineno-0-1996">1996</a></span> |
| <span class="normal"><a href="#__codelineno-0-1997">1997</a></span> |
| <span class="normal"><a href="#__codelineno-0-1998">1998</a></span> |
| <span class="normal"><a href="#__codelineno-0-1999">1999</a></span> |
| <span class="normal"><a href="#__codelineno-0-2000">2000</a></span> |
| <span class="normal"><a href="#__codelineno-0-2001">2001</a></span> |
| <span class="normal"><a href="#__codelineno-0-2002">2002</a></span> |
| <span class="normal"><a href="#__codelineno-0-2003">2003</a></span> |
| <span class="normal"><a href="#__codelineno-0-2004">2004</a></span> |
| <span class="normal"><a href="#__codelineno-0-2005">2005</a></span> |
| <span class="normal"><a href="#__codelineno-0-2006">2006</a></span> |
| <span class="normal"><a href="#__codelineno-0-2007">2007</a></span> |
| <span class="normal"><a href="#__codelineno-0-2008">2008</a></span> |
| <span class="normal"><a href="#__codelineno-0-2009">2009</a></span> |
| <span class="normal"><a href="#__codelineno-0-2010">2010</a></span> |
| <span class="normal"><a href="#__codelineno-0-2011">2011</a></span> |
| <span class="normal"><a href="#__codelineno-0-2012">2012</a></span> |
| <span class="normal"><a href="#__codelineno-0-2013">2013</a></span> |
| <span class="normal"><a href="#__codelineno-0-2014">2014</a></span> |
| <span class="normal"><a href="#__codelineno-0-2015">2015</a></span> |
| <span class="normal"><a href="#__codelineno-0-2016">2016</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1945" name="__codelineno-0-1945"></a><span class="k">def</span><span class="w"> </span><span class="nf">plan_files</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">FileScanTask</span><span class="p">]:</span> |
| <a id="__codelineno-0-1946" name="__codelineno-0-1946"></a><span class="w"> </span><span class="sd">"""Plans the relevant files by filtering on the PartitionSpecs.</span> |
| <a id="__codelineno-0-1947" name="__codelineno-0-1947"></a> |
| <a id="__codelineno-0-1948" name="__codelineno-0-1948"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1949" name="__codelineno-0-1949"></a><span class="sd"> List of FileScanTasks that contain both data and delete files.</span> |
| <a id="__codelineno-0-1950" name="__codelineno-0-1950"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1951" name="__codelineno-0-1951"></a> <span class="n">snapshot</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot</span><span class="p">()</span> |
| <a id="__codelineno-0-1952" name="__codelineno-0-1952"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">snapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-1953" name="__codelineno-0-1953"></a> <span class="k">return</span> <span class="nb">iter</span><span class="p">([])</span> |
| <a id="__codelineno-0-1954" name="__codelineno-0-1954"></a> |
| <a id="__codelineno-0-1955" name="__codelineno-0-1955"></a> <span class="c1"># step 1: filter manifests using partition summaries</span> |
| <a id="__codelineno-0-1956" name="__codelineno-0-1956"></a> <span class="c1"># the filter depends on the partition spec used to write the manifest file, so create a cache of filters for each spec id</span> |
| <a id="__codelineno-0-1957" name="__codelineno-0-1957"></a> |
| <a id="__codelineno-0-1958" name="__codelineno-0-1958"></a> <span class="n">manifest_evaluators</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">ManifestFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]]</span> <span class="o">=</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_manifest_evaluator</span><span class="p">)</span> |
| <a id="__codelineno-0-1959" name="__codelineno-0-1959"></a> |
| <a id="__codelineno-0-1960" name="__codelineno-0-1960"></a> <span class="n">residual_evaluators</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="n">ResidualEvaluator</span><span class="p">]]</span> <span class="o">=</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_residual_evaluator</span><span class="p">)</span> |
| <a id="__codelineno-0-1961" name="__codelineno-0-1961"></a> |
| <a id="__codelineno-0-1962" name="__codelineno-0-1962"></a> <span class="n">manifests</span> <span class="o">=</span> <span class="p">[</span> |
| <a id="__codelineno-0-1963" name="__codelineno-0-1963"></a> <span class="n">manifest_file</span> |
| <a id="__codelineno-0-1964" name="__codelineno-0-1964"></a> <span class="k">for</span> <span class="n">manifest_file</span> <span class="ow">in</span> <span class="n">snapshot</span><span class="o">.</span><span class="n">manifests</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">)</span> |
| <a id="__codelineno-0-1965" name="__codelineno-0-1965"></a> <span class="k">if</span> <span class="n">manifest_evaluators</span><span class="p">[</span><span class="n">manifest_file</span><span class="o">.</span><span class="n">partition_spec_id</span><span class="p">](</span><span class="n">manifest_file</span><span class="p">)</span> |
| <a id="__codelineno-0-1966" name="__codelineno-0-1966"></a> <span class="p">]</span> |
| <a id="__codelineno-0-1967" name="__codelineno-0-1967"></a> |
| <a id="__codelineno-0-1968" name="__codelineno-0-1968"></a> <span class="c1"># step 2: filter the data files in each manifest</span> |
| <a id="__codelineno-0-1969" name="__codelineno-0-1969"></a> <span class="c1"># this filter depends on the partition spec used to write the manifest file</span> |
| <a id="__codelineno-0-1970" name="__codelineno-0-1970"></a> |
| <a id="__codelineno-0-1971" name="__codelineno-0-1971"></a> <span class="n">partition_evaluators</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">DataFile</span><span class="p">],</span> <span class="nb">bool</span><span class="p">]]</span> <span class="o">=</span> <span class="n">KeyDefaultDict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_build_partition_evaluator</span><span class="p">)</span> |
| <a id="__codelineno-0-1972" name="__codelineno-0-1972"></a> |
| <a id="__codelineno-0-1973" name="__codelineno-0-1973"></a> <span class="n">min_sequence_number</span> <span class="o">=</span> <span class="n">_min_sequence_number</span><span class="p">(</span><span class="n">manifests</span><span class="p">)</span> |
| <a id="__codelineno-0-1974" name="__codelineno-0-1974"></a> |
| <a id="__codelineno-0-1975" name="__codelineno-0-1975"></a> <span class="n">data_entries</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">ManifestEntry</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-1976" name="__codelineno-0-1976"></a> <span class="n">positional_delete_entries</span> <span class="o">=</span> <span class="n">SortedList</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">entry</span><span class="p">:</span> <span class="n">entry</span><span class="o">.</span><span class="n">sequence_number</span> <span class="ow">or</span> <span class="n">INITIAL_SEQUENCE_NUMBER</span><span class="p">)</span> |
| <a id="__codelineno-0-1977" name="__codelineno-0-1977"></a> |
| <a id="__codelineno-0-1978" name="__codelineno-0-1978"></a> <span class="n">executor</span> <span class="o">=</span> <span class="n">ExecutorFactory</span><span class="o">.</span><span class="n">get_or_create</span><span class="p">()</span> |
| <a id="__codelineno-0-1979" name="__codelineno-0-1979"></a> <span class="k">for</span> <span class="n">manifest_entry</span> <span class="ow">in</span> <span class="n">chain</span><span class="p">(</span> |
| <a id="__codelineno-0-1980" name="__codelineno-0-1980"></a> <span class="o">*</span><span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span> |
| <a id="__codelineno-0-1981" name="__codelineno-0-1981"></a> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">_open_manifest</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">),</span> |
| <a id="__codelineno-0-1982" name="__codelineno-0-1982"></a> <span class="p">[</span> |
| <a id="__codelineno-0-1983" name="__codelineno-0-1983"></a> <span class="p">(</span> |
| <a id="__codelineno-0-1984" name="__codelineno-0-1984"></a> <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-1985" name="__codelineno-0-1985"></a> <span class="n">manifest</span><span class="p">,</span> |
| <a id="__codelineno-0-1986" name="__codelineno-0-1986"></a> <span class="n">partition_evaluators</span><span class="p">[</span><span class="n">manifest</span><span class="o">.</span><span class="n">partition_spec_id</span><span class="p">],</span> |
| <a id="__codelineno-0-1987" name="__codelineno-0-1987"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_build_metrics_evaluator</span><span class="p">(),</span> |
| <a id="__codelineno-0-1988" name="__codelineno-0-1988"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1989" name="__codelineno-0-1989"></a> <span class="k">for</span> <span class="n">manifest</span> <span class="ow">in</span> <span class="n">manifests</span> |
| <a id="__codelineno-0-1990" name="__codelineno-0-1990"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_sequence_number</span><span class="p">(</span><span class="n">min_sequence_number</span><span class="p">,</span> <span class="n">manifest</span><span class="p">)</span> |
| <a id="__codelineno-0-1991" name="__codelineno-0-1991"></a> <span class="p">],</span> |
| <a id="__codelineno-0-1992" name="__codelineno-0-1992"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1993" name="__codelineno-0-1993"></a> <span class="p">):</span> |
| <a id="__codelineno-0-1994" name="__codelineno-0-1994"></a> <span class="n">data_file</span> <span class="o">=</span> <span class="n">manifest_entry</span><span class="o">.</span><span class="n">data_file</span> |
| <a id="__codelineno-0-1995" name="__codelineno-0-1995"></a> <span class="k">if</span> <span class="n">data_file</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">DataFileContent</span><span class="o">.</span><span class="n">DATA</span><span class="p">:</span> |
| <a id="__codelineno-0-1996" name="__codelineno-0-1996"></a> <span class="n">data_entries</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">manifest_entry</span><span class="p">)</span> |
| <a id="__codelineno-0-1997" name="__codelineno-0-1997"></a> <span class="k">elif</span> <span class="n">data_file</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">DataFileContent</span><span class="o">.</span><span class="n">POSITION_DELETES</span><span class="p">:</span> |
| <a id="__codelineno-0-1998" name="__codelineno-0-1998"></a> <span class="n">positional_delete_entries</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">manifest_entry</span><span class="p">)</span> |
| <a id="__codelineno-0-1999" name="__codelineno-0-1999"></a> <span class="k">elif</span> <span class="n">data_file</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">DataFileContent</span><span class="o">.</span><span class="n">EQUALITY_DELETES</span><span class="p">:</span> |
| <a id="__codelineno-0-2000" name="__codelineno-0-2000"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"PyIceberg does not yet support equality deletes: https://github.com/apache/iceberg/issues/6568"</span><span class="p">)</span> |
| <a id="__codelineno-0-2001" name="__codelineno-0-2001"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-2002" name="__codelineno-0-2002"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unknown DataFileContent (</span><span class="si">{</span><span class="n">data_file</span><span class="o">.</span><span class="n">content</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">manifest_entry</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-2003" name="__codelineno-0-2003"></a> |
| <a id="__codelineno-0-2004" name="__codelineno-0-2004"></a> <span class="k">return</span> <span class="p">[</span> |
| <a id="__codelineno-0-2005" name="__codelineno-0-2005"></a> <span class="n">FileScanTask</span><span class="p">(</span> |
| <a id="__codelineno-0-2006" name="__codelineno-0-2006"></a> <span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="p">,</span> |
| <a id="__codelineno-0-2007" name="__codelineno-0-2007"></a> <span class="n">delete_files</span><span class="o">=</span><span class="n">_match_deletes_to_data_file</span><span class="p">(</span> |
| <a id="__codelineno-0-2008" name="__codelineno-0-2008"></a> <span class="n">data_entry</span><span class="p">,</span> |
| <a id="__codelineno-0-2009" name="__codelineno-0-2009"></a> <span class="n">positional_delete_entries</span><span class="p">,</span> |
| <a id="__codelineno-0-2010" name="__codelineno-0-2010"></a> <span class="p">),</span> |
| <a id="__codelineno-0-2011" name="__codelineno-0-2011"></a> <span class="n">residual</span><span class="o">=</span><span class="n">residual_evaluators</span><span class="p">[</span><span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="o">.</span><span class="n">spec_id</span><span class="p">](</span><span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="p">)</span><span class="o">.</span><span class="n">residual_for</span><span class="p">(</span> |
| <a id="__codelineno-0-2012" name="__codelineno-0-2012"></a> <span class="n">data_entry</span><span class="o">.</span><span class="n">data_file</span><span class="o">.</span><span class="n">partition</span> |
| <a id="__codelineno-0-2013" name="__codelineno-0-2013"></a> <span class="p">),</span> |
| <a id="__codelineno-0-2014" name="__codelineno-0-2014"></a> <span class="p">)</span> |
| <a id="__codelineno-0-2015" name="__codelineno-0-2015"></a> <span class="k">for</span> <span class="n">data_entry</span> <span class="ow">in</span> <span class="n">data_entries</span> |
| <a id="__codelineno-0-2016" name="__codelineno-0-2016"></a> <span class="p">]</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.to_arrow" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_arrow</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.to_arrow" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Read an Arrow table eagerly from this DataScan.</p> |
| <p>All rows will be loaded into memory at once.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="pyarrow.Table">Table</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>pa.Table: Materialized Arrow Table from the Iceberg table's DataScan</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2018">2018</a></span> |
| <span class="normal"><a href="#__codelineno-0-2019">2019</a></span> |
| <span class="normal"><a href="#__codelineno-0-2020">2020</a></span> |
| <span class="normal"><a href="#__codelineno-0-2021">2021</a></span> |
| <span class="normal"><a href="#__codelineno-0-2022">2022</a></span> |
| <span class="normal"><a href="#__codelineno-0-2023">2023</a></span> |
| <span class="normal"><a href="#__codelineno-0-2024">2024</a></span> |
| <span class="normal"><a href="#__codelineno-0-2025">2025</a></span> |
| <span class="normal"><a href="#__codelineno-0-2026">2026</a></span> |
| <span class="normal"><a href="#__codelineno-0-2027">2027</a></span> |
| <span class="normal"><a href="#__codelineno-0-2028">2028</a></span> |
| <span class="normal"><a href="#__codelineno-0-2029">2029</a></span> |
| <span class="normal"><a href="#__codelineno-0-2030">2030</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2018" name="__codelineno-0-2018"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_arrow</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-2019" name="__codelineno-0-2019"></a><span class="w"> </span><span class="sd">"""Read an Arrow table eagerly from this DataScan.</span> |
| <a id="__codelineno-0-2020" name="__codelineno-0-2020"></a> |
| <a id="__codelineno-0-2021" name="__codelineno-0-2021"></a><span class="sd"> All rows will be loaded into memory at once.</span> |
| <a id="__codelineno-0-2022" name="__codelineno-0-2022"></a> |
| <a id="__codelineno-0-2023" name="__codelineno-0-2023"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2024" name="__codelineno-0-2024"></a><span class="sd"> pa.Table: Materialized Arrow Table from the Iceberg table's DataScan</span> |
| <a id="__codelineno-0-2025" name="__codelineno-0-2025"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2026" name="__codelineno-0-2026"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">ArrowScan</span> |
| <a id="__codelineno-0-2027" name="__codelineno-0-2027"></a> |
| <a id="__codelineno-0-2028" name="__codelineno-0-2028"></a> <span class="k">return</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-2029" name="__codelineno-0-2029"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit</span> |
| <a id="__codelineno-0-2030" name="__codelineno-0-2030"></a> <span class="p">)</span><span class="o">.</span><span class="n">to_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plan_files</span><span class="p">())</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.to_arrow_batch_reader" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_arrow_batch_reader</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.to_arrow_batch_reader" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return an Arrow RecordBatchReader from this DataScan.</p> |
| <p>For large results, using a RecordBatchReader requires less memory than |
| loading an Arrow Table for the same DataScan, because a RecordBatch |
| is read one at a time.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="pyarrow.RecordBatchReader">RecordBatchReader</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>pa.RecordBatchReader: Arrow RecordBatchReader from the Iceberg table's DataScan |
| which can be used to read a stream of record batches one by one.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2032">2032</a></span> |
| <span class="normal"><a href="#__codelineno-0-2033">2033</a></span> |
| <span class="normal"><a href="#__codelineno-0-2034">2034</a></span> |
| <span class="normal"><a href="#__codelineno-0-2035">2035</a></span> |
| <span class="normal"><a href="#__codelineno-0-2036">2036</a></span> |
| <span class="normal"><a href="#__codelineno-0-2037">2037</a></span> |
| <span class="normal"><a href="#__codelineno-0-2038">2038</a></span> |
| <span class="normal"><a href="#__codelineno-0-2039">2039</a></span> |
| <span class="normal"><a href="#__codelineno-0-2040">2040</a></span> |
| <span class="normal"><a href="#__codelineno-0-2041">2041</a></span> |
| <span class="normal"><a href="#__codelineno-0-2042">2042</a></span> |
| <span class="normal"><a href="#__codelineno-0-2043">2043</a></span> |
| <span class="normal"><a href="#__codelineno-0-2044">2044</a></span> |
| <span class="normal"><a href="#__codelineno-0-2045">2045</a></span> |
| <span class="normal"><a href="#__codelineno-0-2046">2046</a></span> |
| <span class="normal"><a href="#__codelineno-0-2047">2047</a></span> |
| <span class="normal"><a href="#__codelineno-0-2048">2048</a></span> |
| <span class="normal"><a href="#__codelineno-0-2049">2049</a></span> |
| <span class="normal"><a href="#__codelineno-0-2050">2050</a></span> |
| <span class="normal"><a href="#__codelineno-0-2051">2051</a></span> |
| <span class="normal"><a href="#__codelineno-0-2052">2052</a></span> |
| <span class="normal"><a href="#__codelineno-0-2053">2053</a></span> |
| <span class="normal"><a href="#__codelineno-0-2054">2054</a></span> |
| <span class="normal"><a href="#__codelineno-0-2055">2055</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2032" name="__codelineno-0-2032"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_arrow_batch_reader</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchReader</span><span class="p">:</span> |
| <a id="__codelineno-0-2033" name="__codelineno-0-2033"></a><span class="w"> </span><span class="sd">"""Return an Arrow RecordBatchReader from this DataScan.</span> |
| <a id="__codelineno-0-2034" name="__codelineno-0-2034"></a> |
| <a id="__codelineno-0-2035" name="__codelineno-0-2035"></a><span class="sd"> For large results, using a RecordBatchReader requires less memory than</span> |
| <a id="__codelineno-0-2036" name="__codelineno-0-2036"></a><span class="sd"> loading an Arrow Table for the same DataScan, because a RecordBatch</span> |
| <a id="__codelineno-0-2037" name="__codelineno-0-2037"></a><span class="sd"> is read one at a time.</span> |
| <a id="__codelineno-0-2038" name="__codelineno-0-2038"></a> |
| <a id="__codelineno-0-2039" name="__codelineno-0-2039"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2040" name="__codelineno-0-2040"></a><span class="sd"> pa.RecordBatchReader: Arrow RecordBatchReader from the Iceberg table's DataScan</span> |
| <a id="__codelineno-0-2041" name="__codelineno-0-2041"></a><span class="sd"> which can be used to read a stream of record batches one by one.</span> |
| <a id="__codelineno-0-2042" name="__codelineno-0-2042"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2043" name="__codelineno-0-2043"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-2044" name="__codelineno-0-2044"></a> |
| <a id="__codelineno-0-2045" name="__codelineno-0-2045"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">ArrowScan</span><span class="p">,</span> <span class="n">schema_to_pyarrow</span> |
| <a id="__codelineno-0-2046" name="__codelineno-0-2046"></a> |
| <a id="__codelineno-0-2047" name="__codelineno-0-2047"></a> <span class="n">target_schema</span> <span class="o">=</span> <span class="n">schema_to_pyarrow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">())</span> |
| <a id="__codelineno-0-2048" name="__codelineno-0-2048"></a> <span class="n">batches</span> <span class="o">=</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-2049" name="__codelineno-0-2049"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit</span> |
| <a id="__codelineno-0-2050" name="__codelineno-0-2050"></a> <span class="p">)</span><span class="o">.</span><span class="n">to_record_batches</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plan_files</span><span class="p">())</span> |
| <a id="__codelineno-0-2051" name="__codelineno-0-2051"></a> |
| <a id="__codelineno-0-2052" name="__codelineno-0-2052"></a> <span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchReader</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span> |
| <a id="__codelineno-0-2053" name="__codelineno-0-2053"></a> <span class="n">target_schema</span><span class="p">,</span> |
| <a id="__codelineno-0-2054" name="__codelineno-0-2054"></a> <span class="n">batches</span><span class="p">,</span> |
| <a id="__codelineno-0-2055" name="__codelineno-0-2055"></a> <span class="p">)</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">target_schema</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.to_duckdb" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_duckdb</span><span class="p">(</span><span class="n">table_name</span><span class="p">,</span> <span class="n">connection</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.to_duckdb" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for loading the Iceberg Table in DuckDB.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td><code>DuckDBPyConnection</code></td> <td> |
| <code><span title="duckdb.DuckDBPyConnection">DuckDBPyConnection</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>In memory DuckDB connection with the Iceberg table.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2065">2065</a></span> |
| <span class="normal"><a href="#__codelineno-0-2066">2066</a></span> |
| <span class="normal"><a href="#__codelineno-0-2067">2067</a></span> |
| <span class="normal"><a href="#__codelineno-0-2068">2068</a></span> |
| <span class="normal"><a href="#__codelineno-0-2069">2069</a></span> |
| <span class="normal"><a href="#__codelineno-0-2070">2070</a></span> |
| <span class="normal"><a href="#__codelineno-0-2071">2071</a></span> |
| <span class="normal"><a href="#__codelineno-0-2072">2072</a></span> |
| <span class="normal"><a href="#__codelineno-0-2073">2073</a></span> |
| <span class="normal"><a href="#__codelineno-0-2074">2074</a></span> |
| <span class="normal"><a href="#__codelineno-0-2075">2075</a></span> |
| <span class="normal"><a href="#__codelineno-0-2076">2076</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2065" name="__codelineno-0-2065"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_duckdb</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">connection</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DuckDBPyConnection</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">DuckDBPyConnection</span><span class="p">:</span> |
| <a id="__codelineno-0-2066" name="__codelineno-0-2066"></a><span class="w"> </span><span class="sd">"""Shorthand for loading the Iceberg Table in DuckDB.</span> |
| <a id="__codelineno-0-2067" name="__codelineno-0-2067"></a> |
| <a id="__codelineno-0-2068" name="__codelineno-0-2068"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2069" name="__codelineno-0-2069"></a><span class="sd"> DuckDBPyConnection: In memory DuckDB connection with the Iceberg table.</span> |
| <a id="__codelineno-0-2070" name="__codelineno-0-2070"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2071" name="__codelineno-0-2071"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">duckdb</span> |
| <a id="__codelineno-0-2072" name="__codelineno-0-2072"></a> |
| <a id="__codelineno-0-2073" name="__codelineno-0-2073"></a> <span class="n">con</span> <span class="o">=</span> <span class="n">connection</span> <span class="ow">or</span> <span class="n">duckdb</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">database</span><span class="o">=</span><span class="s2">":memory:"</span><span class="p">)</span> |
| <a id="__codelineno-0-2074" name="__codelineno-0-2074"></a> <span class="n">con</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">table_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span> |
| <a id="__codelineno-0-2075" name="__codelineno-0-2075"></a> |
| <a id="__codelineno-0-2076" name="__codelineno-0-2076"></a> <span class="k">return</span> <span class="n">con</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.to_pandas" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_pandas</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.to_pandas" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Read a Pandas DataFrame eagerly from this Iceberg table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="pandas.DataFrame">DataFrame</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>pd.DataFrame: Materialized Pandas Dataframe from the Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2057">2057</a></span> |
| <span class="normal"><a href="#__codelineno-0-2058">2058</a></span> |
| <span class="normal"><a href="#__codelineno-0-2059">2059</a></span> |
| <span class="normal"><a href="#__codelineno-0-2060">2060</a></span> |
| <span class="normal"><a href="#__codelineno-0-2061">2061</a></span> |
| <span class="normal"><a href="#__codelineno-0-2062">2062</a></span> |
| <span class="normal"><a href="#__codelineno-0-2063">2063</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2057" name="__codelineno-0-2057"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_pandas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-2058" name="__codelineno-0-2058"></a><span class="w"> </span><span class="sd">"""Read a Pandas DataFrame eagerly from this Iceberg table.</span> |
| <a id="__codelineno-0-2059" name="__codelineno-0-2059"></a> |
| <a id="__codelineno-0-2060" name="__codelineno-0-2060"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2061" name="__codelineno-0-2061"></a><span class="sd"> pd.DataFrame: Materialized Pandas Dataframe from the Iceberg table</span> |
| <a id="__codelineno-0-2062" name="__codelineno-0-2062"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2063" name="__codelineno-0-2063"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">()</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.to_polars" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_polars</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.to_polars" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Read a Polars DataFrame from this Iceberg table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="polars.DataFrame">DataFrame</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>pl.DataFrame: Materialized Polars Dataframe from the Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2088">2088</a></span> |
| <span class="normal"><a href="#__codelineno-0-2089">2089</a></span> |
| <span class="normal"><a href="#__codelineno-0-2090">2090</a></span> |
| <span class="normal"><a href="#__codelineno-0-2091">2091</a></span> |
| <span class="normal"><a href="#__codelineno-0-2092">2092</a></span> |
| <span class="normal"><a href="#__codelineno-0-2093">2093</a></span> |
| <span class="normal"><a href="#__codelineno-0-2094">2094</a></span> |
| <span class="normal"><a href="#__codelineno-0-2095">2095</a></span> |
| <span class="normal"><a href="#__codelineno-0-2096">2096</a></span> |
| <span class="normal"><a href="#__codelineno-0-2097">2097</a></span> |
| <span class="normal"><a href="#__codelineno-0-2098">2098</a></span> |
| <span class="normal"><a href="#__codelineno-0-2099">2099</a></span> |
| <span class="normal"><a href="#__codelineno-0-2100">2100</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2088" name="__codelineno-0-2088"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_polars</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pl</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-2089" name="__codelineno-0-2089"></a><span class="w"> </span><span class="sd">"""Read a Polars DataFrame from this Iceberg table.</span> |
| <a id="__codelineno-0-2090" name="__codelineno-0-2090"></a> |
| <a id="__codelineno-0-2091" name="__codelineno-0-2091"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2092" name="__codelineno-0-2092"></a><span class="sd"> pl.DataFrame: Materialized Polars Dataframe from the Iceberg table</span> |
| <a id="__codelineno-0-2093" name="__codelineno-0-2093"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2094" name="__codelineno-0-2094"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">polars</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pl</span> |
| <a id="__codelineno-0-2095" name="__codelineno-0-2095"></a> |
| <a id="__codelineno-0-2096" name="__codelineno-0-2096"></a> <span class="n">result</span> <span class="o">=</span> <span class="n">pl</span><span class="o">.</span><span class="n">from_arrow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span> |
| <a id="__codelineno-0-2097" name="__codelineno-0-2097"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">pl</span><span class="o">.</span><span class="n">Series</span><span class="p">):</span> |
| <a id="__codelineno-0-2098" name="__codelineno-0-2098"></a> <span class="n">result</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">to_frame</span><span class="p">()</span> |
| <a id="__codelineno-0-2099" name="__codelineno-0-2099"></a> |
| <a id="__codelineno-0-2100" name="__codelineno-0-2100"></a> <span class="k">return</span> <span class="n">result</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.DataScan.to_ray" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_ray</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.DataScan.to_ray" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Read a Ray Dataset eagerly from this Iceberg table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="ray.data.dataset.Dataset">Dataset</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>ray.data.dataset.Dataset: Materialized Ray Dataset from the Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2078">2078</a></span> |
| <span class="normal"><a href="#__codelineno-0-2079">2079</a></span> |
| <span class="normal"><a href="#__codelineno-0-2080">2080</a></span> |
| <span class="normal"><a href="#__codelineno-0-2081">2081</a></span> |
| <span class="normal"><a href="#__codelineno-0-2082">2082</a></span> |
| <span class="normal"><a href="#__codelineno-0-2083">2083</a></span> |
| <span class="normal"><a href="#__codelineno-0-2084">2084</a></span> |
| <span class="normal"><a href="#__codelineno-0-2085">2085</a></span> |
| <span class="normal"><a href="#__codelineno-0-2086">2086</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2078" name="__codelineno-0-2078"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_ray</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ray</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">Dataset</span><span class="p">:</span> |
| <a id="__codelineno-0-2079" name="__codelineno-0-2079"></a><span class="w"> </span><span class="sd">"""Read a Ray Dataset eagerly from this Iceberg table.</span> |
| <a id="__codelineno-0-2080" name="__codelineno-0-2080"></a> |
| <a id="__codelineno-0-2081" name="__codelineno-0-2081"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-2082" name="__codelineno-0-2082"></a><span class="sd"> ray.data.dataset.Dataset: Materialized Ray Dataset from the Iceberg table</span> |
| <a id="__codelineno-0-2083" name="__codelineno-0-2083"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-2084" name="__codelineno-0-2084"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">ray</span> |
| <a id="__codelineno-0-2085" name="__codelineno-0-2085"></a> |
| <a id="__codelineno-0-2086" name="__codelineno-0-2086"></a> <span class="k">return</span> <span class="n">ray</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">from_arrow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.FileScanTask" class="doc doc-heading"> |
| <code>FileScanTask</code> |
| |
| |
| <span class="doc doc-labels"> |
| <small class="doc doc-label doc-label-dataclass"><code>dataclass</code></small> |
| </span> |
| |
| <a href="#pyiceberg.table.FileScanTask" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><span title="pyiceberg.table.ScanTask">ScanTask</span></code></p> |
| |
| |
| <p>Task representing a data file and its corresponding delete files.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1793">1793</a></span> |
| <span class="normal"><a href="#__codelineno-0-1794">1794</a></span> |
| <span class="normal"><a href="#__codelineno-0-1795">1795</a></span> |
| <span class="normal"><a href="#__codelineno-0-1796">1796</a></span> |
| <span class="normal"><a href="#__codelineno-0-1797">1797</a></span> |
| <span class="normal"><a href="#__codelineno-0-1798">1798</a></span> |
| <span class="normal"><a href="#__codelineno-0-1799">1799</a></span> |
| <span class="normal"><a href="#__codelineno-0-1800">1800</a></span> |
| <span class="normal"><a href="#__codelineno-0-1801">1801</a></span> |
| <span class="normal"><a href="#__codelineno-0-1802">1802</a></span> |
| <span class="normal"><a href="#__codelineno-0-1803">1803</a></span> |
| <span class="normal"><a href="#__codelineno-0-1804">1804</a></span> |
| <span class="normal"><a href="#__codelineno-0-1805">1805</a></span> |
| <span class="normal"><a href="#__codelineno-0-1806">1806</a></span> |
| <span class="normal"><a href="#__codelineno-0-1807">1807</a></span> |
| <span class="normal"><a href="#__codelineno-0-1808">1808</a></span> |
| <span class="normal"><a href="#__codelineno-0-1809">1809</a></span> |
| <span class="normal"><a href="#__codelineno-0-1810">1810</a></span> |
| <span class="normal"><a href="#__codelineno-0-1811">1811</a></span> |
| <span class="normal"><a href="#__codelineno-0-1812">1812</a></span> |
| <span class="normal"><a href="#__codelineno-0-1813">1813</a></span> |
| <span class="normal"><a href="#__codelineno-0-1814">1814</a></span> |
| <span class="normal"><a href="#__codelineno-0-1815">1815</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1793" name="__codelineno-0-1793"></a><span class="nd">@dataclass</span><span class="p">(</span><span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
| <a id="__codelineno-0-1794" name="__codelineno-0-1794"></a><span class="k">class</span><span class="w"> </span><span class="nc">FileScanTask</span><span class="p">(</span><span class="n">ScanTask</span><span class="p">):</span> |
| <a id="__codelineno-0-1795" name="__codelineno-0-1795"></a><span class="w"> </span><span class="sd">"""Task representing a data file and its corresponding delete files."""</span> |
| <a id="__codelineno-0-1796" name="__codelineno-0-1796"></a> |
| <a id="__codelineno-0-1797" name="__codelineno-0-1797"></a> <span class="n">file</span><span class="p">:</span> <span class="n">DataFile</span> |
| <a id="__codelineno-0-1798" name="__codelineno-0-1798"></a> <span class="n">delete_files</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="n">DataFile</span><span class="p">]</span> |
| <a id="__codelineno-0-1799" name="__codelineno-0-1799"></a> <span class="n">start</span><span class="p">:</span> <span class="nb">int</span> |
| <a id="__codelineno-0-1800" name="__codelineno-0-1800"></a> <span class="n">length</span><span class="p">:</span> <span class="nb">int</span> |
| <a id="__codelineno-0-1801" name="__codelineno-0-1801"></a> <span class="n">residual</span><span class="p">:</span> <span class="n">BooleanExpression</span> |
| <a id="__codelineno-0-1802" name="__codelineno-0-1802"></a> |
| <a id="__codelineno-0-1803" name="__codelineno-0-1803"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span> |
| <a id="__codelineno-0-1804" name="__codelineno-0-1804"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1805" name="__codelineno-0-1805"></a> <span class="n">data_file</span><span class="p">:</span> <span class="n">DataFile</span><span class="p">,</span> |
| <a id="__codelineno-0-1806" name="__codelineno-0-1806"></a> <span class="n">delete_files</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Set</span><span class="p">[</span><span class="n">DataFile</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1807" name="__codelineno-0-1807"></a> <span class="n">start</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1808" name="__codelineno-0-1808"></a> <span class="n">length</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1809" name="__codelineno-0-1809"></a> <span class="n">residual</span><span class="p">:</span> <span class="n">BooleanExpression</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1810" name="__codelineno-0-1810"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1811" name="__codelineno-0-1811"></a> <span class="bp">self</span><span class="o">.</span><span class="n">file</span> <span class="o">=</span> <span class="n">data_file</span> |
| <a id="__codelineno-0-1812" name="__codelineno-0-1812"></a> <span class="bp">self</span><span class="o">.</span><span class="n">delete_files</span> <span class="o">=</span> <span class="n">delete_files</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> |
| <a id="__codelineno-0-1813" name="__codelineno-0-1813"></a> <span class="bp">self</span><span class="o">.</span><span class="n">start</span> <span class="o">=</span> <span class="n">start</span> <span class="ow">or</span> <span class="mi">0</span> |
| <a id="__codelineno-0-1814" name="__codelineno-0-1814"></a> <span class="bp">self</span><span class="o">.</span><span class="n">length</span> <span class="o">=</span> <span class="n">length</span> <span class="ow">or</span> <span class="n">data_file</span><span class="o">.</span><span class="n">file_size_in_bytes</span> |
| <a id="__codelineno-0-1815" name="__codelineno-0-1815"></a> <span class="bp">self</span><span class="o">.</span><span class="n">residual</span> <span class="o">=</span> <span class="n">residual</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.Namespace" class="doc doc-heading"> |
| <code>Namespace</code> |
| |
| |
| <a href="#pyiceberg.table.Namespace" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="IcebergRootModel (pyiceberg.typedef.IcebergRootModel)" href="../typedef/#pyiceberg.typedef.IcebergRootModel">IcebergRootModel</a>[<span title="typing.List">List</span>[<span title="str">str</span>]]</code></p> |
| |
| |
| <p>Reference to one or more levels of a namespace.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1031">1031</a></span> |
| <span class="normal"><a href="#__codelineno-0-1032">1032</a></span> |
| <span class="normal"><a href="#__codelineno-0-1033">1033</a></span> |
| <span class="normal"><a href="#__codelineno-0-1034">1034</a></span> |
| <span class="normal"><a href="#__codelineno-0-1035">1035</a></span> |
| <span class="normal"><a href="#__codelineno-0-1036">1036</a></span> |
| <span class="normal"><a href="#__codelineno-0-1037">1037</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1031" name="__codelineno-0-1031"></a><span class="k">class</span><span class="w"> </span><span class="nc">Namespace</span><span class="p">(</span><span class="n">IcebergRootModel</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]):</span> |
| <a id="__codelineno-0-1032" name="__codelineno-0-1032"></a><span class="w"> </span><span class="sd">"""Reference to one or more levels of a namespace."""</span> |
| <a id="__codelineno-0-1033" name="__codelineno-0-1033"></a> |
| <a id="__codelineno-0-1034" name="__codelineno-0-1034"></a> <span class="n">root</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span> |
| <a id="__codelineno-0-1035" name="__codelineno-0-1035"></a> <span class="o">...</span><span class="p">,</span> |
| <a id="__codelineno-0-1036" name="__codelineno-0-1036"></a> <span class="n">description</span><span class="o">=</span><span class="s2">"Reference to one or more levels of a namespace"</span><span class="p">,</span> |
| <a id="__codelineno-0-1037" name="__codelineno-0-1037"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.StaticTable" class="doc doc-heading"> |
| <code>StaticTable</code> |
| |
| |
| <a href="#pyiceberg.table.StaticTable" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="Table (pyiceberg.table.Table)" href="#pyiceberg.table.Table">Table</a></code></p> |
| |
| |
| <p>Load a table directly from a metadata file (i.e., without using a catalog).</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1607">1607</a></span> |
| <span class="normal"><a href="#__codelineno-0-1608">1608</a></span> |
| <span class="normal"><a href="#__codelineno-0-1609">1609</a></span> |
| <span class="normal"><a href="#__codelineno-0-1610">1610</a></span> |
| <span class="normal"><a href="#__codelineno-0-1611">1611</a></span> |
| <span class="normal"><a href="#__codelineno-0-1612">1612</a></span> |
| <span class="normal"><a href="#__codelineno-0-1613">1613</a></span> |
| <span class="normal"><a href="#__codelineno-0-1614">1614</a></span> |
| <span class="normal"><a href="#__codelineno-0-1615">1615</a></span> |
| <span class="normal"><a href="#__codelineno-0-1616">1616</a></span> |
| <span class="normal"><a href="#__codelineno-0-1617">1617</a></span> |
| <span class="normal"><a href="#__codelineno-0-1618">1618</a></span> |
| <span class="normal"><a href="#__codelineno-0-1619">1619</a></span> |
| <span class="normal"><a href="#__codelineno-0-1620">1620</a></span> |
| <span class="normal"><a href="#__codelineno-0-1621">1621</a></span> |
| <span class="normal"><a href="#__codelineno-0-1622">1622</a></span> |
| <span class="normal"><a href="#__codelineno-0-1623">1623</a></span> |
| <span class="normal"><a href="#__codelineno-0-1624">1624</a></span> |
| <span class="normal"><a href="#__codelineno-0-1625">1625</a></span> |
| <span class="normal"><a href="#__codelineno-0-1626">1626</a></span> |
| <span class="normal"><a href="#__codelineno-0-1627">1627</a></span> |
| <span class="normal"><a href="#__codelineno-0-1628">1628</a></span> |
| <span class="normal"><a href="#__codelineno-0-1629">1629</a></span> |
| <span class="normal"><a href="#__codelineno-0-1630">1630</a></span> |
| <span class="normal"><a href="#__codelineno-0-1631">1631</a></span> |
| <span class="normal"><a href="#__codelineno-0-1632">1632</a></span> |
| <span class="normal"><a href="#__codelineno-0-1633">1633</a></span> |
| <span class="normal"><a href="#__codelineno-0-1634">1634</a></span> |
| <span class="normal"><a href="#__codelineno-0-1635">1635</a></span> |
| <span class="normal"><a href="#__codelineno-0-1636">1636</a></span> |
| <span class="normal"><a href="#__codelineno-0-1637">1637</a></span> |
| <span class="normal"><a href="#__codelineno-0-1638">1638</a></span> |
| <span class="normal"><a href="#__codelineno-0-1639">1639</a></span> |
| <span class="normal"><a href="#__codelineno-0-1640">1640</a></span> |
| <span class="normal"><a href="#__codelineno-0-1641">1641</a></span> |
| <span class="normal"><a href="#__codelineno-0-1642">1642</a></span> |
| <span class="normal"><a href="#__codelineno-0-1643">1643</a></span> |
| <span class="normal"><a href="#__codelineno-0-1644">1644</a></span> |
| <span class="normal"><a href="#__codelineno-0-1645">1645</a></span> |
| <span class="normal"><a href="#__codelineno-0-1646">1646</a></span> |
| <span class="normal"><a href="#__codelineno-0-1647">1647</a></span> |
| <span class="normal"><a href="#__codelineno-0-1648">1648</a></span> |
| <span class="normal"><a href="#__codelineno-0-1649">1649</a></span> |
| <span class="normal"><a href="#__codelineno-0-1650">1650</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1607" name="__codelineno-0-1607"></a><span class="k">class</span><span class="w"> </span><span class="nc">StaticTable</span><span class="p">(</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-1608" name="__codelineno-0-1608"></a><span class="w"> </span><span class="sd">"""Load a table directly from a metadata file (i.e., without using a catalog)."""</span> |
| <a id="__codelineno-0-1609" name="__codelineno-0-1609"></a> |
| <a id="__codelineno-0-1610" name="__codelineno-0-1610"></a> <span class="k">def</span><span class="w"> </span><span class="nf">refresh</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1611" name="__codelineno-0-1611"></a><span class="w"> </span><span class="sd">"""Refresh the current table metadata."""</span> |
| <a id="__codelineno-0-1612" name="__codelineno-0-1612"></a> <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"To be implemented"</span><span class="p">)</span> |
| <a id="__codelineno-0-1613" name="__codelineno-0-1613"></a> |
| <a id="__codelineno-0-1614" name="__codelineno-0-1614"></a> <span class="nd">@classmethod</span> |
| <a id="__codelineno-0-1615" name="__codelineno-0-1615"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_metadata_location_from_version_hint</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">metadata_location</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">properties</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <a id="__codelineno-0-1616" name="__codelineno-0-1616"></a> <span class="n">version_hint_location</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata_location</span><span class="p">,</span> <span class="s2">"metadata"</span><span class="p">,</span> <span class="s2">"version-hint.text"</span><span class="p">)</span> |
| <a id="__codelineno-0-1617" name="__codelineno-0-1617"></a> <span class="n">io</span> <span class="o">=</span> <span class="n">load_file_io</span><span class="p">(</span><span class="n">properties</span><span class="o">=</span><span class="n">properties</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="n">version_hint_location</span><span class="p">)</span> |
| <a id="__codelineno-0-1618" name="__codelineno-0-1618"></a> <span class="n">file</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">new_input</span><span class="p">(</span><span class="n">version_hint_location</span><span class="p">)</span> |
| <a id="__codelineno-0-1619" name="__codelineno-0-1619"></a> |
| <a id="__codelineno-0-1620" name="__codelineno-0-1620"></a> <span class="k">with</span> <span class="n">file</span><span class="o">.</span><span class="n">open</span><span class="p">()</span> <span class="k">as</span> <span class="n">stream</span><span class="p">:</span> |
| <a id="__codelineno-0-1621" name="__codelineno-0-1621"></a> <span class="n">content</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">)</span> |
| <a id="__codelineno-0-1622" name="__codelineno-0-1622"></a> |
| <a id="__codelineno-0-1623" name="__codelineno-0-1623"></a> <span class="k">if</span> <span class="n">content</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".metadata.json"</span><span class="p">):</span> |
| <a id="__codelineno-0-1624" name="__codelineno-0-1624"></a> <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata_location</span><span class="p">,</span> <span class="s2">"metadata"</span><span class="p">,</span> <span class="n">content</span><span class="p">)</span> |
| <a id="__codelineno-0-1625" name="__codelineno-0-1625"></a> <span class="k">elif</span> <span class="n">content</span><span class="o">.</span><span class="n">isnumeric</span><span class="p">():</span> |
| <a id="__codelineno-0-1626" name="__codelineno-0-1626"></a> <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata_location</span><span class="p">,</span> <span class="s2">"metadata"</span><span class="p">,</span> <span class="s2">"v</span><span class="si">%s</span><span class="s2">.metadata.json"</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> |
| <a id="__codelineno-0-1627" name="__codelineno-0-1627"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-1628" name="__codelineno-0-1628"></a> <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata_location</span><span class="p">,</span> <span class="s2">"metadata"</span><span class="p">,</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.metadata.json"</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> |
| <a id="__codelineno-0-1629" name="__codelineno-0-1629"></a> |
| <a id="__codelineno-0-1630" name="__codelineno-0-1630"></a> <span class="nd">@classmethod</span> |
| <a id="__codelineno-0-1631" name="__codelineno-0-1631"></a> <span class="k">def</span><span class="w"> </span><span class="nf">from_metadata</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">metadata_location</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">properties</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">)</span> <span class="o">-></span> <span class="n">StaticTable</span><span class="p">:</span> |
| <a id="__codelineno-0-1632" name="__codelineno-0-1632"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">metadata_location</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".metadata.json"</span><span class="p">):</span> |
| <a id="__codelineno-0-1633" name="__codelineno-0-1633"></a> <span class="n">metadata_location</span> <span class="o">=</span> <span class="n">StaticTable</span><span class="o">.</span><span class="n">_metadata_location_from_version_hint</span><span class="p">(</span><span class="n">metadata_location</span><span class="p">,</span> <span class="n">properties</span><span class="p">)</span> |
| <a id="__codelineno-0-1634" name="__codelineno-0-1634"></a> |
| <a id="__codelineno-0-1635" name="__codelineno-0-1635"></a> <span class="n">io</span> <span class="o">=</span> <span class="n">load_file_io</span><span class="p">(</span><span class="n">properties</span><span class="o">=</span><span class="n">properties</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="n">metadata_location</span><span class="p">)</span> |
| <a id="__codelineno-0-1636" name="__codelineno-0-1636"></a> <span class="n">file</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">new_input</span><span class="p">(</span><span class="n">metadata_location</span><span class="p">)</span> |
| <a id="__codelineno-0-1637" name="__codelineno-0-1637"></a> |
| <a id="__codelineno-0-1638" name="__codelineno-0-1638"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.serializers</span><span class="w"> </span><span class="kn">import</span> <span class="n">FromInputFile</span> |
| <a id="__codelineno-0-1639" name="__codelineno-0-1639"></a> |
| <a id="__codelineno-0-1640" name="__codelineno-0-1640"></a> <span class="n">metadata</span> <span class="o">=</span> <span class="n">FromInputFile</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">(</span><span class="n">file</span><span class="p">)</span> |
| <a id="__codelineno-0-1641" name="__codelineno-0-1641"></a> |
| <a id="__codelineno-0-1642" name="__codelineno-0-1642"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.catalog.noop</span><span class="w"> </span><span class="kn">import</span> <span class="n">NoopCatalog</span> |
| <a id="__codelineno-0-1643" name="__codelineno-0-1643"></a> |
| <a id="__codelineno-0-1644" name="__codelineno-0-1644"></a> <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span> |
| <a id="__codelineno-0-1645" name="__codelineno-0-1645"></a> <span class="n">identifier</span><span class="o">=</span><span class="p">(</span><span class="s2">"static-table"</span><span class="p">,</span> <span class="n">metadata_location</span><span class="p">),</span> |
| <a id="__codelineno-0-1646" name="__codelineno-0-1646"></a> <span class="n">metadata_location</span><span class="o">=</span><span class="n">metadata_location</span><span class="p">,</span> |
| <a id="__codelineno-0-1647" name="__codelineno-0-1647"></a> <span class="n">metadata</span><span class="o">=</span><span class="n">metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-1648" name="__codelineno-0-1648"></a> <span class="n">io</span><span class="o">=</span><span class="n">load_file_io</span><span class="p">({</span><span class="o">**</span><span class="n">properties</span><span class="p">,</span> <span class="o">**</span><span class="n">metadata</span><span class="o">.</span><span class="n">properties</span><span class="p">}),</span> |
| <a id="__codelineno-0-1649" name="__codelineno-0-1649"></a> <span class="n">catalog</span><span class="o">=</span><span class="n">NoopCatalog</span><span class="p">(</span><span class="s2">"static-table"</span><span class="p">),</span> |
| <a id="__codelineno-0-1650" name="__codelineno-0-1650"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.StaticTable.refresh" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">refresh</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.StaticTable.refresh" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Refresh the current table metadata.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1610">1610</a></span> |
| <span class="normal"><a href="#__codelineno-0-1611">1611</a></span> |
| <span class="normal"><a href="#__codelineno-0-1612">1612</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1610" name="__codelineno-0-1610"></a><span class="k">def</span><span class="w"> </span><span class="nf">refresh</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1611" name="__codelineno-0-1611"></a><span class="w"> </span><span class="sd">"""Refresh the current table metadata."""</span> |
| <a id="__codelineno-0-1612" name="__codelineno-0-1612"></a> <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"To be implemented"</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.Table" class="doc doc-heading"> |
| <code>Table</code> |
| |
| |
| <a href="#pyiceberg.table.Table" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| |
| |
| <p>An Iceberg table.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1062">1062</a></span> |
| <span class="normal"><a href="#__codelineno-0-1063">1063</a></span> |
| <span class="normal"><a href="#__codelineno-0-1064">1064</a></span> |
| <span class="normal"><a href="#__codelineno-0-1065">1065</a></span> |
| <span class="normal"><a href="#__codelineno-0-1066">1066</a></span> |
| <span class="normal"><a href="#__codelineno-0-1067">1067</a></span> |
| <span class="normal"><a href="#__codelineno-0-1068">1068</a></span> |
| <span class="normal"><a href="#__codelineno-0-1069">1069</a></span> |
| <span class="normal"><a href="#__codelineno-0-1070">1070</a></span> |
| <span class="normal"><a href="#__codelineno-0-1071">1071</a></span> |
| <span class="normal"><a href="#__codelineno-0-1072">1072</a></span> |
| <span class="normal"><a href="#__codelineno-0-1073">1073</a></span> |
| <span class="normal"><a href="#__codelineno-0-1074">1074</a></span> |
| <span class="normal"><a href="#__codelineno-0-1075">1075</a></span> |
| <span class="normal"><a href="#__codelineno-0-1076">1076</a></span> |
| <span class="normal"><a href="#__codelineno-0-1077">1077</a></span> |
| <span class="normal"><a href="#__codelineno-0-1078">1078</a></span> |
| <span class="normal"><a href="#__codelineno-0-1079">1079</a></span> |
| <span class="normal"><a href="#__codelineno-0-1080">1080</a></span> |
| <span class="normal"><a href="#__codelineno-0-1081">1081</a></span> |
| <span class="normal"><a href="#__codelineno-0-1082">1082</a></span> |
| <span class="normal"><a href="#__codelineno-0-1083">1083</a></span> |
| <span class="normal"><a href="#__codelineno-0-1084">1084</a></span> |
| <span class="normal"><a href="#__codelineno-0-1085">1085</a></span> |
| <span class="normal"><a href="#__codelineno-0-1086">1086</a></span> |
| <span class="normal"><a href="#__codelineno-0-1087">1087</a></span> |
| <span class="normal"><a href="#__codelineno-0-1088">1088</a></span> |
| <span class="normal"><a href="#__codelineno-0-1089">1089</a></span> |
| <span class="normal"><a href="#__codelineno-0-1090">1090</a></span> |
| <span class="normal"><a href="#__codelineno-0-1091">1091</a></span> |
| <span class="normal"><a href="#__codelineno-0-1092">1092</a></span> |
| <span class="normal"><a href="#__codelineno-0-1093">1093</a></span> |
| <span class="normal"><a href="#__codelineno-0-1094">1094</a></span> |
| <span class="normal"><a href="#__codelineno-0-1095">1095</a></span> |
| <span class="normal"><a href="#__codelineno-0-1096">1096</a></span> |
| <span class="normal"><a href="#__codelineno-0-1097">1097</a></span> |
| <span class="normal"><a href="#__codelineno-0-1098">1098</a></span> |
| <span class="normal"><a href="#__codelineno-0-1099">1099</a></span> |
| <span class="normal"><a href="#__codelineno-0-1100">1100</a></span> |
| <span class="normal"><a href="#__codelineno-0-1101">1101</a></span> |
| <span class="normal"><a href="#__codelineno-0-1102">1102</a></span> |
| <span class="normal"><a href="#__codelineno-0-1103">1103</a></span> |
| <span class="normal"><a href="#__codelineno-0-1104">1104</a></span> |
| <span class="normal"><a href="#__codelineno-0-1105">1105</a></span> |
| <span class="normal"><a href="#__codelineno-0-1106">1106</a></span> |
| <span class="normal"><a href="#__codelineno-0-1107">1107</a></span> |
| <span class="normal"><a href="#__codelineno-0-1108">1108</a></span> |
| <span class="normal"><a href="#__codelineno-0-1109">1109</a></span> |
| <span class="normal"><a href="#__codelineno-0-1110">1110</a></span> |
| <span class="normal"><a href="#__codelineno-0-1111">1111</a></span> |
| <span class="normal"><a href="#__codelineno-0-1112">1112</a></span> |
| <span class="normal"><a href="#__codelineno-0-1113">1113</a></span> |
| <span class="normal"><a href="#__codelineno-0-1114">1114</a></span> |
| <span class="normal"><a href="#__codelineno-0-1115">1115</a></span> |
| <span class="normal"><a href="#__codelineno-0-1116">1116</a></span> |
| <span class="normal"><a href="#__codelineno-0-1117">1117</a></span> |
| <span class="normal"><a href="#__codelineno-0-1118">1118</a></span> |
| <span class="normal"><a href="#__codelineno-0-1119">1119</a></span> |
| <span class="normal"><a href="#__codelineno-0-1120">1120</a></span> |
| <span class="normal"><a href="#__codelineno-0-1121">1121</a></span> |
| <span class="normal"><a href="#__codelineno-0-1122">1122</a></span> |
| <span class="normal"><a href="#__codelineno-0-1123">1123</a></span> |
| <span class="normal"><a href="#__codelineno-0-1124">1124</a></span> |
| <span class="normal"><a href="#__codelineno-0-1125">1125</a></span> |
| <span class="normal"><a href="#__codelineno-0-1126">1126</a></span> |
| <span class="normal"><a href="#__codelineno-0-1127">1127</a></span> |
| <span class="normal"><a href="#__codelineno-0-1128">1128</a></span> |
| <span class="normal"><a href="#__codelineno-0-1129">1129</a></span> |
| <span class="normal"><a href="#__codelineno-0-1130">1130</a></span> |
| <span class="normal"><a href="#__codelineno-0-1131">1131</a></span> |
| <span class="normal"><a href="#__codelineno-0-1132">1132</a></span> |
| <span class="normal"><a href="#__codelineno-0-1133">1133</a></span> |
| <span class="normal"><a href="#__codelineno-0-1134">1134</a></span> |
| <span class="normal"><a href="#__codelineno-0-1135">1135</a></span> |
| <span class="normal"><a href="#__codelineno-0-1136">1136</a></span> |
| <span class="normal"><a href="#__codelineno-0-1137">1137</a></span> |
| <span class="normal"><a href="#__codelineno-0-1138">1138</a></span> |
| <span class="normal"><a href="#__codelineno-0-1139">1139</a></span> |
| <span class="normal"><a href="#__codelineno-0-1140">1140</a></span> |
| <span class="normal"><a href="#__codelineno-0-1141">1141</a></span> |
| <span class="normal"><a href="#__codelineno-0-1142">1142</a></span> |
| <span class="normal"><a href="#__codelineno-0-1143">1143</a></span> |
| <span class="normal"><a href="#__codelineno-0-1144">1144</a></span> |
| <span class="normal"><a href="#__codelineno-0-1145">1145</a></span> |
| <span class="normal"><a href="#__codelineno-0-1146">1146</a></span> |
| <span class="normal"><a href="#__codelineno-0-1147">1147</a></span> |
| <span class="normal"><a href="#__codelineno-0-1148">1148</a></span> |
| <span class="normal"><a href="#__codelineno-0-1149">1149</a></span> |
| <span class="normal"><a href="#__codelineno-0-1150">1150</a></span> |
| <span class="normal"><a href="#__codelineno-0-1151">1151</a></span> |
| <span class="normal"><a href="#__codelineno-0-1152">1152</a></span> |
| <span class="normal"><a href="#__codelineno-0-1153">1153</a></span> |
| <span class="normal"><a href="#__codelineno-0-1154">1154</a></span> |
| <span class="normal"><a href="#__codelineno-0-1155">1155</a></span> |
| <span class="normal"><a href="#__codelineno-0-1156">1156</a></span> |
| <span class="normal"><a href="#__codelineno-0-1157">1157</a></span> |
| <span class="normal"><a href="#__codelineno-0-1158">1158</a></span> |
| <span class="normal"><a href="#__codelineno-0-1159">1159</a></span> |
| <span class="normal"><a href="#__codelineno-0-1160">1160</a></span> |
| <span class="normal"><a href="#__codelineno-0-1161">1161</a></span> |
| <span class="normal"><a href="#__codelineno-0-1162">1162</a></span> |
| <span class="normal"><a href="#__codelineno-0-1163">1163</a></span> |
| <span class="normal"><a href="#__codelineno-0-1164">1164</a></span> |
| <span class="normal"><a href="#__codelineno-0-1165">1165</a></span> |
| <span class="normal"><a href="#__codelineno-0-1166">1166</a></span> |
| <span class="normal"><a href="#__codelineno-0-1167">1167</a></span> |
| <span class="normal"><a href="#__codelineno-0-1168">1168</a></span> |
| <span class="normal"><a href="#__codelineno-0-1169">1169</a></span> |
| <span class="normal"><a href="#__codelineno-0-1170">1170</a></span> |
| <span class="normal"><a href="#__codelineno-0-1171">1171</a></span> |
| <span class="normal"><a href="#__codelineno-0-1172">1172</a></span> |
| <span class="normal"><a href="#__codelineno-0-1173">1173</a></span> |
| <span class="normal"><a href="#__codelineno-0-1174">1174</a></span> |
| <span class="normal"><a href="#__codelineno-0-1175">1175</a></span> |
| <span class="normal"><a href="#__codelineno-0-1176">1176</a></span> |
| <span class="normal"><a href="#__codelineno-0-1177">1177</a></span> |
| <span class="normal"><a href="#__codelineno-0-1178">1178</a></span> |
| <span class="normal"><a href="#__codelineno-0-1179">1179</a></span> |
| <span class="normal"><a href="#__codelineno-0-1180">1180</a></span> |
| <span class="normal"><a href="#__codelineno-0-1181">1181</a></span> |
| <span class="normal"><a href="#__codelineno-0-1182">1182</a></span> |
| <span class="normal"><a href="#__codelineno-0-1183">1183</a></span> |
| <span class="normal"><a href="#__codelineno-0-1184">1184</a></span> |
| <span class="normal"><a href="#__codelineno-0-1185">1185</a></span> |
| <span class="normal"><a href="#__codelineno-0-1186">1186</a></span> |
| <span class="normal"><a href="#__codelineno-0-1187">1187</a></span> |
| <span class="normal"><a href="#__codelineno-0-1188">1188</a></span> |
| <span class="normal"><a href="#__codelineno-0-1189">1189</a></span> |
| <span class="normal"><a href="#__codelineno-0-1190">1190</a></span> |
| <span class="normal"><a href="#__codelineno-0-1191">1191</a></span> |
| <span class="normal"><a href="#__codelineno-0-1192">1192</a></span> |
| <span class="normal"><a href="#__codelineno-0-1193">1193</a></span> |
| <span class="normal"><a href="#__codelineno-0-1194">1194</a></span> |
| <span class="normal"><a href="#__codelineno-0-1195">1195</a></span> |
| <span class="normal"><a href="#__codelineno-0-1196">1196</a></span> |
| <span class="normal"><a href="#__codelineno-0-1197">1197</a></span> |
| <span class="normal"><a href="#__codelineno-0-1198">1198</a></span> |
| <span class="normal"><a href="#__codelineno-0-1199">1199</a></span> |
| <span class="normal"><a href="#__codelineno-0-1200">1200</a></span> |
| <span class="normal"><a href="#__codelineno-0-1201">1201</a></span> |
| <span class="normal"><a href="#__codelineno-0-1202">1202</a></span> |
| <span class="normal"><a href="#__codelineno-0-1203">1203</a></span> |
| <span class="normal"><a href="#__codelineno-0-1204">1204</a></span> |
| <span class="normal"><a href="#__codelineno-0-1205">1205</a></span> |
| <span class="normal"><a href="#__codelineno-0-1206">1206</a></span> |
| <span class="normal"><a href="#__codelineno-0-1207">1207</a></span> |
| <span class="normal"><a href="#__codelineno-0-1208">1208</a></span> |
| <span class="normal"><a href="#__codelineno-0-1209">1209</a></span> |
| <span class="normal"><a href="#__codelineno-0-1210">1210</a></span> |
| <span class="normal"><a href="#__codelineno-0-1211">1211</a></span> |
| <span class="normal"><a href="#__codelineno-0-1212">1212</a></span> |
| <span class="normal"><a href="#__codelineno-0-1213">1213</a></span> |
| <span class="normal"><a href="#__codelineno-0-1214">1214</a></span> |
| <span class="normal"><a href="#__codelineno-0-1215">1215</a></span> |
| <span class="normal"><a href="#__codelineno-0-1216">1216</a></span> |
| <span class="normal"><a href="#__codelineno-0-1217">1217</a></span> |
| <span class="normal"><a href="#__codelineno-0-1218">1218</a></span> |
| <span class="normal"><a href="#__codelineno-0-1219">1219</a></span> |
| <span class="normal"><a href="#__codelineno-0-1220">1220</a></span> |
| <span class="normal"><a href="#__codelineno-0-1221">1221</a></span> |
| <span class="normal"><a href="#__codelineno-0-1222">1222</a></span> |
| <span class="normal"><a href="#__codelineno-0-1223">1223</a></span> |
| <span class="normal"><a href="#__codelineno-0-1224">1224</a></span> |
| <span class="normal"><a href="#__codelineno-0-1225">1225</a></span> |
| <span class="normal"><a href="#__codelineno-0-1226">1226</a></span> |
| <span class="normal"><a href="#__codelineno-0-1227">1227</a></span> |
| <span class="normal"><a href="#__codelineno-0-1228">1228</a></span> |
| <span class="normal"><a href="#__codelineno-0-1229">1229</a></span> |
| <span class="normal"><a href="#__codelineno-0-1230">1230</a></span> |
| <span class="normal"><a href="#__codelineno-0-1231">1231</a></span> |
| <span class="normal"><a href="#__codelineno-0-1232">1232</a></span> |
| <span class="normal"><a href="#__codelineno-0-1233">1233</a></span> |
| <span class="normal"><a href="#__codelineno-0-1234">1234</a></span> |
| <span class="normal"><a href="#__codelineno-0-1235">1235</a></span> |
| <span class="normal"><a href="#__codelineno-0-1236">1236</a></span> |
| <span class="normal"><a href="#__codelineno-0-1237">1237</a></span> |
| <span class="normal"><a href="#__codelineno-0-1238">1238</a></span> |
| <span class="normal"><a href="#__codelineno-0-1239">1239</a></span> |
| <span class="normal"><a href="#__codelineno-0-1240">1240</a></span> |
| <span class="normal"><a href="#__codelineno-0-1241">1241</a></span> |
| <span class="normal"><a href="#__codelineno-0-1242">1242</a></span> |
| <span class="normal"><a href="#__codelineno-0-1243">1243</a></span> |
| <span class="normal"><a href="#__codelineno-0-1244">1244</a></span> |
| <span class="normal"><a href="#__codelineno-0-1245">1245</a></span> |
| <span class="normal"><a href="#__codelineno-0-1246">1246</a></span> |
| <span class="normal"><a href="#__codelineno-0-1247">1247</a></span> |
| <span class="normal"><a href="#__codelineno-0-1248">1248</a></span> |
| <span class="normal"><a href="#__codelineno-0-1249">1249</a></span> |
| <span class="normal"><a href="#__codelineno-0-1250">1250</a></span> |
| <span class="normal"><a href="#__codelineno-0-1251">1251</a></span> |
| <span class="normal"><a href="#__codelineno-0-1252">1252</a></span> |
| <span class="normal"><a href="#__codelineno-0-1253">1253</a></span> |
| <span class="normal"><a href="#__codelineno-0-1254">1254</a></span> |
| <span class="normal"><a href="#__codelineno-0-1255">1255</a></span> |
| <span class="normal"><a href="#__codelineno-0-1256">1256</a></span> |
| <span class="normal"><a href="#__codelineno-0-1257">1257</a></span> |
| <span class="normal"><a href="#__codelineno-0-1258">1258</a></span> |
| <span class="normal"><a href="#__codelineno-0-1259">1259</a></span> |
| <span class="normal"><a href="#__codelineno-0-1260">1260</a></span> |
| <span class="normal"><a href="#__codelineno-0-1261">1261</a></span> |
| <span class="normal"><a href="#__codelineno-0-1262">1262</a></span> |
| <span class="normal"><a href="#__codelineno-0-1263">1263</a></span> |
| <span class="normal"><a href="#__codelineno-0-1264">1264</a></span> |
| <span class="normal"><a href="#__codelineno-0-1265">1265</a></span> |
| <span class="normal"><a href="#__codelineno-0-1266">1266</a></span> |
| <span class="normal"><a href="#__codelineno-0-1267">1267</a></span> |
| <span class="normal"><a href="#__codelineno-0-1268">1268</a></span> |
| <span class="normal"><a href="#__codelineno-0-1269">1269</a></span> |
| <span class="normal"><a href="#__codelineno-0-1270">1270</a></span> |
| <span class="normal"><a href="#__codelineno-0-1271">1271</a></span> |
| <span class="normal"><a href="#__codelineno-0-1272">1272</a></span> |
| <span class="normal"><a href="#__codelineno-0-1273">1273</a></span> |
| <span class="normal"><a href="#__codelineno-0-1274">1274</a></span> |
| <span class="normal"><a href="#__codelineno-0-1275">1275</a></span> |
| <span class="normal"><a href="#__codelineno-0-1276">1276</a></span> |
| <span class="normal"><a href="#__codelineno-0-1277">1277</a></span> |
| <span class="normal"><a href="#__codelineno-0-1278">1278</a></span> |
| <span class="normal"><a href="#__codelineno-0-1279">1279</a></span> |
| <span class="normal"><a href="#__codelineno-0-1280">1280</a></span> |
| <span class="normal"><a href="#__codelineno-0-1281">1281</a></span> |
| <span class="normal"><a href="#__codelineno-0-1282">1282</a></span> |
| <span class="normal"><a href="#__codelineno-0-1283">1283</a></span> |
| <span class="normal"><a href="#__codelineno-0-1284">1284</a></span> |
| <span class="normal"><a href="#__codelineno-0-1285">1285</a></span> |
| <span class="normal"><a href="#__codelineno-0-1286">1286</a></span> |
| <span class="normal"><a href="#__codelineno-0-1287">1287</a></span> |
| <span class="normal"><a href="#__codelineno-0-1288">1288</a></span> |
| <span class="normal"><a href="#__codelineno-0-1289">1289</a></span> |
| <span class="normal"><a href="#__codelineno-0-1290">1290</a></span> |
| <span class="normal"><a href="#__codelineno-0-1291">1291</a></span> |
| <span class="normal"><a href="#__codelineno-0-1292">1292</a></span> |
| <span class="normal"><a href="#__codelineno-0-1293">1293</a></span> |
| <span class="normal"><a href="#__codelineno-0-1294">1294</a></span> |
| <span class="normal"><a href="#__codelineno-0-1295">1295</a></span> |
| <span class="normal"><a href="#__codelineno-0-1296">1296</a></span> |
| <span class="normal"><a href="#__codelineno-0-1297">1297</a></span> |
| <span class="normal"><a href="#__codelineno-0-1298">1298</a></span> |
| <span class="normal"><a href="#__codelineno-0-1299">1299</a></span> |
| <span class="normal"><a href="#__codelineno-0-1300">1300</a></span> |
| <span class="normal"><a href="#__codelineno-0-1301">1301</a></span> |
| <span class="normal"><a href="#__codelineno-0-1302">1302</a></span> |
| <span class="normal"><a href="#__codelineno-0-1303">1303</a></span> |
| <span class="normal"><a href="#__codelineno-0-1304">1304</a></span> |
| <span class="normal"><a href="#__codelineno-0-1305">1305</a></span> |
| <span class="normal"><a href="#__codelineno-0-1306">1306</a></span> |
| <span class="normal"><a href="#__codelineno-0-1307">1307</a></span> |
| <span class="normal"><a href="#__codelineno-0-1308">1308</a></span> |
| <span class="normal"><a href="#__codelineno-0-1309">1309</a></span> |
| <span class="normal"><a href="#__codelineno-0-1310">1310</a></span> |
| <span class="normal"><a href="#__codelineno-0-1311">1311</a></span> |
| <span class="normal"><a href="#__codelineno-0-1312">1312</a></span> |
| <span class="normal"><a href="#__codelineno-0-1313">1313</a></span> |
| <span class="normal"><a href="#__codelineno-0-1314">1314</a></span> |
| <span class="normal"><a href="#__codelineno-0-1315">1315</a></span> |
| <span class="normal"><a href="#__codelineno-0-1316">1316</a></span> |
| <span class="normal"><a href="#__codelineno-0-1317">1317</a></span> |
| <span class="normal"><a href="#__codelineno-0-1318">1318</a></span> |
| <span class="normal"><a href="#__codelineno-0-1319">1319</a></span> |
| <span class="normal"><a href="#__codelineno-0-1320">1320</a></span> |
| <span class="normal"><a href="#__codelineno-0-1321">1321</a></span> |
| <span class="normal"><a href="#__codelineno-0-1322">1322</a></span> |
| <span class="normal"><a href="#__codelineno-0-1323">1323</a></span> |
| <span class="normal"><a href="#__codelineno-0-1324">1324</a></span> |
| <span class="normal"><a href="#__codelineno-0-1325">1325</a></span> |
| <span class="normal"><a href="#__codelineno-0-1326">1326</a></span> |
| <span class="normal"><a href="#__codelineno-0-1327">1327</a></span> |
| <span class="normal"><a href="#__codelineno-0-1328">1328</a></span> |
| <span class="normal"><a href="#__codelineno-0-1329">1329</a></span> |
| <span class="normal"><a href="#__codelineno-0-1330">1330</a></span> |
| <span class="normal"><a href="#__codelineno-0-1331">1331</a></span> |
| <span class="normal"><a href="#__codelineno-0-1332">1332</a></span> |
| <span class="normal"><a href="#__codelineno-0-1333">1333</a></span> |
| <span class="normal"><a href="#__codelineno-0-1334">1334</a></span> |
| <span class="normal"><a href="#__codelineno-0-1335">1335</a></span> |
| <span class="normal"><a href="#__codelineno-0-1336">1336</a></span> |
| <span class="normal"><a href="#__codelineno-0-1337">1337</a></span> |
| <span class="normal"><a href="#__codelineno-0-1338">1338</a></span> |
| <span class="normal"><a href="#__codelineno-0-1339">1339</a></span> |
| <span class="normal"><a href="#__codelineno-0-1340">1340</a></span> |
| <span class="normal"><a href="#__codelineno-0-1341">1341</a></span> |
| <span class="normal"><a href="#__codelineno-0-1342">1342</a></span> |
| <span class="normal"><a href="#__codelineno-0-1343">1343</a></span> |
| <span class="normal"><a href="#__codelineno-0-1344">1344</a></span> |
| <span class="normal"><a href="#__codelineno-0-1345">1345</a></span> |
| <span class="normal"><a href="#__codelineno-0-1346">1346</a></span> |
| <span class="normal"><a href="#__codelineno-0-1347">1347</a></span> |
| <span class="normal"><a href="#__codelineno-0-1348">1348</a></span> |
| <span class="normal"><a href="#__codelineno-0-1349">1349</a></span> |
| <span class="normal"><a href="#__codelineno-0-1350">1350</a></span> |
| <span class="normal"><a href="#__codelineno-0-1351">1351</a></span> |
| <span class="normal"><a href="#__codelineno-0-1352">1352</a></span> |
| <span class="normal"><a href="#__codelineno-0-1353">1353</a></span> |
| <span class="normal"><a href="#__codelineno-0-1354">1354</a></span> |
| <span class="normal"><a href="#__codelineno-0-1355">1355</a></span> |
| <span class="normal"><a href="#__codelineno-0-1356">1356</a></span> |
| <span class="normal"><a href="#__codelineno-0-1357">1357</a></span> |
| <span class="normal"><a href="#__codelineno-0-1358">1358</a></span> |
| <span class="normal"><a href="#__codelineno-0-1359">1359</a></span> |
| <span class="normal"><a href="#__codelineno-0-1360">1360</a></span> |
| <span class="normal"><a href="#__codelineno-0-1361">1361</a></span> |
| <span class="normal"><a href="#__codelineno-0-1362">1362</a></span> |
| <span class="normal"><a href="#__codelineno-0-1363">1363</a></span> |
| <span class="normal"><a href="#__codelineno-0-1364">1364</a></span> |
| <span class="normal"><a href="#__codelineno-0-1365">1365</a></span> |
| <span class="normal"><a href="#__codelineno-0-1366">1366</a></span> |
| <span class="normal"><a href="#__codelineno-0-1367">1367</a></span> |
| <span class="normal"><a href="#__codelineno-0-1368">1368</a></span> |
| <span class="normal"><a href="#__codelineno-0-1369">1369</a></span> |
| <span class="normal"><a href="#__codelineno-0-1370">1370</a></span> |
| <span class="normal"><a href="#__codelineno-0-1371">1371</a></span> |
| <span class="normal"><a href="#__codelineno-0-1372">1372</a></span> |
| <span class="normal"><a href="#__codelineno-0-1373">1373</a></span> |
| <span class="normal"><a href="#__codelineno-0-1374">1374</a></span> |
| <span class="normal"><a href="#__codelineno-0-1375">1375</a></span> |
| <span class="normal"><a href="#__codelineno-0-1376">1376</a></span> |
| <span class="normal"><a href="#__codelineno-0-1377">1377</a></span> |
| <span class="normal"><a href="#__codelineno-0-1378">1378</a></span> |
| <span class="normal"><a href="#__codelineno-0-1379">1379</a></span> |
| <span class="normal"><a href="#__codelineno-0-1380">1380</a></span> |
| <span class="normal"><a href="#__codelineno-0-1381">1381</a></span> |
| <span class="normal"><a href="#__codelineno-0-1382">1382</a></span> |
| <span class="normal"><a href="#__codelineno-0-1383">1383</a></span> |
| <span class="normal"><a href="#__codelineno-0-1384">1384</a></span> |
| <span class="normal"><a href="#__codelineno-0-1385">1385</a></span> |
| <span class="normal"><a href="#__codelineno-0-1386">1386</a></span> |
| <span class="normal"><a href="#__codelineno-0-1387">1387</a></span> |
| <span class="normal"><a href="#__codelineno-0-1388">1388</a></span> |
| <span class="normal"><a href="#__codelineno-0-1389">1389</a></span> |
| <span class="normal"><a href="#__codelineno-0-1390">1390</a></span> |
| <span class="normal"><a href="#__codelineno-0-1391">1391</a></span> |
| <span class="normal"><a href="#__codelineno-0-1392">1392</a></span> |
| <span class="normal"><a href="#__codelineno-0-1393">1393</a></span> |
| <span class="normal"><a href="#__codelineno-0-1394">1394</a></span> |
| <span class="normal"><a href="#__codelineno-0-1395">1395</a></span> |
| <span class="normal"><a href="#__codelineno-0-1396">1396</a></span> |
| <span class="normal"><a href="#__codelineno-0-1397">1397</a></span> |
| <span class="normal"><a href="#__codelineno-0-1398">1398</a></span> |
| <span class="normal"><a href="#__codelineno-0-1399">1399</a></span> |
| <span class="normal"><a href="#__codelineno-0-1400">1400</a></span> |
| <span class="normal"><a href="#__codelineno-0-1401">1401</a></span> |
| <span class="normal"><a href="#__codelineno-0-1402">1402</a></span> |
| <span class="normal"><a href="#__codelineno-0-1403">1403</a></span> |
| <span class="normal"><a href="#__codelineno-0-1404">1404</a></span> |
| <span class="normal"><a href="#__codelineno-0-1405">1405</a></span> |
| <span class="normal"><a href="#__codelineno-0-1406">1406</a></span> |
| <span class="normal"><a href="#__codelineno-0-1407">1407</a></span> |
| <span class="normal"><a href="#__codelineno-0-1408">1408</a></span> |
| <span class="normal"><a href="#__codelineno-0-1409">1409</a></span> |
| <span class="normal"><a href="#__codelineno-0-1410">1410</a></span> |
| <span class="normal"><a href="#__codelineno-0-1411">1411</a></span> |
| <span class="normal"><a href="#__codelineno-0-1412">1412</a></span> |
| <span class="normal"><a href="#__codelineno-0-1413">1413</a></span> |
| <span class="normal"><a href="#__codelineno-0-1414">1414</a></span> |
| <span class="normal"><a href="#__codelineno-0-1415">1415</a></span> |
| <span class="normal"><a href="#__codelineno-0-1416">1416</a></span> |
| <span class="normal"><a href="#__codelineno-0-1417">1417</a></span> |
| <span class="normal"><a href="#__codelineno-0-1418">1418</a></span> |
| <span class="normal"><a href="#__codelineno-0-1419">1419</a></span> |
| <span class="normal"><a href="#__codelineno-0-1420">1420</a></span> |
| <span class="normal"><a href="#__codelineno-0-1421">1421</a></span> |
| <span class="normal"><a href="#__codelineno-0-1422">1422</a></span> |
| <span class="normal"><a href="#__codelineno-0-1423">1423</a></span> |
| <span class="normal"><a href="#__codelineno-0-1424">1424</a></span> |
| <span class="normal"><a href="#__codelineno-0-1425">1425</a></span> |
| <span class="normal"><a href="#__codelineno-0-1426">1426</a></span> |
| <span class="normal"><a href="#__codelineno-0-1427">1427</a></span> |
| <span class="normal"><a href="#__codelineno-0-1428">1428</a></span> |
| <span class="normal"><a href="#__codelineno-0-1429">1429</a></span> |
| <span class="normal"><a href="#__codelineno-0-1430">1430</a></span> |
| <span class="normal"><a href="#__codelineno-0-1431">1431</a></span> |
| <span class="normal"><a href="#__codelineno-0-1432">1432</a></span> |
| <span class="normal"><a href="#__codelineno-0-1433">1433</a></span> |
| <span class="normal"><a href="#__codelineno-0-1434">1434</a></span> |
| <span class="normal"><a href="#__codelineno-0-1435">1435</a></span> |
| <span class="normal"><a href="#__codelineno-0-1436">1436</a></span> |
| <span class="normal"><a href="#__codelineno-0-1437">1437</a></span> |
| <span class="normal"><a href="#__codelineno-0-1438">1438</a></span> |
| <span class="normal"><a href="#__codelineno-0-1439">1439</a></span> |
| <span class="normal"><a href="#__codelineno-0-1440">1440</a></span> |
| <span class="normal"><a href="#__codelineno-0-1441">1441</a></span> |
| <span class="normal"><a href="#__codelineno-0-1442">1442</a></span> |
| <span class="normal"><a href="#__codelineno-0-1443">1443</a></span> |
| <span class="normal"><a href="#__codelineno-0-1444">1444</a></span> |
| <span class="normal"><a href="#__codelineno-0-1445">1445</a></span> |
| <span class="normal"><a href="#__codelineno-0-1446">1446</a></span> |
| <span class="normal"><a href="#__codelineno-0-1447">1447</a></span> |
| <span class="normal"><a href="#__codelineno-0-1448">1448</a></span> |
| <span class="normal"><a href="#__codelineno-0-1449">1449</a></span> |
| <span class="normal"><a href="#__codelineno-0-1450">1450</a></span> |
| <span class="normal"><a href="#__codelineno-0-1451">1451</a></span> |
| <span class="normal"><a href="#__codelineno-0-1452">1452</a></span> |
| <span class="normal"><a href="#__codelineno-0-1453">1453</a></span> |
| <span class="normal"><a href="#__codelineno-0-1454">1454</a></span> |
| <span class="normal"><a href="#__codelineno-0-1455">1455</a></span> |
| <span class="normal"><a href="#__codelineno-0-1456">1456</a></span> |
| <span class="normal"><a href="#__codelineno-0-1457">1457</a></span> |
| <span class="normal"><a href="#__codelineno-0-1458">1458</a></span> |
| <span class="normal"><a href="#__codelineno-0-1459">1459</a></span> |
| <span class="normal"><a href="#__codelineno-0-1460">1460</a></span> |
| <span class="normal"><a href="#__codelineno-0-1461">1461</a></span> |
| <span class="normal"><a href="#__codelineno-0-1462">1462</a></span> |
| <span class="normal"><a href="#__codelineno-0-1463">1463</a></span> |
| <span class="normal"><a href="#__codelineno-0-1464">1464</a></span> |
| <span class="normal"><a href="#__codelineno-0-1465">1465</a></span> |
| <span class="normal"><a href="#__codelineno-0-1466">1466</a></span> |
| <span class="normal"><a href="#__codelineno-0-1467">1467</a></span> |
| <span class="normal"><a href="#__codelineno-0-1468">1468</a></span> |
| <span class="normal"><a href="#__codelineno-0-1469">1469</a></span> |
| <span class="normal"><a href="#__codelineno-0-1470">1470</a></span> |
| <span class="normal"><a href="#__codelineno-0-1471">1471</a></span> |
| <span class="normal"><a href="#__codelineno-0-1472">1472</a></span> |
| <span class="normal"><a href="#__codelineno-0-1473">1473</a></span> |
| <span class="normal"><a href="#__codelineno-0-1474">1474</a></span> |
| <span class="normal"><a href="#__codelineno-0-1475">1475</a></span> |
| <span class="normal"><a href="#__codelineno-0-1476">1476</a></span> |
| <span class="normal"><a href="#__codelineno-0-1477">1477</a></span> |
| <span class="normal"><a href="#__codelineno-0-1478">1478</a></span> |
| <span class="normal"><a href="#__codelineno-0-1479">1479</a></span> |
| <span class="normal"><a href="#__codelineno-0-1480">1480</a></span> |
| <span class="normal"><a href="#__codelineno-0-1481">1481</a></span> |
| <span class="normal"><a href="#__codelineno-0-1482">1482</a></span> |
| <span class="normal"><a href="#__codelineno-0-1483">1483</a></span> |
| <span class="normal"><a href="#__codelineno-0-1484">1484</a></span> |
| <span class="normal"><a href="#__codelineno-0-1485">1485</a></span> |
| <span class="normal"><a href="#__codelineno-0-1486">1486</a></span> |
| <span class="normal"><a href="#__codelineno-0-1487">1487</a></span> |
| <span class="normal"><a href="#__codelineno-0-1488">1488</a></span> |
| <span class="normal"><a href="#__codelineno-0-1489">1489</a></span> |
| <span class="normal"><a href="#__codelineno-0-1490">1490</a></span> |
| <span class="normal"><a href="#__codelineno-0-1491">1491</a></span> |
| <span class="normal"><a href="#__codelineno-0-1492">1492</a></span> |
| <span class="normal"><a href="#__codelineno-0-1493">1493</a></span> |
| <span class="normal"><a href="#__codelineno-0-1494">1494</a></span> |
| <span class="normal"><a href="#__codelineno-0-1495">1495</a></span> |
| <span class="normal"><a href="#__codelineno-0-1496">1496</a></span> |
| <span class="normal"><a href="#__codelineno-0-1497">1497</a></span> |
| <span class="normal"><a href="#__codelineno-0-1498">1498</a></span> |
| <span class="normal"><a href="#__codelineno-0-1499">1499</a></span> |
| <span class="normal"><a href="#__codelineno-0-1500">1500</a></span> |
| <span class="normal"><a href="#__codelineno-0-1501">1501</a></span> |
| <span class="normal"><a href="#__codelineno-0-1502">1502</a></span> |
| <span class="normal"><a href="#__codelineno-0-1503">1503</a></span> |
| <span class="normal"><a href="#__codelineno-0-1504">1504</a></span> |
| <span class="normal"><a href="#__codelineno-0-1505">1505</a></span> |
| <span class="normal"><a href="#__codelineno-0-1506">1506</a></span> |
| <span class="normal"><a href="#__codelineno-0-1507">1507</a></span> |
| <span class="normal"><a href="#__codelineno-0-1508">1508</a></span> |
| <span class="normal"><a href="#__codelineno-0-1509">1509</a></span> |
| <span class="normal"><a href="#__codelineno-0-1510">1510</a></span> |
| <span class="normal"><a href="#__codelineno-0-1511">1511</a></span> |
| <span class="normal"><a href="#__codelineno-0-1512">1512</a></span> |
| <span class="normal"><a href="#__codelineno-0-1513">1513</a></span> |
| <span class="normal"><a href="#__codelineno-0-1514">1514</a></span> |
| <span class="normal"><a href="#__codelineno-0-1515">1515</a></span> |
| <span class="normal"><a href="#__codelineno-0-1516">1516</a></span> |
| <span class="normal"><a href="#__codelineno-0-1517">1517</a></span> |
| <span class="normal"><a href="#__codelineno-0-1518">1518</a></span> |
| <span class="normal"><a href="#__codelineno-0-1519">1519</a></span> |
| <span class="normal"><a href="#__codelineno-0-1520">1520</a></span> |
| <span class="normal"><a href="#__codelineno-0-1521">1521</a></span> |
| <span class="normal"><a href="#__codelineno-0-1522">1522</a></span> |
| <span class="normal"><a href="#__codelineno-0-1523">1523</a></span> |
| <span class="normal"><a href="#__codelineno-0-1524">1524</a></span> |
| <span class="normal"><a href="#__codelineno-0-1525">1525</a></span> |
| <span class="normal"><a href="#__codelineno-0-1526">1526</a></span> |
| <span class="normal"><a href="#__codelineno-0-1527">1527</a></span> |
| <span class="normal"><a href="#__codelineno-0-1528">1528</a></span> |
| <span class="normal"><a href="#__codelineno-0-1529">1529</a></span> |
| <span class="normal"><a href="#__codelineno-0-1530">1530</a></span> |
| <span class="normal"><a href="#__codelineno-0-1531">1531</a></span> |
| <span class="normal"><a href="#__codelineno-0-1532">1532</a></span> |
| <span class="normal"><a href="#__codelineno-0-1533">1533</a></span> |
| <span class="normal"><a href="#__codelineno-0-1534">1534</a></span> |
| <span class="normal"><a href="#__codelineno-0-1535">1535</a></span> |
| <span class="normal"><a href="#__codelineno-0-1536">1536</a></span> |
| <span class="normal"><a href="#__codelineno-0-1537">1537</a></span> |
| <span class="normal"><a href="#__codelineno-0-1538">1538</a></span> |
| <span class="normal"><a href="#__codelineno-0-1539">1539</a></span> |
| <span class="normal"><a href="#__codelineno-0-1540">1540</a></span> |
| <span class="normal"><a href="#__codelineno-0-1541">1541</a></span> |
| <span class="normal"><a href="#__codelineno-0-1542">1542</a></span> |
| <span class="normal"><a href="#__codelineno-0-1543">1543</a></span> |
| <span class="normal"><a href="#__codelineno-0-1544">1544</a></span> |
| <span class="normal"><a href="#__codelineno-0-1545">1545</a></span> |
| <span class="normal"><a href="#__codelineno-0-1546">1546</a></span> |
| <span class="normal"><a href="#__codelineno-0-1547">1547</a></span> |
| <span class="normal"><a href="#__codelineno-0-1548">1548</a></span> |
| <span class="normal"><a href="#__codelineno-0-1549">1549</a></span> |
| <span class="normal"><a href="#__codelineno-0-1550">1550</a></span> |
| <span class="normal"><a href="#__codelineno-0-1551">1551</a></span> |
| <span class="normal"><a href="#__codelineno-0-1552">1552</a></span> |
| <span class="normal"><a href="#__codelineno-0-1553">1553</a></span> |
| <span class="normal"><a href="#__codelineno-0-1554">1554</a></span> |
| <span class="normal"><a href="#__codelineno-0-1555">1555</a></span> |
| <span class="normal"><a href="#__codelineno-0-1556">1556</a></span> |
| <span class="normal"><a href="#__codelineno-0-1557">1557</a></span> |
| <span class="normal"><a href="#__codelineno-0-1558">1558</a></span> |
| <span class="normal"><a href="#__codelineno-0-1559">1559</a></span> |
| <span class="normal"><a href="#__codelineno-0-1560">1560</a></span> |
| <span class="normal"><a href="#__codelineno-0-1561">1561</a></span> |
| <span class="normal"><a href="#__codelineno-0-1562">1562</a></span> |
| <span class="normal"><a href="#__codelineno-0-1563">1563</a></span> |
| <span class="normal"><a href="#__codelineno-0-1564">1564</a></span> |
| <span class="normal"><a href="#__codelineno-0-1565">1565</a></span> |
| <span class="normal"><a href="#__codelineno-0-1566">1566</a></span> |
| <span class="normal"><a href="#__codelineno-0-1567">1567</a></span> |
| <span class="normal"><a href="#__codelineno-0-1568">1568</a></span> |
| <span class="normal"><a href="#__codelineno-0-1569">1569</a></span> |
| <span class="normal"><a href="#__codelineno-0-1570">1570</a></span> |
| <span class="normal"><a href="#__codelineno-0-1571">1571</a></span> |
| <span class="normal"><a href="#__codelineno-0-1572">1572</a></span> |
| <span class="normal"><a href="#__codelineno-0-1573">1573</a></span> |
| <span class="normal"><a href="#__codelineno-0-1574">1574</a></span> |
| <span class="normal"><a href="#__codelineno-0-1575">1575</a></span> |
| <span class="normal"><a href="#__codelineno-0-1576">1576</a></span> |
| <span class="normal"><a href="#__codelineno-0-1577">1577</a></span> |
| <span class="normal"><a href="#__codelineno-0-1578">1578</a></span> |
| <span class="normal"><a href="#__codelineno-0-1579">1579</a></span> |
| <span class="normal"><a href="#__codelineno-0-1580">1580</a></span> |
| <span class="normal"><a href="#__codelineno-0-1581">1581</a></span> |
| <span class="normal"><a href="#__codelineno-0-1582">1582</a></span> |
| <span class="normal"><a href="#__codelineno-0-1583">1583</a></span> |
| <span class="normal"><a href="#__codelineno-0-1584">1584</a></span> |
| <span class="normal"><a href="#__codelineno-0-1585">1585</a></span> |
| <span class="normal"><a href="#__codelineno-0-1586">1586</a></span> |
| <span class="normal"><a href="#__codelineno-0-1587">1587</a></span> |
| <span class="normal"><a href="#__codelineno-0-1588">1588</a></span> |
| <span class="normal"><a href="#__codelineno-0-1589">1589</a></span> |
| <span class="normal"><a href="#__codelineno-0-1590">1590</a></span> |
| <span class="normal"><a href="#__codelineno-0-1591">1591</a></span> |
| <span class="normal"><a href="#__codelineno-0-1592">1592</a></span> |
| <span class="normal"><a href="#__codelineno-0-1593">1593</a></span> |
| <span class="normal"><a href="#__codelineno-0-1594">1594</a></span> |
| <span class="normal"><a href="#__codelineno-0-1595">1595</a></span> |
| <span class="normal"><a href="#__codelineno-0-1596">1596</a></span> |
| <span class="normal"><a href="#__codelineno-0-1597">1597</a></span> |
| <span class="normal"><a href="#__codelineno-0-1598">1598</a></span> |
| <span class="normal"><a href="#__codelineno-0-1599">1599</a></span> |
| <span class="normal"><a href="#__codelineno-0-1600">1600</a></span> |
| <span class="normal"><a href="#__codelineno-0-1601">1601</a></span> |
| <span class="normal"><a href="#__codelineno-0-1602">1602</a></span> |
| <span class="normal"><a href="#__codelineno-0-1603">1603</a></span> |
| <span class="normal"><a href="#__codelineno-0-1604">1604</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1062" name="__codelineno-0-1062"></a><span class="k">class</span><span class="w"> </span><span class="nc">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1063" name="__codelineno-0-1063"></a><span class="w"> </span><span class="sd">"""An Iceberg table."""</span> |
| <a id="__codelineno-0-1064" name="__codelineno-0-1064"></a> |
| <a id="__codelineno-0-1065" name="__codelineno-0-1065"></a> <span class="n">_identifier</span><span class="p">:</span> <span class="n">Identifier</span> <span class="o">=</span> <span class="n">Field</span><span class="p">()</span> |
| <a id="__codelineno-0-1066" name="__codelineno-0-1066"></a> <span class="n">metadata</span><span class="p">:</span> <span class="n">TableMetadata</span> |
| <a id="__codelineno-0-1067" name="__codelineno-0-1067"></a> <span class="n">metadata_location</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">Field</span><span class="p">()</span> |
| <a id="__codelineno-0-1068" name="__codelineno-0-1068"></a> <span class="n">io</span><span class="p">:</span> <span class="n">FileIO</span> |
| <a id="__codelineno-0-1069" name="__codelineno-0-1069"></a> <span class="n">catalog</span><span class="p">:</span> <span class="n">Catalog</span> |
| <a id="__codelineno-0-1070" name="__codelineno-0-1070"></a> <span class="n">config</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> |
| <a id="__codelineno-0-1071" name="__codelineno-0-1071"></a> |
| <a id="__codelineno-0-1072" name="__codelineno-0-1072"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span> |
| <a id="__codelineno-0-1073" name="__codelineno-0-1073"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1074" name="__codelineno-0-1074"></a> <span class="n">identifier</span><span class="p">:</span> <span class="n">Identifier</span><span class="p">,</span> |
| <a id="__codelineno-0-1075" name="__codelineno-0-1075"></a> <span class="n">metadata</span><span class="p">:</span> <span class="n">TableMetadata</span><span class="p">,</span> |
| <a id="__codelineno-0-1076" name="__codelineno-0-1076"></a> <span class="n">metadata_location</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <a id="__codelineno-0-1077" name="__codelineno-0-1077"></a> <span class="n">io</span><span class="p">:</span> <span class="n">FileIO</span><span class="p">,</span> |
| <a id="__codelineno-0-1078" name="__codelineno-0-1078"></a> <span class="n">catalog</span><span class="p">:</span> <span class="n">Catalog</span><span class="p">,</span> |
| <a id="__codelineno-0-1079" name="__codelineno-0-1079"></a> <span class="n">config</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1080" name="__codelineno-0-1080"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1081" name="__codelineno-0-1081"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span> <span class="o">=</span> <span class="n">identifier</span> |
| <a id="__codelineno-0-1082" name="__codelineno-0-1082"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">metadata</span> |
| <a id="__codelineno-0-1083" name="__codelineno-0-1083"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span> <span class="o">=</span> <span class="n">metadata_location</span> |
| <a id="__codelineno-0-1084" name="__codelineno-0-1084"></a> <span class="bp">self</span><span class="o">.</span><span class="n">io</span> <span class="o">=</span> <span class="n">io</span> |
| <a id="__codelineno-0-1085" name="__codelineno-0-1085"></a> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span> <span class="o">=</span> <span class="n">catalog</span> |
| <a id="__codelineno-0-1086" name="__codelineno-0-1086"></a> <span class="bp">self</span><span class="o">.</span><span class="n">config</span> <span class="o">=</span> <span class="n">config</span> |
| <a id="__codelineno-0-1087" name="__codelineno-0-1087"></a> |
| <a id="__codelineno-0-1088" name="__codelineno-0-1088"></a> <span class="k">def</span><span class="w"> </span><span class="nf">transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-1089" name="__codelineno-0-1089"></a><span class="w"> </span><span class="sd">"""Create a new transaction object to first stage the changes, and then commit them to the catalog.</span> |
| <a id="__codelineno-0-1090" name="__codelineno-0-1090"></a> |
| <a id="__codelineno-0-1091" name="__codelineno-0-1091"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1092" name="__codelineno-0-1092"></a><span class="sd"> The transaction object</span> |
| <a id="__codelineno-0-1093" name="__codelineno-0-1093"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1094" name="__codelineno-0-1094"></a> <span class="k">return</span> <span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-1095" name="__codelineno-0-1095"></a> |
| <a id="__codelineno-0-1096" name="__codelineno-0-1096"></a> <span class="nd">@property</span> |
| <a id="__codelineno-0-1097" name="__codelineno-0-1097"></a> <span class="k">def</span><span class="w"> </span><span class="nf">inspect</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">InspectTable</span><span class="p">:</span> |
| <a id="__codelineno-0-1098" name="__codelineno-0-1098"></a><span class="w"> </span><span class="sd">"""Return the InspectTable object to browse the table metadata.</span> |
| <a id="__codelineno-0-1099" name="__codelineno-0-1099"></a> |
| <a id="__codelineno-0-1100" name="__codelineno-0-1100"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1101" name="__codelineno-0-1101"></a><span class="sd"> InspectTable object based on this Table.</span> |
| <a id="__codelineno-0-1102" name="__codelineno-0-1102"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1103" name="__codelineno-0-1103"></a> <span class="k">return</span> <span class="n">InspectTable</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-1104" name="__codelineno-0-1104"></a> |
| <a id="__codelineno-0-1105" name="__codelineno-0-1105"></a> <span class="nd">@property</span> |
| <a id="__codelineno-0-1106" name="__codelineno-0-1106"></a> <span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">MaintenanceTable</span><span class="p">:</span> |
| <a id="__codelineno-0-1107" name="__codelineno-0-1107"></a><span class="w"> </span><span class="sd">"""Return the MaintenanceTable object for maintenance.</span> |
| <a id="__codelineno-0-1108" name="__codelineno-0-1108"></a> |
| <a id="__codelineno-0-1109" name="__codelineno-0-1109"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1110" name="__codelineno-0-1110"></a><span class="sd"> MaintenanceTable object based on this Table.</span> |
| <a id="__codelineno-0-1111" name="__codelineno-0-1111"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1112" name="__codelineno-0-1112"></a> <span class="k">return</span> <span class="n">MaintenanceTable</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-1113" name="__codelineno-0-1113"></a> |
| <a id="__codelineno-0-1114" name="__codelineno-0-1114"></a> <span class="k">def</span><span class="w"> </span><span class="nf">refresh</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1115" name="__codelineno-0-1115"></a><span class="w"> </span><span class="sd">"""Refresh the current table metadata.</span> |
| <a id="__codelineno-0-1116" name="__codelineno-0-1116"></a> |
| <a id="__codelineno-0-1117" name="__codelineno-0-1117"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1118" name="__codelineno-0-1118"></a><span class="sd"> An updated instance of the same Iceberg table</span> |
| <a id="__codelineno-0-1119" name="__codelineno-0-1119"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1120" name="__codelineno-0-1120"></a> <span class="n">fresh</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span><span class="o">.</span><span class="n">load_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span><span class="p">)</span> |
| <a id="__codelineno-0-1121" name="__codelineno-0-1121"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">fresh</span><span class="o">.</span><span class="n">metadata</span> |
| <a id="__codelineno-0-1122" name="__codelineno-0-1122"></a> <span class="bp">self</span><span class="o">.</span><span class="n">io</span> <span class="o">=</span> <span class="n">fresh</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-1123" name="__codelineno-0-1123"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span> <span class="o">=</span> <span class="n">fresh</span><span class="o">.</span><span class="n">metadata_location</span> |
| <a id="__codelineno-0-1124" name="__codelineno-0-1124"></a> <span class="k">return</span> <span class="bp">self</span> |
| <a id="__codelineno-0-1125" name="__codelineno-0-1125"></a> |
| <a id="__codelineno-0-1126" name="__codelineno-0-1126"></a> <span class="k">def</span><span class="w"> </span><span class="nf">name</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Identifier</span><span class="p">:</span> |
| <a id="__codelineno-0-1127" name="__codelineno-0-1127"></a><span class="w"> </span><span class="sd">"""Return the identifier of this table.</span> |
| <a id="__codelineno-0-1128" name="__codelineno-0-1128"></a> |
| <a id="__codelineno-0-1129" name="__codelineno-0-1129"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1130" name="__codelineno-0-1130"></a><span class="sd"> An Identifier tuple of the table name</span> |
| <a id="__codelineno-0-1131" name="__codelineno-0-1131"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1132" name="__codelineno-0-1132"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span> |
| <a id="__codelineno-0-1133" name="__codelineno-0-1133"></a> |
| <a id="__codelineno-0-1134" name="__codelineno-0-1134"></a> <span class="k">def</span><span class="w"> </span><span class="nf">scan</span><span class="p">(</span> |
| <a id="__codelineno-0-1135" name="__codelineno-0-1135"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1136" name="__codelineno-0-1136"></a> <span class="n">row_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1137" name="__codelineno-0-1137"></a> <span class="n">selected_fields</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"*"</span><span class="p">,),</span> |
| <a id="__codelineno-0-1138" name="__codelineno-0-1138"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1139" name="__codelineno-0-1139"></a> <span class="n">snapshot_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1140" name="__codelineno-0-1140"></a> <span class="n">options</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1141" name="__codelineno-0-1141"></a> <span class="n">limit</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1142" name="__codelineno-0-1142"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">DataScan</span><span class="p">:</span> |
| <a id="__codelineno-0-1143" name="__codelineno-0-1143"></a><span class="w"> </span><span class="sd">"""Fetch a DataScan based on the table's current metadata.</span> |
| <a id="__codelineno-0-1144" name="__codelineno-0-1144"></a> |
| <a id="__codelineno-0-1145" name="__codelineno-0-1145"></a><span class="sd"> The data scan can be used to project the table's data</span> |
| <a id="__codelineno-0-1146" name="__codelineno-0-1146"></a><span class="sd"> that matches the provided row_filter onto the table's</span> |
| <a id="__codelineno-0-1147" name="__codelineno-0-1147"></a><span class="sd"> current schema.</span> |
| <a id="__codelineno-0-1148" name="__codelineno-0-1148"></a> |
| <a id="__codelineno-0-1149" name="__codelineno-0-1149"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1150" name="__codelineno-0-1150"></a><span class="sd"> row_filter:</span> |
| <a id="__codelineno-0-1151" name="__codelineno-0-1151"></a><span class="sd"> A string or BooleanExpression that describes the</span> |
| <a id="__codelineno-0-1152" name="__codelineno-0-1152"></a><span class="sd"> desired rows</span> |
| <a id="__codelineno-0-1153" name="__codelineno-0-1153"></a><span class="sd"> selected_fields:</span> |
| <a id="__codelineno-0-1154" name="__codelineno-0-1154"></a><span class="sd"> A tuple of strings representing the column names</span> |
| <a id="__codelineno-0-1155" name="__codelineno-0-1155"></a><span class="sd"> to return in the output dataframe.</span> |
| <a id="__codelineno-0-1156" name="__codelineno-0-1156"></a><span class="sd"> case_sensitive:</span> |
| <a id="__codelineno-0-1157" name="__codelineno-0-1157"></a><span class="sd"> If True column matching is case sensitive</span> |
| <a id="__codelineno-0-1158" name="__codelineno-0-1158"></a><span class="sd"> snapshot_id:</span> |
| <a id="__codelineno-0-1159" name="__codelineno-0-1159"></a><span class="sd"> Optional Snapshot ID to time travel to. If None,</span> |
| <a id="__codelineno-0-1160" name="__codelineno-0-1160"></a><span class="sd"> scans the table as of the current snapshot ID.</span> |
| <a id="__codelineno-0-1161" name="__codelineno-0-1161"></a><span class="sd"> options:</span> |
| <a id="__codelineno-0-1162" name="__codelineno-0-1162"></a><span class="sd"> Additional Table properties as a dictionary of</span> |
| <a id="__codelineno-0-1163" name="__codelineno-0-1163"></a><span class="sd"> string key value pairs to use for this scan.</span> |
| <a id="__codelineno-0-1164" name="__codelineno-0-1164"></a><span class="sd"> limit:</span> |
| <a id="__codelineno-0-1165" name="__codelineno-0-1165"></a><span class="sd"> An integer representing the number of rows to</span> |
| <a id="__codelineno-0-1166" name="__codelineno-0-1166"></a><span class="sd"> return in the scan result. If None, fetches all</span> |
| <a id="__codelineno-0-1167" name="__codelineno-0-1167"></a><span class="sd"> matching rows.</span> |
| <a id="__codelineno-0-1168" name="__codelineno-0-1168"></a> |
| <a id="__codelineno-0-1169" name="__codelineno-0-1169"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1170" name="__codelineno-0-1170"></a><span class="sd"> A DataScan based on the table's current metadata.</span> |
| <a id="__codelineno-0-1171" name="__codelineno-0-1171"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1172" name="__codelineno-0-1172"></a> <span class="k">return</span> <span class="n">DataScan</span><span class="p">(</span> |
| <a id="__codelineno-0-1173" name="__codelineno-0-1173"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-1174" name="__codelineno-0-1174"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-1175" name="__codelineno-0-1175"></a> <span class="n">row_filter</span><span class="o">=</span><span class="n">row_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-1176" name="__codelineno-0-1176"></a> <span class="n">selected_fields</span><span class="o">=</span><span class="n">selected_fields</span><span class="p">,</span> |
| <a id="__codelineno-0-1177" name="__codelineno-0-1177"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1178" name="__codelineno-0-1178"></a> <span class="n">snapshot_id</span><span class="o">=</span><span class="n">snapshot_id</span><span class="p">,</span> |
| <a id="__codelineno-0-1179" name="__codelineno-0-1179"></a> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> |
| <a id="__codelineno-0-1180" name="__codelineno-0-1180"></a> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">,</span> |
| <a id="__codelineno-0-1181" name="__codelineno-0-1181"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1182" name="__codelineno-0-1182"></a> |
| <a id="__codelineno-0-1183" name="__codelineno-0-1183"></a> <span class="nd">@property</span> |
| <a id="__codelineno-0-1184" name="__codelineno-0-1184"></a> <span class="k">def</span><span class="w"> </span><span class="nf">format_version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">TableVersion</span><span class="p">:</span> |
| <a id="__codelineno-0-1185" name="__codelineno-0-1185"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">format_version</span> |
| <a id="__codelineno-0-1186" name="__codelineno-0-1186"></a> |
| <a id="__codelineno-0-1187" name="__codelineno-0-1187"></a> <span class="k">def</span><span class="w"> </span><span class="nf">schema</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Schema</span><span class="p">:</span> |
| <a id="__codelineno-0-1188" name="__codelineno-0-1188"></a><span class="w"> </span><span class="sd">"""Return the schema for this table."""</span> |
| <a id="__codelineno-0-1189" name="__codelineno-0-1189"></a> <span class="k">return</span> <span class="nb">next</span><span class="p">(</span><span class="n">schema</span> <span class="k">for</span> <span class="n">schema</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">schemas</span> <span class="k">if</span> <span class="n">schema</span><span class="o">.</span><span class="n">schema_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">current_schema_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1190" name="__codelineno-0-1190"></a> |
| <a id="__codelineno-0-1191" name="__codelineno-0-1191"></a> <span class="k">def</span><span class="w"> </span><span class="nf">schemas</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Schema</span><span class="p">]:</span> |
| <a id="__codelineno-0-1192" name="__codelineno-0-1192"></a><span class="w"> </span><span class="sd">"""Return a dict of the schema of this table."""</span> |
| <a id="__codelineno-0-1193" name="__codelineno-0-1193"></a> <span class="k">return</span> <span class="p">{</span><span class="n">schema</span><span class="o">.</span><span class="n">schema_id</span><span class="p">:</span> <span class="n">schema</span> <span class="k">for</span> <span class="n">schema</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">schemas</span><span class="p">}</span> |
| <a id="__codelineno-0-1194" name="__codelineno-0-1194"></a> |
| <a id="__codelineno-0-1195" name="__codelineno-0-1195"></a> <span class="k">def</span><span class="w"> </span><span class="nf">spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">PartitionSpec</span><span class="p">:</span> |
| <a id="__codelineno-0-1196" name="__codelineno-0-1196"></a><span class="w"> </span><span class="sd">"""Return the partition spec of this table."""</span> |
| <a id="__codelineno-0-1197" name="__codelineno-0-1197"></a> <span class="k">return</span> <span class="nb">next</span><span class="p">(</span><span class="n">spec</span> <span class="k">for</span> <span class="n">spec</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">partition_specs</span> <span class="k">if</span> <span class="n">spec</span><span class="o">.</span><span class="n">spec_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">default_spec_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1198" name="__codelineno-0-1198"></a> |
| <a id="__codelineno-0-1199" name="__codelineno-0-1199"></a> <span class="k">def</span><span class="w"> </span><span class="nf">specs</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">PartitionSpec</span><span class="p">]:</span> |
| <a id="__codelineno-0-1200" name="__codelineno-0-1200"></a><span class="w"> </span><span class="sd">"""Return a dict the partition specs this table."""</span> |
| <a id="__codelineno-0-1201" name="__codelineno-0-1201"></a> <span class="k">return</span> <span class="p">{</span><span class="n">spec</span><span class="o">.</span><span class="n">spec_id</span><span class="p">:</span> <span class="n">spec</span> <span class="k">for</span> <span class="n">spec</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">partition_specs</span><span class="p">}</span> |
| <a id="__codelineno-0-1202" name="__codelineno-0-1202"></a> |
| <a id="__codelineno-0-1203" name="__codelineno-0-1203"></a> <span class="k">def</span><span class="w"> </span><span class="nf">sort_order</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">SortOrder</span><span class="p">:</span> |
| <a id="__codelineno-0-1204" name="__codelineno-0-1204"></a><span class="w"> </span><span class="sd">"""Return the sort order of this table."""</span> |
| <a id="__codelineno-0-1205" name="__codelineno-0-1205"></a> <span class="k">return</span> <span class="nb">next</span><span class="p">(</span> |
| <a id="__codelineno-0-1206" name="__codelineno-0-1206"></a> <span class="n">sort_order</span> <span class="k">for</span> <span class="n">sort_order</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">sort_orders</span> <span class="k">if</span> <span class="n">sort_order</span><span class="o">.</span><span class="n">order_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">default_sort_order_id</span> |
| <a id="__codelineno-0-1207" name="__codelineno-0-1207"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1208" name="__codelineno-0-1208"></a> |
| <a id="__codelineno-0-1209" name="__codelineno-0-1209"></a> <span class="k">def</span><span class="w"> </span><span class="nf">sort_orders</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">SortOrder</span><span class="p">]:</span> |
| <a id="__codelineno-0-1210" name="__codelineno-0-1210"></a><span class="w"> </span><span class="sd">"""Return a dict of the sort orders of this table."""</span> |
| <a id="__codelineno-0-1211" name="__codelineno-0-1211"></a> <span class="k">return</span> <span class="p">{</span><span class="n">sort_order</span><span class="o">.</span><span class="n">order_id</span><span class="p">:</span> <span class="n">sort_order</span> <span class="k">for</span> <span class="n">sort_order</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">sort_orders</span><span class="p">}</span> |
| <a id="__codelineno-0-1212" name="__codelineno-0-1212"></a> |
| <a id="__codelineno-0-1213" name="__codelineno-0-1213"></a> <span class="k">def</span><span class="w"> </span><span class="nf">last_partition_id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> |
| <a id="__codelineno-0-1214" name="__codelineno-0-1214"></a><span class="w"> </span><span class="sd">"""Return the highest assigned partition field ID across all specs or 999 if only the unpartitioned spec exists."""</span> |
| <a id="__codelineno-0-1215" name="__codelineno-0-1215"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">last_partition_id</span><span class="p">:</span> |
| <a id="__codelineno-0-1216" name="__codelineno-0-1216"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">last_partition_id</span> |
| <a id="__codelineno-0-1217" name="__codelineno-0-1217"></a> <span class="k">return</span> <span class="n">PARTITION_FIELD_ID_START</span> <span class="o">-</span> <span class="mi">1</span> |
| <a id="__codelineno-0-1218" name="__codelineno-0-1218"></a> |
| <a id="__codelineno-0-1219" name="__codelineno-0-1219"></a> <span class="nd">@property</span> |
| <a id="__codelineno-0-1220" name="__codelineno-0-1220"></a> <span class="k">def</span><span class="w"> </span><span class="nf">properties</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span> |
| <a id="__codelineno-0-1221" name="__codelineno-0-1221"></a><span class="w"> </span><span class="sd">"""Properties of the table."""</span> |
| <a id="__codelineno-0-1222" name="__codelineno-0-1222"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">properties</span> |
| <a id="__codelineno-0-1223" name="__codelineno-0-1223"></a> |
| <a id="__codelineno-0-1224" name="__codelineno-0-1224"></a> <span class="k">def</span><span class="w"> </span><span class="nf">location</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <a id="__codelineno-0-1225" name="__codelineno-0-1225"></a><span class="w"> </span><span class="sd">"""Return the table's base location."""</span> |
| <a id="__codelineno-0-1226" name="__codelineno-0-1226"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">location</span> |
| <a id="__codelineno-0-1227" name="__codelineno-0-1227"></a> |
| <a id="__codelineno-0-1228" name="__codelineno-0-1228"></a> <span class="k">def</span><span class="w"> </span><span class="nf">location_provider</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">LocationProvider</span><span class="p">:</span> |
| <a id="__codelineno-0-1229" name="__codelineno-0-1229"></a><span class="w"> </span><span class="sd">"""Return the table's location provider."""</span> |
| <a id="__codelineno-0-1230" name="__codelineno-0-1230"></a> <span class="k">return</span> <span class="n">load_location_provider</span><span class="p">(</span><span class="n">table_location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">location</span><span class="p">,</span> <span class="n">table_properties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">properties</span><span class="p">)</span> |
| <a id="__codelineno-0-1231" name="__codelineno-0-1231"></a> |
| <a id="__codelineno-0-1232" name="__codelineno-0-1232"></a> <span class="nd">@property</span> |
| <a id="__codelineno-0-1233" name="__codelineno-0-1233"></a> <span class="k">def</span><span class="w"> </span><span class="nf">last_sequence_number</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> |
| <a id="__codelineno-0-1234" name="__codelineno-0-1234"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">last_sequence_number</span> |
| <a id="__codelineno-0-1235" name="__codelineno-0-1235"></a> |
| <a id="__codelineno-0-1236" name="__codelineno-0-1236"></a> <span class="k">def</span><span class="w"> </span><span class="nf">current_snapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1237" name="__codelineno-0-1237"></a><span class="w"> </span><span class="sd">"""Get the current snapshot for this table, or None if there is no current snapshot."""</span> |
| <a id="__codelineno-0-1238" name="__codelineno-0-1238"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">current_snapshot_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1239" name="__codelineno-0-1239"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">current_snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1240" name="__codelineno-0-1240"></a> <span class="k">return</span> <span class="kc">None</span> |
| <a id="__codelineno-0-1241" name="__codelineno-0-1241"></a> |
| <a id="__codelineno-0-1242" name="__codelineno-0-1242"></a> <span class="k">def</span><span class="w"> </span><span class="nf">snapshots</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1243" name="__codelineno-0-1243"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">snapshots</span> |
| <a id="__codelineno-0-1244" name="__codelineno-0-1244"></a> |
| <a id="__codelineno-0-1245" name="__codelineno-0-1245"></a> <span class="k">def</span><span class="w"> </span><span class="nf">snapshot_by_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">snapshot_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1246" name="__codelineno-0-1246"></a><span class="w"> </span><span class="sd">"""Get the snapshot of this table with the given id, or None if there is no matching snapshot."""</span> |
| <a id="__codelineno-0-1247" name="__codelineno-0-1247"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1248" name="__codelineno-0-1248"></a> |
| <a id="__codelineno-0-1249" name="__codelineno-0-1249"></a> <span class="k">def</span><span class="w"> </span><span class="nf">snapshot_by_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1250" name="__codelineno-0-1250"></a><span class="w"> </span><span class="sd">"""Return the snapshot referenced by the given name or null if no such reference exists."""</span> |
| <a id="__codelineno-0-1251" name="__codelineno-0-1251"></a> <span class="k">if</span> <span class="n">ref</span> <span class="o">:=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">refs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">):</span> |
| <a id="__codelineno-0-1252" name="__codelineno-0-1252"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">ref</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1253" name="__codelineno-0-1253"></a> <span class="k">return</span> <span class="kc">None</span> |
| <a id="__codelineno-0-1254" name="__codelineno-0-1254"></a> |
| <a id="__codelineno-0-1255" name="__codelineno-0-1255"></a> <span class="k">def</span><span class="w"> </span><span class="nf">snapshot_as_of_timestamp</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">timestamp_ms</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inclusive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1256" name="__codelineno-0-1256"></a><span class="w"> </span><span class="sd">"""Get the snapshot that was current as of or right before the given timestamp, or None if there is no matching snapshot.</span> |
| <a id="__codelineno-0-1257" name="__codelineno-0-1257"></a> |
| <a id="__codelineno-0-1258" name="__codelineno-0-1258"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1259" name="__codelineno-0-1259"></a><span class="sd"> timestamp_ms: Find snapshot that was current at/before this timestamp</span> |
| <a id="__codelineno-0-1260" name="__codelineno-0-1260"></a><span class="sd"> inclusive: Includes timestamp_ms in search when True. Excludes timestamp_ms when False</span> |
| <a id="__codelineno-0-1261" name="__codelineno-0-1261"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1262" name="__codelineno-0-1262"></a> <span class="k">for</span> <span class="n">log_entry</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">history</span><span class="p">()):</span> |
| <a id="__codelineno-0-1263" name="__codelineno-0-1263"></a> <span class="k">if</span> <span class="p">(</span><span class="n">inclusive</span> <span class="ow">and</span> <span class="n">log_entry</span><span class="o">.</span><span class="n">timestamp_ms</span> <span class="o"><=</span> <span class="n">timestamp_ms</span><span class="p">)</span> <span class="ow">or</span> <span class="n">log_entry</span><span class="o">.</span><span class="n">timestamp_ms</span> <span class="o"><</span> <span class="n">timestamp_ms</span><span class="p">:</span> |
| <a id="__codelineno-0-1264" name="__codelineno-0-1264"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">log_entry</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1265" name="__codelineno-0-1265"></a> <span class="k">return</span> <span class="kc">None</span> |
| <a id="__codelineno-0-1266" name="__codelineno-0-1266"></a> |
| <a id="__codelineno-0-1267" name="__codelineno-0-1267"></a> <span class="k">def</span><span class="w"> </span><span class="nf">history</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">SnapshotLogEntry</span><span class="p">]:</span> |
| <a id="__codelineno-0-1268" name="__codelineno-0-1268"></a><span class="w"> </span><span class="sd">"""Get the snapshot history of this table."""</span> |
| <a id="__codelineno-0-1269" name="__codelineno-0-1269"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">snapshot_log</span> |
| <a id="__codelineno-0-1270" name="__codelineno-0-1270"></a> |
| <a id="__codelineno-0-1271" name="__codelineno-0-1271"></a> <span class="k">def</span><span class="w"> </span><span class="nf">manage_snapshots</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ManageSnapshots</span><span class="p">:</span> |
| <a id="__codelineno-0-1272" name="__codelineno-0-1272"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1273" name="__codelineno-0-1273"></a><span class="sd"> Shorthand to run snapshot management operations like create branch, create tag, etc.</span> |
| <a id="__codelineno-0-1274" name="__codelineno-0-1274"></a> |
| <a id="__codelineno-0-1275" name="__codelineno-0-1275"></a><span class="sd"> Use table.manage_snapshots().<operation>().commit() to run a specific operation.</span> |
| <a id="__codelineno-0-1276" name="__codelineno-0-1276"></a><span class="sd"> Use table.manage_snapshots().<operation-one>().<operation-two>().commit() to run multiple operations.</span> |
| <a id="__codelineno-0-1277" name="__codelineno-0-1277"></a><span class="sd"> Pending changes are applied on commit.</span> |
| <a id="__codelineno-0-1278" name="__codelineno-0-1278"></a> |
| <a id="__codelineno-0-1279" name="__codelineno-0-1279"></a><span class="sd"> We can also use context managers to make more changes. For example,</span> |
| <a id="__codelineno-0-1280" name="__codelineno-0-1280"></a> |
| <a id="__codelineno-0-1281" name="__codelineno-0-1281"></a><span class="sd"> with table.manage_snapshots() as ms:</span> |
| <a id="__codelineno-0-1282" name="__codelineno-0-1282"></a><span class="sd"> ms.create_tag(snapshot_id1, "Tag_A").create_tag(snapshot_id2, "Tag_B")</span> |
| <a id="__codelineno-0-1283" name="__codelineno-0-1283"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1284" name="__codelineno-0-1284"></a> <span class="k">return</span> <span class="n">ManageSnapshots</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> |
| <a id="__codelineno-0-1285" name="__codelineno-0-1285"></a> |
| <a id="__codelineno-0-1286" name="__codelineno-0-1286"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_statistics</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateStatistics</span><span class="p">:</span> |
| <a id="__codelineno-0-1287" name="__codelineno-0-1287"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1288" name="__codelineno-0-1288"></a><span class="sd"> Shorthand to run statistics management operations like add statistics and remove statistics.</span> |
| <a id="__codelineno-0-1289" name="__codelineno-0-1289"></a> |
| <a id="__codelineno-0-1290" name="__codelineno-0-1290"></a><span class="sd"> Use table.update_statistics().<operation>().commit() to run a specific operation.</span> |
| <a id="__codelineno-0-1291" name="__codelineno-0-1291"></a><span class="sd"> Use table.update_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.</span> |
| <a id="__codelineno-0-1292" name="__codelineno-0-1292"></a> |
| <a id="__codelineno-0-1293" name="__codelineno-0-1293"></a><span class="sd"> Pending changes are applied on commit.</span> |
| <a id="__codelineno-0-1294" name="__codelineno-0-1294"></a> |
| <a id="__codelineno-0-1295" name="__codelineno-0-1295"></a><span class="sd"> We can also use context managers to make more changes. For example:</span> |
| <a id="__codelineno-0-1296" name="__codelineno-0-1296"></a> |
| <a id="__codelineno-0-1297" name="__codelineno-0-1297"></a><span class="sd"> with table.update_statistics() as update:</span> |
| <a id="__codelineno-0-1298" name="__codelineno-0-1298"></a><span class="sd"> update.set_statistics(statistics_file=statistics_file)</span> |
| <a id="__codelineno-0-1299" name="__codelineno-0-1299"></a><span class="sd"> update.remove_statistics(snapshot_id=2)</span> |
| <a id="__codelineno-0-1300" name="__codelineno-0-1300"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1301" name="__codelineno-0-1301"></a> <span class="k">return</span> <span class="n">UpdateStatistics</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> |
| <a id="__codelineno-0-1302" name="__codelineno-0-1302"></a> |
| <a id="__codelineno-0-1303" name="__codelineno-0-1303"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">allow_incompatible_changes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSchema</span><span class="p">:</span> |
| <a id="__codelineno-0-1304" name="__codelineno-0-1304"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSchema to alter the columns of this table.</span> |
| <a id="__codelineno-0-1305" name="__codelineno-0-1305"></a> |
| <a id="__codelineno-0-1306" name="__codelineno-0-1306"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1307" name="__codelineno-0-1307"></a><span class="sd"> allow_incompatible_changes: If changes are allowed that might break downstream consumers.</span> |
| <a id="__codelineno-0-1308" name="__codelineno-0-1308"></a><span class="sd"> case_sensitive: If field names are case-sensitive.</span> |
| <a id="__codelineno-0-1309" name="__codelineno-0-1309"></a> |
| <a id="__codelineno-0-1310" name="__codelineno-0-1310"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1311" name="__codelineno-0-1311"></a><span class="sd"> A new UpdateSchema.</span> |
| <a id="__codelineno-0-1312" name="__codelineno-0-1312"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1313" name="__codelineno-0-1313"></a> <span class="k">return</span> <span class="n">UpdateSchema</span><span class="p">(</span> |
| <a id="__codelineno-0-1314" name="__codelineno-0-1314"></a> <span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> |
| <a id="__codelineno-0-1315" name="__codelineno-0-1315"></a> <span class="n">allow_incompatible_changes</span><span class="o">=</span><span class="n">allow_incompatible_changes</span><span class="p">,</span> |
| <a id="__codelineno-0-1316" name="__codelineno-0-1316"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1317" name="__codelineno-0-1317"></a> <span class="n">name_mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">(),</span> |
| <a id="__codelineno-0-1318" name="__codelineno-0-1318"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1319" name="__codelineno-0-1319"></a> |
| <a id="__codelineno-0-1320" name="__codelineno-0-1320"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_sort_order</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSortOrder</span><span class="p">:</span> |
| <a id="__codelineno-0-1321" name="__codelineno-0-1321"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSortOrder to update the sort order of this table.</span> |
| <a id="__codelineno-0-1322" name="__codelineno-0-1322"></a> |
| <a id="__codelineno-0-1323" name="__codelineno-0-1323"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1324" name="__codelineno-0-1324"></a><span class="sd"> A new UpdateSortOrder.</span> |
| <a id="__codelineno-0-1325" name="__codelineno-0-1325"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1326" name="__codelineno-0-1326"></a> <span class="k">return</span> <span class="n">UpdateSortOrder</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-1327" name="__codelineno-0-1327"></a> |
| <a id="__codelineno-0-1328" name="__codelineno-0-1328"></a> <span class="k">def</span><span class="w"> </span><span class="nf">name_mapping</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">NameMapping</span><span class="p">]:</span> |
| <a id="__codelineno-0-1329" name="__codelineno-0-1329"></a><span class="w"> </span><span class="sd">"""Return the table's field-id NameMapping."""</span> |
| <a id="__codelineno-0-1330" name="__codelineno-0-1330"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">()</span> |
| <a id="__codelineno-0-1331" name="__codelineno-0-1331"></a> |
| <a id="__codelineno-0-1332" name="__codelineno-0-1332"></a> <span class="k">def</span><span class="w"> </span><span class="nf">upsert</span><span class="p">(</span> |
| <a id="__codelineno-0-1333" name="__codelineno-0-1333"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1334" name="__codelineno-0-1334"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-1335" name="__codelineno-0-1335"></a> <span class="n">join_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1336" name="__codelineno-0-1336"></a> <span class="n">when_matched_update_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1337" name="__codelineno-0-1337"></a> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1338" name="__codelineno-0-1338"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1339" name="__codelineno-0-1339"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1340" name="__codelineno-0-1340"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">UpsertResult</span><span class="p">:</span> |
| <a id="__codelineno-0-1341" name="__codelineno-0-1341"></a><span class="w"> </span><span class="sd">"""Shorthand API for performing an upsert to an iceberg table.</span> |
| <a id="__codelineno-0-1342" name="__codelineno-0-1342"></a> |
| <a id="__codelineno-0-1343" name="__codelineno-0-1343"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1344" name="__codelineno-0-1344"></a> |
| <a id="__codelineno-0-1345" name="__codelineno-0-1345"></a><span class="sd"> df: The input dataframe to upsert with the table's data.</span> |
| <a id="__codelineno-0-1346" name="__codelineno-0-1346"></a><span class="sd"> join_cols: Columns to join on, if not provided, it will use the identifier-field-ids.</span> |
| <a id="__codelineno-0-1347" name="__codelineno-0-1347"></a><span class="sd"> when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing</span> |
| <a id="__codelineno-0-1348" name="__codelineno-0-1348"></a><span class="sd"> when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any existing rows in the table</span> |
| <a id="__codelineno-0-1349" name="__codelineno-0-1349"></a><span class="sd"> case_sensitive: Bool indicating if the match should be case-sensitive</span> |
| <a id="__codelineno-0-1350" name="__codelineno-0-1350"></a><span class="sd"> branch: Branch Reference to run the upsert operation</span> |
| <a id="__codelineno-0-1351" name="__codelineno-0-1351"></a> |
| <a id="__codelineno-0-1352" name="__codelineno-0-1352"></a><span class="sd"> To learn more about the identifier-field-ids: https://iceberg.apache.org/spec/#identifier-field-ids</span> |
| <a id="__codelineno-0-1353" name="__codelineno-0-1353"></a> |
| <a id="__codelineno-0-1354" name="__codelineno-0-1354"></a><span class="sd"> Example Use Cases:</span> |
| <a id="__codelineno-0-1355" name="__codelineno-0-1355"></a><span class="sd"> Case 1: Both Parameters = True (Full Upsert)</span> |
| <a id="__codelineno-0-1356" name="__codelineno-0-1356"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-1357" name="__codelineno-0-1357"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-1358" name="__codelineno-0-1358"></a> |
| <a id="__codelineno-0-1359" name="__codelineno-0-1359"></a><span class="sd"> Case 2: when_matched_update_all = False, when_not_matched_insert_all = True</span> |
| <a id="__codelineno-0-1360" name="__codelineno-0-1360"></a><span class="sd"> Existing row found → Do nothing (no updates)</span> |
| <a id="__codelineno-0-1361" name="__codelineno-0-1361"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-1362" name="__codelineno-0-1362"></a> |
| <a id="__codelineno-0-1363" name="__codelineno-0-1363"></a><span class="sd"> Case 3: when_matched_update_all = True, when_not_matched_insert_all = False</span> |
| <a id="__codelineno-0-1364" name="__codelineno-0-1364"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-1365" name="__codelineno-0-1365"></a><span class="sd"> New row found → Do nothing (no inserts)</span> |
| <a id="__codelineno-0-1366" name="__codelineno-0-1366"></a> |
| <a id="__codelineno-0-1367" name="__codelineno-0-1367"></a><span class="sd"> Case 4: Both Parameters = False (No Merge Effect)</span> |
| <a id="__codelineno-0-1368" name="__codelineno-0-1368"></a><span class="sd"> Existing row found → Do nothing</span> |
| <a id="__codelineno-0-1369" name="__codelineno-0-1369"></a><span class="sd"> New row found → Do nothing</span> |
| <a id="__codelineno-0-1370" name="__codelineno-0-1370"></a><span class="sd"> (Function effectively does nothing)</span> |
| <a id="__codelineno-0-1371" name="__codelineno-0-1371"></a> |
| <a id="__codelineno-0-1372" name="__codelineno-0-1372"></a> |
| <a id="__codelineno-0-1373" name="__codelineno-0-1373"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1374" name="__codelineno-0-1374"></a><span class="sd"> An UpsertResult class (contains details of rows updated and inserted)</span> |
| <a id="__codelineno-0-1375" name="__codelineno-0-1375"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1376" name="__codelineno-0-1376"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1377" name="__codelineno-0-1377"></a> <span class="k">return</span> <span class="n">tx</span><span class="o">.</span><span class="n">upsert</span><span class="p">(</span> |
| <a id="__codelineno-0-1378" name="__codelineno-0-1378"></a> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> |
| <a id="__codelineno-0-1379" name="__codelineno-0-1379"></a> <span class="n">join_cols</span><span class="o">=</span><span class="n">join_cols</span><span class="p">,</span> |
| <a id="__codelineno-0-1380" name="__codelineno-0-1380"></a> <span class="n">when_matched_update_all</span><span class="o">=</span><span class="n">when_matched_update_all</span><span class="p">,</span> |
| <a id="__codelineno-0-1381" name="__codelineno-0-1381"></a> <span class="n">when_not_matched_insert_all</span><span class="o">=</span><span class="n">when_not_matched_insert_all</span><span class="p">,</span> |
| <a id="__codelineno-0-1382" name="__codelineno-0-1382"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1383" name="__codelineno-0-1383"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-1384" name="__codelineno-0-1384"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1385" name="__codelineno-0-1385"></a> |
| <a id="__codelineno-0-1386" name="__codelineno-0-1386"></a> <span class="k">def</span><span class="w"> </span><span class="nf">append</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1387" name="__codelineno-0-1387"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1388" name="__codelineno-0-1388"></a><span class="sd"> Shorthand API for appending a PyArrow table to the table.</span> |
| <a id="__codelineno-0-1389" name="__codelineno-0-1389"></a> |
| <a id="__codelineno-0-1390" name="__codelineno-0-1390"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1391" name="__codelineno-0-1391"></a><span class="sd"> df: The Arrow dataframe that will be appended to overwrite the table</span> |
| <a id="__codelineno-0-1392" name="__codelineno-0-1392"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1393" name="__codelineno-0-1393"></a><span class="sd"> branch: Branch Reference to run the append operation</span> |
| <a id="__codelineno-0-1394" name="__codelineno-0-1394"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1395" name="__codelineno-0-1395"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1396" name="__codelineno-0-1396"></a> <span class="n">tx</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-1397" name="__codelineno-0-1397"></a> |
| <a id="__codelineno-0-1398" name="__codelineno-0-1398"></a> <span class="k">def</span><span class="w"> </span><span class="nf">dynamic_partition_overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-1399" name="__codelineno-0-1399"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-1400" name="__codelineno-0-1400"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1401" name="__codelineno-0-1401"></a><span class="w"> </span><span class="sd">"""Shorthand for dynamic overwriting the table with a PyArrow table.</span> |
| <a id="__codelineno-0-1402" name="__codelineno-0-1402"></a> |
| <a id="__codelineno-0-1403" name="__codelineno-0-1403"></a><span class="sd"> Old partitions are auto detected and replaced with data files created for input arrow table.</span> |
| <a id="__codelineno-0-1404" name="__codelineno-0-1404"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1405" name="__codelineno-0-1405"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-1406" name="__codelineno-0-1406"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1407" name="__codelineno-0-1407"></a><span class="sd"> branch: Branch Reference to run the dynamic partition overwrite operation</span> |
| <a id="__codelineno-0-1408" name="__codelineno-0-1408"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1409" name="__codelineno-0-1409"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1410" name="__codelineno-0-1410"></a> <span class="n">tx</span><span class="o">.</span><span class="n">dynamic_partition_overwrite</span><span class="p">(</span><span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-1411" name="__codelineno-0-1411"></a> |
| <a id="__codelineno-0-1412" name="__codelineno-0-1412"></a> <span class="k">def</span><span class="w"> </span><span class="nf">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-1413" name="__codelineno-0-1413"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1414" name="__codelineno-0-1414"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-1415" name="__codelineno-0-1415"></a> <span class="n">overwrite_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BooleanExpression</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1416" name="__codelineno-0-1416"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1417" name="__codelineno-0-1417"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1418" name="__codelineno-0-1418"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1419" name="__codelineno-0-1419"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1420" name="__codelineno-0-1420"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1421" name="__codelineno-0-1421"></a><span class="sd"> Shorthand for overwriting the table with a PyArrow table.</span> |
| <a id="__codelineno-0-1422" name="__codelineno-0-1422"></a> |
| <a id="__codelineno-0-1423" name="__codelineno-0-1423"></a><span class="sd"> An overwrite may produce zero or more snapshots based on the operation:</span> |
| <a id="__codelineno-0-1424" name="__codelineno-0-1424"></a> |
| <a id="__codelineno-0-1425" name="__codelineno-0-1425"></a><span class="sd"> - DELETE: In case existing Parquet files can be dropped completely.</span> |
| <a id="__codelineno-0-1426" name="__codelineno-0-1426"></a><span class="sd"> - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the overwrite filter..</span> |
| <a id="__codelineno-0-1427" name="__codelineno-0-1427"></a><span class="sd"> - APPEND: In case new data is being inserted into the table.</span> |
| <a id="__codelineno-0-1428" name="__codelineno-0-1428"></a> |
| <a id="__codelineno-0-1429" name="__codelineno-0-1429"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1430" name="__codelineno-0-1430"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-1431" name="__codelineno-0-1431"></a><span class="sd"> overwrite_filter: ALWAYS_TRUE when you overwrite all the data,</span> |
| <a id="__codelineno-0-1432" name="__codelineno-0-1432"></a><span class="sd"> or a boolean expression in case of a partial overwrite</span> |
| <a id="__codelineno-0-1433" name="__codelineno-0-1433"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1434" name="__codelineno-0-1434"></a><span class="sd"> case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive</span> |
| <a id="__codelineno-0-1435" name="__codelineno-0-1435"></a><span class="sd"> branch: Branch Reference to run the overwrite operation</span> |
| <a id="__codelineno-0-1436" name="__codelineno-0-1436"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1437" name="__codelineno-0-1437"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1438" name="__codelineno-0-1438"></a> <span class="n">tx</span><span class="o">.</span><span class="n">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-1439" name="__codelineno-0-1439"></a> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> |
| <a id="__codelineno-0-1440" name="__codelineno-0-1440"></a> <span class="n">overwrite_filter</span><span class="o">=</span><span class="n">overwrite_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-1441" name="__codelineno-0-1441"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1442" name="__codelineno-0-1442"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> |
| <a id="__codelineno-0-1443" name="__codelineno-0-1443"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-1444" name="__codelineno-0-1444"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1445" name="__codelineno-0-1445"></a> |
| <a id="__codelineno-0-1446" name="__codelineno-0-1446"></a> <span class="k">def</span><span class="w"> </span><span class="nf">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-1447" name="__codelineno-0-1447"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1448" name="__codelineno-0-1448"></a> <span class="n">delete_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BooleanExpression</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1449" name="__codelineno-0-1449"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1450" name="__codelineno-0-1450"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1451" name="__codelineno-0-1451"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1452" name="__codelineno-0-1452"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1453" name="__codelineno-0-1453"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1454" name="__codelineno-0-1454"></a><span class="sd"> Shorthand for deleting rows from the table.</span> |
| <a id="__codelineno-0-1455" name="__codelineno-0-1455"></a> |
| <a id="__codelineno-0-1456" name="__codelineno-0-1456"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1457" name="__codelineno-0-1457"></a><span class="sd"> delete_filter: The predicate that used to remove rows</span> |
| <a id="__codelineno-0-1458" name="__codelineno-0-1458"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1459" name="__codelineno-0-1459"></a><span class="sd"> case_sensitive: A bool determine if the provided `delete_filter` is case-sensitive</span> |
| <a id="__codelineno-0-1460" name="__codelineno-0-1460"></a><span class="sd"> branch: Branch Reference to run the delete operation</span> |
| <a id="__codelineno-0-1461" name="__codelineno-0-1461"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1462" name="__codelineno-0-1462"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1463" name="__codelineno-0-1463"></a> <span class="n">tx</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-1464" name="__codelineno-0-1464"></a> <span class="n">delete_filter</span><span class="o">=</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span> |
| <a id="__codelineno-0-1465" name="__codelineno-0-1465"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1466" name="__codelineno-0-1466"></a> |
| <a id="__codelineno-0-1467" name="__codelineno-0-1467"></a> <span class="k">def</span><span class="w"> </span><span class="nf">add_files</span><span class="p">(</span> |
| <a id="__codelineno-0-1468" name="__codelineno-0-1468"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1469" name="__codelineno-0-1469"></a> <span class="n">file_paths</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <a id="__codelineno-0-1470" name="__codelineno-0-1470"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1471" name="__codelineno-0-1471"></a> <span class="n">check_duplicate_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1472" name="__codelineno-0-1472"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1473" name="__codelineno-0-1473"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1474" name="__codelineno-0-1474"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1475" name="__codelineno-0-1475"></a><span class="sd"> Shorthand API for adding files as data files to the table.</span> |
| <a id="__codelineno-0-1476" name="__codelineno-0-1476"></a> |
| <a id="__codelineno-0-1477" name="__codelineno-0-1477"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1478" name="__codelineno-0-1478"></a><span class="sd"> file_paths: The list of full file paths to be added as data files to the table</span> |
| <a id="__codelineno-0-1479" name="__codelineno-0-1479"></a> |
| <a id="__codelineno-0-1480" name="__codelineno-0-1480"></a><span class="sd"> Raises:</span> |
| <a id="__codelineno-0-1481" name="__codelineno-0-1481"></a><span class="sd"> FileNotFoundError: If the file does not exist.</span> |
| <a id="__codelineno-0-1482" name="__codelineno-0-1482"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1483" name="__codelineno-0-1483"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1484" name="__codelineno-0-1484"></a> <span class="n">tx</span><span class="o">.</span><span class="n">add_files</span><span class="p">(</span> |
| <a id="__codelineno-0-1485" name="__codelineno-0-1485"></a> <span class="n">file_paths</span><span class="o">=</span><span class="n">file_paths</span><span class="p">,</span> |
| <a id="__codelineno-0-1486" name="__codelineno-0-1486"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> |
| <a id="__codelineno-0-1487" name="__codelineno-0-1487"></a> <span class="n">check_duplicate_files</span><span class="o">=</span><span class="n">check_duplicate_files</span><span class="p">,</span> |
| <a id="__codelineno-0-1488" name="__codelineno-0-1488"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-1489" name="__codelineno-0-1489"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1490" name="__codelineno-0-1490"></a> |
| <a id="__codelineno-0-1491" name="__codelineno-0-1491"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSpec</span><span class="p">:</span> |
| <a id="__codelineno-0-1492" name="__codelineno-0-1492"></a> <span class="k">return</span> <span class="n">UpdateSpec</span><span class="p">(</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-1493" name="__codelineno-0-1493"></a> |
| <a id="__codelineno-0-1494" name="__codelineno-0-1494"></a> <span class="k">def</span><span class="w"> </span><span class="nf">refs</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">SnapshotRef</span><span class="p">]:</span> |
| <a id="__codelineno-0-1495" name="__codelineno-0-1495"></a><span class="w"> </span><span class="sd">"""Return the snapshot references in the table."""</span> |
| <a id="__codelineno-0-1496" name="__codelineno-0-1496"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">refs</span> |
| <a id="__codelineno-0-1497" name="__codelineno-0-1497"></a> |
| <a id="__codelineno-0-1498" name="__codelineno-0-1498"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_do_commit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">updates</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableUpdate</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span> <span class="n">requirements</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableRequirement</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1499" name="__codelineno-0-1499"></a> <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span><span class="o">.</span><span class="n">commit_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">requirements</span><span class="p">,</span> <span class="n">updates</span><span class="p">)</span> |
| <a id="__codelineno-0-1500" name="__codelineno-0-1500"></a> |
| <a id="__codelineno-0-1501" name="__codelineno-0-1501"></a> <span class="c1"># https://github.com/apache/iceberg/blob/f6faa58/core/src/main/java/org/apache/iceberg/CatalogUtil.java#L527</span> |
| <a id="__codelineno-0-1502" name="__codelineno-0-1502"></a> <span class="c1"># delete old metadata if METADATA_DELETE_AFTER_COMMIT_ENABLED is set to true and uses</span> |
| <a id="__codelineno-0-1503" name="__codelineno-0-1503"></a> <span class="c1"># TableProperties.METADATA_PREVIOUS_VERSIONS_MAX to determine how many previous versions to keep -</span> |
| <a id="__codelineno-0-1504" name="__codelineno-0-1504"></a> <span class="c1"># everything else will be removed.</span> |
| <a id="__codelineno-0-1505" name="__codelineno-0-1505"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-1506" name="__codelineno-0-1506"></a> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span><span class="o">.</span><span class="n">_delete_old_metadata</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">metadata</span><span class="p">)</span> |
| <a id="__codelineno-0-1507" name="__codelineno-0-1507"></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-1508" name="__codelineno-0-1508"></a> <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Failed to delete old metadata after commit: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-1509" name="__codelineno-0-1509"></a> |
| <a id="__codelineno-0-1510" name="__codelineno-0-1510"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">metadata</span> |
| <a id="__codelineno-0-1511" name="__codelineno-0-1511"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">metadata_location</span> |
| <a id="__codelineno-0-1512" name="__codelineno-0-1512"></a> |
| <a id="__codelineno-0-1513" name="__codelineno-0-1513"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <a id="__codelineno-0-1514" name="__codelineno-0-1514"></a><span class="w"> </span><span class="sd">"""Return the equality of two instances of the Table class."""</span> |
| <a id="__codelineno-0-1515" name="__codelineno-0-1515"></a> <span class="k">return</span> <span class="p">(</span> |
| <a id="__codelineno-0-1516" name="__codelineno-0-1516"></a> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">()</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">name</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">metadata</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">metadata_location</span> |
| <a id="__codelineno-0-1517" name="__codelineno-0-1517"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Table</span><span class="p">)</span> |
| <a id="__codelineno-0-1518" name="__codelineno-0-1518"></a> <span class="k">else</span> <span class="kc">False</span> |
| <a id="__codelineno-0-1519" name="__codelineno-0-1519"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1520" name="__codelineno-0-1520"></a> |
| <a id="__codelineno-0-1521" name="__codelineno-0-1521"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <a id="__codelineno-0-1522" name="__codelineno-0-1522"></a><span class="w"> </span><span class="sd">"""Return the string representation of the Table class."""</span> |
| <a id="__codelineno-0-1523" name="__codelineno-0-1523"></a> <span class="n">table_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span><span class="o">.</span><span class="n">table_name_from</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span><span class="p">)</span> |
| <a id="__codelineno-0-1524" name="__codelineno-0-1524"></a> <span class="n">schema_str</span> <span class="o">=</span> <span class="s2">",</span><span class="se">\n</span><span class="s2"> "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">column</span><span class="p">)</span> <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">columns</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">())</span> |
| <a id="__codelineno-0-1525" name="__codelineno-0-1525"></a> <span class="n">partition_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"partition by: [</span><span class="si">{</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">name</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">field</span><span class="w"> </span><span class="ow">in</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span><span class="o">.</span><span class="n">fields</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">spec</span><span class="p">())</span><span class="si">}</span><span class="s2">]"</span> |
| <a id="__codelineno-0-1526" name="__codelineno-0-1526"></a> <span class="n">sort_order_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"sort order: [</span><span class="si">{</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">field</span><span class="p">)</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">field</span><span class="w"> </span><span class="ow">in</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">sort_order</span><span class="p">()</span><span class="o">.</span><span class="n">fields</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">sort_order</span><span class="p">())</span><span class="si">}</span><span class="s2">]"</span> |
| <a id="__codelineno-0-1527" name="__codelineno-0-1527"></a> <span class="n">snapshot_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"snapshot: </span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">current_snapshot</span><span class="p">())</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">current_snapshot</span><span class="p">()</span><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="s1">'null'</span><span class="si">}</span><span class="s2">"</span> |
| <a id="__codelineno-0-1528" name="__codelineno-0-1528"></a> <span class="n">result_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">table_name</span><span class="si">}</span><span class="s2">(</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">schema_str</span><span class="si">}</span><span class="se">\n</span><span class="s2">),</span><span class="se">\n</span><span class="si">{</span><span class="n">partition_str</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="si">{</span><span class="n">sort_order_str</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="si">{</span><span class="n">snapshot_str</span><span class="si">}</span><span class="s2">"</span> |
| <a id="__codelineno-0-1529" name="__codelineno-0-1529"></a> <span class="k">return</span> <span class="n">result_str</span> |
| <a id="__codelineno-0-1530" name="__codelineno-0-1530"></a> |
| <a id="__codelineno-0-1531" name="__codelineno-0-1531"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_daft</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">daft</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-1532" name="__codelineno-0-1532"></a><span class="w"> </span><span class="sd">"""Read a Daft DataFrame lazily from this Iceberg table.</span> |
| <a id="__codelineno-0-1533" name="__codelineno-0-1533"></a> |
| <a id="__codelineno-0-1534" name="__codelineno-0-1534"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1535" name="__codelineno-0-1535"></a><span class="sd"> daft.DataFrame: Unmaterialized Daft Dataframe created from the Iceberg table</span> |
| <a id="__codelineno-0-1536" name="__codelineno-0-1536"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1537" name="__codelineno-0-1537"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">daft</span> |
| <a id="__codelineno-0-1538" name="__codelineno-0-1538"></a> |
| <a id="__codelineno-0-1539" name="__codelineno-0-1539"></a> <span class="k">return</span> <span class="n">daft</span><span class="o">.</span><span class="n">read_iceberg</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-1540" name="__codelineno-0-1540"></a> |
| <a id="__codelineno-0-1541" name="__codelineno-0-1541"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_bodo</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">bd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-1542" name="__codelineno-0-1542"></a><span class="w"> </span><span class="sd">"""Read a bodo DataFrame lazily from this Iceberg table.</span> |
| <a id="__codelineno-0-1543" name="__codelineno-0-1543"></a> |
| <a id="__codelineno-0-1544" name="__codelineno-0-1544"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1545" name="__codelineno-0-1545"></a><span class="sd"> bd.DataFrame: Unmaterialized Bodo Dataframe created from the Iceberg table</span> |
| <a id="__codelineno-0-1546" name="__codelineno-0-1546"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1547" name="__codelineno-0-1547"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">bodo.pandas</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">bd</span> |
| <a id="__codelineno-0-1548" name="__codelineno-0-1548"></a> |
| <a id="__codelineno-0-1549" name="__codelineno-0-1549"></a> <span class="k">return</span> <span class="n">bd</span><span class="o">.</span><span class="n">read_iceberg_table</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-1550" name="__codelineno-0-1550"></a> |
| <a id="__codelineno-0-1551" name="__codelineno-0-1551"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_polars</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pl</span><span class="o">.</span><span class="n">LazyFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-1552" name="__codelineno-0-1552"></a><span class="w"> </span><span class="sd">"""Lazily read from this Apache Iceberg table.</span> |
| <a id="__codelineno-0-1553" name="__codelineno-0-1553"></a> |
| <a id="__codelineno-0-1554" name="__codelineno-0-1554"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1555" name="__codelineno-0-1555"></a><span class="sd"> pl.LazyFrame: Unmaterialized Polars LazyFrame created from the Iceberg table</span> |
| <a id="__codelineno-0-1556" name="__codelineno-0-1556"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1557" name="__codelineno-0-1557"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">polars</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pl</span> |
| <a id="__codelineno-0-1558" name="__codelineno-0-1558"></a> |
| <a id="__codelineno-0-1559" name="__codelineno-0-1559"></a> <span class="k">return</span> <span class="n">pl</span><span class="o">.</span><span class="n">scan_iceberg</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-1560" name="__codelineno-0-1560"></a> |
| <a id="__codelineno-0-1561" name="__codelineno-0-1561"></a> <span class="k">def</span><span class="w"> </span><span class="nf">__datafusion_table_provider__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"IcebergDataFusionTable"</span><span class="p">:</span> |
| <a id="__codelineno-0-1562" name="__codelineno-0-1562"></a><span class="w"> </span><span class="sd">"""Return the DataFusion table provider PyCapsule interface.</span> |
| <a id="__codelineno-0-1563" name="__codelineno-0-1563"></a> |
| <a id="__codelineno-0-1564" name="__codelineno-0-1564"></a><span class="sd"> To support DataFusion features such as push down filtering, this function will return a PyCapsule</span> |
| <a id="__codelineno-0-1565" name="__codelineno-0-1565"></a><span class="sd"> interface that conforms to the FFI Table Provider required by DataFusion. From an end user perspective</span> |
| <a id="__codelineno-0-1566" name="__codelineno-0-1566"></a><span class="sd"> you should not need to call this function directly. Instead you can use ``register_table_provider`` in</span> |
| <a id="__codelineno-0-1567" name="__codelineno-0-1567"></a><span class="sd"> the DataFusion SessionContext.</span> |
| <a id="__codelineno-0-1568" name="__codelineno-0-1568"></a> |
| <a id="__codelineno-0-1569" name="__codelineno-0-1569"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1570" name="__codelineno-0-1570"></a><span class="sd"> A PyCapsule DataFusion TableProvider interface.</span> |
| <a id="__codelineno-0-1571" name="__codelineno-0-1571"></a> |
| <a id="__codelineno-0-1572" name="__codelineno-0-1572"></a><span class="sd"> Example:</span> |
| <a id="__codelineno-0-1573" name="__codelineno-0-1573"></a><span class="sd"> ```python</span> |
| <a id="__codelineno-0-1574" name="__codelineno-0-1574"></a><span class="sd"> from datafusion import SessionContext</span> |
| <a id="__codelineno-0-1575" name="__codelineno-0-1575"></a><span class="sd"> from pyiceberg.catalog import load_catalog</span> |
| <a id="__codelineno-0-1576" name="__codelineno-0-1576"></a><span class="sd"> import pyarrow as pa</span> |
| <a id="__codelineno-0-1577" name="__codelineno-0-1577"></a><span class="sd"> catalog = load_catalog("catalog", type="in-memory")</span> |
| <a id="__codelineno-0-1578" name="__codelineno-0-1578"></a><span class="sd"> catalog.create_namespace_if_not_exists("default")</span> |
| <a id="__codelineno-0-1579" name="__codelineno-0-1579"></a><span class="sd"> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})</span> |
| <a id="__codelineno-0-1580" name="__codelineno-0-1580"></a><span class="sd"> iceberg_table = catalog.create_table("default.test", schema=data.schema)</span> |
| <a id="__codelineno-0-1581" name="__codelineno-0-1581"></a><span class="sd"> iceberg_table.append(data)</span> |
| <a id="__codelineno-0-1582" name="__codelineno-0-1582"></a><span class="sd"> ctx = SessionContext()</span> |
| <a id="__codelineno-0-1583" name="__codelineno-0-1583"></a><span class="sd"> ctx.register_table_provider("test", iceberg_table)</span> |
| <a id="__codelineno-0-1584" name="__codelineno-0-1584"></a><span class="sd"> ctx.table("test").show()</span> |
| <a id="__codelineno-0-1585" name="__codelineno-0-1585"></a><span class="sd"> ```</span> |
| <a id="__codelineno-0-1586" name="__codelineno-0-1586"></a><span class="sd"> Results in</span> |
| <a id="__codelineno-0-1587" name="__codelineno-0-1587"></a><span class="sd"> ```</span> |
| <a id="__codelineno-0-1588" name="__codelineno-0-1588"></a><span class="sd"> DataFrame()</span> |
| <a id="__codelineno-0-1589" name="__codelineno-0-1589"></a><span class="sd"> +---+---+</span> |
| <a id="__codelineno-0-1590" name="__codelineno-0-1590"></a><span class="sd"> | x | y |</span> |
| <a id="__codelineno-0-1591" name="__codelineno-0-1591"></a><span class="sd"> +---+---+</span> |
| <a id="__codelineno-0-1592" name="__codelineno-0-1592"></a><span class="sd"> | 1 | 4 |</span> |
| <a id="__codelineno-0-1593" name="__codelineno-0-1593"></a><span class="sd"> | 2 | 5 |</span> |
| <a id="__codelineno-0-1594" name="__codelineno-0-1594"></a><span class="sd"> | 3 | 6 |</span> |
| <a id="__codelineno-0-1595" name="__codelineno-0-1595"></a><span class="sd"> +---+---+</span> |
| <a id="__codelineno-0-1596" name="__codelineno-0-1596"></a><span class="sd"> ```</span> |
| <a id="__codelineno-0-1597" name="__codelineno-0-1597"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1598" name="__codelineno-0-1598"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg_core.datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">IcebergDataFusionTable</span> |
| <a id="__codelineno-0-1599" name="__codelineno-0-1599"></a> |
| <a id="__codelineno-0-1600" name="__codelineno-0-1600"></a> <span class="k">return</span> <span class="n">IcebergDataFusionTable</span><span class="p">(</span> |
| <a id="__codelineno-0-1601" name="__codelineno-0-1601"></a> <span class="n">identifier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">(),</span> |
| <a id="__codelineno-0-1602" name="__codelineno-0-1602"></a> <span class="n">metadata_location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span><span class="p">,</span> |
| <a id="__codelineno-0-1603" name="__codelineno-0-1603"></a> <span class="n">file_io_properties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">properties</span><span class="p">,</span> |
| <a id="__codelineno-0-1604" name="__codelineno-0-1604"></a> <span class="p">)</span><span class="o">.</span><span class="n">__datafusion_table_provider__</span><span class="p">()</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-attribute"> |
| |
| |
| |
| <h3 id="pyiceberg.table.Table.inspect" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">inspect</span></code> |
| |
| <span class="doc doc-labels"> |
| <small class="doc doc-label doc-label-property"><code>property</code></small> |
| </span> |
| |
| <a href="#pyiceberg.table.Table.inspect" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the InspectTable object to browse the table metadata.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="pyiceberg.table.inspect.InspectTable">InspectTable</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>InspectTable object based on this Table.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-attribute"> |
| |
| |
| |
| <h3 id="pyiceberg.table.Table.maintenance" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">maintenance</span></code> |
| |
| <span class="doc doc-labels"> |
| <small class="doc doc-label doc-label-property"><code>property</code></small> |
| </span> |
| |
| <a href="#pyiceberg.table.Table.maintenance" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the MaintenanceTable object for maintenance.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="MaintenanceTable (pyiceberg.table.maintenance.MaintenanceTable)" href="maintenance/#pyiceberg.table.maintenance.MaintenanceTable">MaintenanceTable</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>MaintenanceTable object based on this Table.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-attribute"> |
| |
| |
| |
| <h3 id="pyiceberg.table.Table.properties" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">properties</span></code> |
| |
| <span class="doc doc-labels"> |
| <small class="doc doc-label doc-label-property"><code>property</code></small> |
| </span> |
| |
| <a href="#pyiceberg.table.Table.properties" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Properties of the table.</p> |
| |
| </div> |
| |
| </div> |
| |
| |
| |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.__datafusion_table_provider__" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">__datafusion_table_provider__</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.__datafusion_table_provider__" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the DataFusion table provider PyCapsule interface.</p> |
| <p>To support DataFusion features such as push down filtering, this function will return a PyCapsule |
| interface that conforms to the FFI Table Provider required by DataFusion. From an end user perspective |
| you should not need to call this function directly. Instead you can use <code>register_table_provider</code> in |
| the DataFusion SessionContext.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>'IcebergDataFusionTable'</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A PyCapsule DataFusion TableProvider interface.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="example" open> |
| <summary>Example</summary> |
| <p><div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span> |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.catalog</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_catalog</span> |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="n">catalog</span> <span class="o">=</span> <span class="n">load_catalog</span><span class="p">(</span><span class="s2">"catalog"</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="s2">"in-memory"</span><span class="p">)</span> |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="n">catalog</span><span class="o">.</span><span class="n">create_namespace_if_not_exists</span><span class="p">(</span><span class="s2">"default"</span><span class="p">)</span> |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="n">data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]})</span> |
| <a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="n">iceberg_table</span> <span class="o">=</span> <span class="n">catalog</span><span class="o">.</span><span class="n">create_table</span><span class="p">(</span><span class="s2">"default.test"</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> |
| <a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="n">iceberg_table</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> |
| <a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span> |
| <a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="n">ctx</span><span class="o">.</span><span class="n">register_table_provider</span><span class="p">(</span><span class="s2">"test"</span><span class="p">,</span> <span class="n">iceberg_table</span><span class="p">)</span> |
| <a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a><span class="n">ctx</span><span class="o">.</span><span class="n">table</span><span class="p">(</span><span class="s2">"test"</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span> |
| </code></pre></div> |
| Results in |
| <div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>DataFrame() |
| <a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>+---+---+ |
| <a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>| x | y | |
| <a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a>+---+---+ |
| <a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a>| 1 | 4 | |
| <a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a>| 2 | 5 | |
| <a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a>| 3 | 6 | |
| <a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a>+---+---+ |
| </code></pre></div></p> |
| </details> |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1561">1561</a></span> |
| <span class="normal"><a href="#__codelineno-0-1562">1562</a></span> |
| <span class="normal"><a href="#__codelineno-0-1563">1563</a></span> |
| <span class="normal"><a href="#__codelineno-0-1564">1564</a></span> |
| <span class="normal"><a href="#__codelineno-0-1565">1565</a></span> |
| <span class="normal"><a href="#__codelineno-0-1566">1566</a></span> |
| <span class="normal"><a href="#__codelineno-0-1567">1567</a></span> |
| <span class="normal"><a href="#__codelineno-0-1568">1568</a></span> |
| <span class="normal"><a href="#__codelineno-0-1569">1569</a></span> |
| <span class="normal"><a href="#__codelineno-0-1570">1570</a></span> |
| <span class="normal"><a href="#__codelineno-0-1571">1571</a></span> |
| <span class="normal"><a href="#__codelineno-0-1572">1572</a></span> |
| <span class="normal"><a href="#__codelineno-0-1573">1573</a></span> |
| <span class="normal"><a href="#__codelineno-0-1574">1574</a></span> |
| <span class="normal"><a href="#__codelineno-0-1575">1575</a></span> |
| <span class="normal"><a href="#__codelineno-0-1576">1576</a></span> |
| <span class="normal"><a href="#__codelineno-0-1577">1577</a></span> |
| <span class="normal"><a href="#__codelineno-0-1578">1578</a></span> |
| <span class="normal"><a href="#__codelineno-0-1579">1579</a></span> |
| <span class="normal"><a href="#__codelineno-0-1580">1580</a></span> |
| <span class="normal"><a href="#__codelineno-0-1581">1581</a></span> |
| <span class="normal"><a href="#__codelineno-0-1582">1582</a></span> |
| <span class="normal"><a href="#__codelineno-0-1583">1583</a></span> |
| <span class="normal"><a href="#__codelineno-0-1584">1584</a></span> |
| <span class="normal"><a href="#__codelineno-0-1585">1585</a></span> |
| <span class="normal"><a href="#__codelineno-0-1586">1586</a></span> |
| <span class="normal"><a href="#__codelineno-0-1587">1587</a></span> |
| <span class="normal"><a href="#__codelineno-0-1588">1588</a></span> |
| <span class="normal"><a href="#__codelineno-0-1589">1589</a></span> |
| <span class="normal"><a href="#__codelineno-0-1590">1590</a></span> |
| <span class="normal"><a href="#__codelineno-0-1591">1591</a></span> |
| <span class="normal"><a href="#__codelineno-0-1592">1592</a></span> |
| <span class="normal"><a href="#__codelineno-0-1593">1593</a></span> |
| <span class="normal"><a href="#__codelineno-0-1594">1594</a></span> |
| <span class="normal"><a href="#__codelineno-0-1595">1595</a></span> |
| <span class="normal"><a href="#__codelineno-0-1596">1596</a></span> |
| <span class="normal"><a href="#__codelineno-0-1597">1597</a></span> |
| <span class="normal"><a href="#__codelineno-0-1598">1598</a></span> |
| <span class="normal"><a href="#__codelineno-0-1599">1599</a></span> |
| <span class="normal"><a href="#__codelineno-0-1600">1600</a></span> |
| <span class="normal"><a href="#__codelineno-0-1601">1601</a></span> |
| <span class="normal"><a href="#__codelineno-0-1602">1602</a></span> |
| <span class="normal"><a href="#__codelineno-0-1603">1603</a></span> |
| <span class="normal"><a href="#__codelineno-0-1604">1604</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1561" name="__codelineno-0-1561"></a><span class="k">def</span><span class="w"> </span><span class="nf">__datafusion_table_provider__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"IcebergDataFusionTable"</span><span class="p">:</span> |
| <a id="__codelineno-0-1562" name="__codelineno-0-1562"></a><span class="w"> </span><span class="sd">"""Return the DataFusion table provider PyCapsule interface.</span> |
| <a id="__codelineno-0-1563" name="__codelineno-0-1563"></a> |
| <a id="__codelineno-0-1564" name="__codelineno-0-1564"></a><span class="sd"> To support DataFusion features such as push down filtering, this function will return a PyCapsule</span> |
| <a id="__codelineno-0-1565" name="__codelineno-0-1565"></a><span class="sd"> interface that conforms to the FFI Table Provider required by DataFusion. From an end user perspective</span> |
| <a id="__codelineno-0-1566" name="__codelineno-0-1566"></a><span class="sd"> you should not need to call this function directly. Instead you can use ``register_table_provider`` in</span> |
| <a id="__codelineno-0-1567" name="__codelineno-0-1567"></a><span class="sd"> the DataFusion SessionContext.</span> |
| <a id="__codelineno-0-1568" name="__codelineno-0-1568"></a> |
| <a id="__codelineno-0-1569" name="__codelineno-0-1569"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1570" name="__codelineno-0-1570"></a><span class="sd"> A PyCapsule DataFusion TableProvider interface.</span> |
| <a id="__codelineno-0-1571" name="__codelineno-0-1571"></a> |
| <a id="__codelineno-0-1572" name="__codelineno-0-1572"></a><span class="sd"> Example:</span> |
| <a id="__codelineno-0-1573" name="__codelineno-0-1573"></a><span class="sd"> ```python</span> |
| <a id="__codelineno-0-1574" name="__codelineno-0-1574"></a><span class="sd"> from datafusion import SessionContext</span> |
| <a id="__codelineno-0-1575" name="__codelineno-0-1575"></a><span class="sd"> from pyiceberg.catalog import load_catalog</span> |
| <a id="__codelineno-0-1576" name="__codelineno-0-1576"></a><span class="sd"> import pyarrow as pa</span> |
| <a id="__codelineno-0-1577" name="__codelineno-0-1577"></a><span class="sd"> catalog = load_catalog("catalog", type="in-memory")</span> |
| <a id="__codelineno-0-1578" name="__codelineno-0-1578"></a><span class="sd"> catalog.create_namespace_if_not_exists("default")</span> |
| <a id="__codelineno-0-1579" name="__codelineno-0-1579"></a><span class="sd"> data = pa.table({"x": [1, 2, 3], "y": [4, 5, 6]})</span> |
| <a id="__codelineno-0-1580" name="__codelineno-0-1580"></a><span class="sd"> iceberg_table = catalog.create_table("default.test", schema=data.schema)</span> |
| <a id="__codelineno-0-1581" name="__codelineno-0-1581"></a><span class="sd"> iceberg_table.append(data)</span> |
| <a id="__codelineno-0-1582" name="__codelineno-0-1582"></a><span class="sd"> ctx = SessionContext()</span> |
| <a id="__codelineno-0-1583" name="__codelineno-0-1583"></a><span class="sd"> ctx.register_table_provider("test", iceberg_table)</span> |
| <a id="__codelineno-0-1584" name="__codelineno-0-1584"></a><span class="sd"> ctx.table("test").show()</span> |
| <a id="__codelineno-0-1585" name="__codelineno-0-1585"></a><span class="sd"> ```</span> |
| <a id="__codelineno-0-1586" name="__codelineno-0-1586"></a><span class="sd"> Results in</span> |
| <a id="__codelineno-0-1587" name="__codelineno-0-1587"></a><span class="sd"> ```</span> |
| <a id="__codelineno-0-1588" name="__codelineno-0-1588"></a><span class="sd"> DataFrame()</span> |
| <a id="__codelineno-0-1589" name="__codelineno-0-1589"></a><span class="sd"> +---+---+</span> |
| <a id="__codelineno-0-1590" name="__codelineno-0-1590"></a><span class="sd"> | x | y |</span> |
| <a id="__codelineno-0-1591" name="__codelineno-0-1591"></a><span class="sd"> +---+---+</span> |
| <a id="__codelineno-0-1592" name="__codelineno-0-1592"></a><span class="sd"> | 1 | 4 |</span> |
| <a id="__codelineno-0-1593" name="__codelineno-0-1593"></a><span class="sd"> | 2 | 5 |</span> |
| <a id="__codelineno-0-1594" name="__codelineno-0-1594"></a><span class="sd"> | 3 | 6 |</span> |
| <a id="__codelineno-0-1595" name="__codelineno-0-1595"></a><span class="sd"> +---+---+</span> |
| <a id="__codelineno-0-1596" name="__codelineno-0-1596"></a><span class="sd"> ```</span> |
| <a id="__codelineno-0-1597" name="__codelineno-0-1597"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1598" name="__codelineno-0-1598"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg_core.datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">IcebergDataFusionTable</span> |
| <a id="__codelineno-0-1599" name="__codelineno-0-1599"></a> |
| <a id="__codelineno-0-1600" name="__codelineno-0-1600"></a> <span class="k">return</span> <span class="n">IcebergDataFusionTable</span><span class="p">(</span> |
| <a id="__codelineno-0-1601" name="__codelineno-0-1601"></a> <span class="n">identifier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">(),</span> |
| <a id="__codelineno-0-1602" name="__codelineno-0-1602"></a> <span class="n">metadata_location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span><span class="p">,</span> |
| <a id="__codelineno-0-1603" name="__codelineno-0-1603"></a> <span class="n">file_io_properties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">properties</span><span class="p">,</span> |
| <a id="__codelineno-0-1604" name="__codelineno-0-1604"></a> <span class="p">)</span><span class="o">.</span><span class="n">__datafusion_table_provider__</span><span class="p">()</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.__eq__" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.__eq__" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the equality of two instances of the Table class.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1513">1513</a></span> |
| <span class="normal"><a href="#__codelineno-0-1514">1514</a></span> |
| <span class="normal"><a href="#__codelineno-0-1515">1515</a></span> |
| <span class="normal"><a href="#__codelineno-0-1516">1516</a></span> |
| <span class="normal"><a href="#__codelineno-0-1517">1517</a></span> |
| <span class="normal"><a href="#__codelineno-0-1518">1518</a></span> |
| <span class="normal"><a href="#__codelineno-0-1519">1519</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1513" name="__codelineno-0-1513"></a><span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <a id="__codelineno-0-1514" name="__codelineno-0-1514"></a><span class="w"> </span><span class="sd">"""Return the equality of two instances of the Table class."""</span> |
| <a id="__codelineno-0-1515" name="__codelineno-0-1515"></a> <span class="k">return</span> <span class="p">(</span> |
| <a id="__codelineno-0-1516" name="__codelineno-0-1516"></a> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">()</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">name</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">metadata</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">metadata_location</span> |
| <a id="__codelineno-0-1517" name="__codelineno-0-1517"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Table</span><span class="p">)</span> |
| <a id="__codelineno-0-1518" name="__codelineno-0-1518"></a> <span class="k">else</span> <span class="kc">False</span> |
| <a id="__codelineno-0-1519" name="__codelineno-0-1519"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.__repr__" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="fm">__repr__</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.__repr__" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the string representation of the Table class.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1521">1521</a></span> |
| <span class="normal"><a href="#__codelineno-0-1522">1522</a></span> |
| <span class="normal"><a href="#__codelineno-0-1523">1523</a></span> |
| <span class="normal"><a href="#__codelineno-0-1524">1524</a></span> |
| <span class="normal"><a href="#__codelineno-0-1525">1525</a></span> |
| <span class="normal"><a href="#__codelineno-0-1526">1526</a></span> |
| <span class="normal"><a href="#__codelineno-0-1527">1527</a></span> |
| <span class="normal"><a href="#__codelineno-0-1528">1528</a></span> |
| <span class="normal"><a href="#__codelineno-0-1529">1529</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1521" name="__codelineno-0-1521"></a><span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <a id="__codelineno-0-1522" name="__codelineno-0-1522"></a><span class="w"> </span><span class="sd">"""Return the string representation of the Table class."""</span> |
| <a id="__codelineno-0-1523" name="__codelineno-0-1523"></a> <span class="n">table_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span><span class="o">.</span><span class="n">table_name_from</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span><span class="p">)</span> |
| <a id="__codelineno-0-1524" name="__codelineno-0-1524"></a> <span class="n">schema_str</span> <span class="o">=</span> <span class="s2">",</span><span class="se">\n</span><span class="s2"> "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">column</span><span class="p">)</span> <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">columns</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">())</span> |
| <a id="__codelineno-0-1525" name="__codelineno-0-1525"></a> <span class="n">partition_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"partition by: [</span><span class="si">{</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">name</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">field</span><span class="w"> </span><span class="ow">in</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span><span class="o">.</span><span class="n">fields</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">spec</span><span class="p">())</span><span class="si">}</span><span class="s2">]"</span> |
| <a id="__codelineno-0-1526" name="__codelineno-0-1526"></a> <span class="n">sort_order_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"sort order: [</span><span class="si">{</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">field</span><span class="p">)</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">field</span><span class="w"> </span><span class="ow">in</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">sort_order</span><span class="p">()</span><span class="o">.</span><span class="n">fields</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">sort_order</span><span class="p">())</span><span class="si">}</span><span class="s2">]"</span> |
| <a id="__codelineno-0-1527" name="__codelineno-0-1527"></a> <span class="n">snapshot_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"snapshot: </span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">current_snapshot</span><span class="p">())</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="bp">self</span><span class="o">.</span><span class="n">current_snapshot</span><span class="p">()</span><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="s1">'null'</span><span class="si">}</span><span class="s2">"</span> |
| <a id="__codelineno-0-1528" name="__codelineno-0-1528"></a> <span class="n">result_str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">table_name</span><span class="si">}</span><span class="s2">(</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">schema_str</span><span class="si">}</span><span class="se">\n</span><span class="s2">),</span><span class="se">\n</span><span class="si">{</span><span class="n">partition_str</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="si">{</span><span class="n">sort_order_str</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="si">{</span><span class="n">snapshot_str</span><span class="si">}</span><span class="s2">"</span> |
| <a id="__codelineno-0-1529" name="__codelineno-0-1529"></a> <span class="k">return</span> <span class="n">result_str</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.add_files" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">add_files</span><span class="p">(</span><span class="n">file_paths</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">check_duplicate_files</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.add_files" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand API for adding files as data files to the table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>file_paths</code> |
| </td> |
| <td> |
| <code><span title="typing.List">List</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The list of full file paths to be added as data files to the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Raises:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="FileNotFoundError">FileNotFoundError</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If the file does not exist.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1467">1467</a></span> |
| <span class="normal"><a href="#__codelineno-0-1468">1468</a></span> |
| <span class="normal"><a href="#__codelineno-0-1469">1469</a></span> |
| <span class="normal"><a href="#__codelineno-0-1470">1470</a></span> |
| <span class="normal"><a href="#__codelineno-0-1471">1471</a></span> |
| <span class="normal"><a href="#__codelineno-0-1472">1472</a></span> |
| <span class="normal"><a href="#__codelineno-0-1473">1473</a></span> |
| <span class="normal"><a href="#__codelineno-0-1474">1474</a></span> |
| <span class="normal"><a href="#__codelineno-0-1475">1475</a></span> |
| <span class="normal"><a href="#__codelineno-0-1476">1476</a></span> |
| <span class="normal"><a href="#__codelineno-0-1477">1477</a></span> |
| <span class="normal"><a href="#__codelineno-0-1478">1478</a></span> |
| <span class="normal"><a href="#__codelineno-0-1479">1479</a></span> |
| <span class="normal"><a href="#__codelineno-0-1480">1480</a></span> |
| <span class="normal"><a href="#__codelineno-0-1481">1481</a></span> |
| <span class="normal"><a href="#__codelineno-0-1482">1482</a></span> |
| <span class="normal"><a href="#__codelineno-0-1483">1483</a></span> |
| <span class="normal"><a href="#__codelineno-0-1484">1484</a></span> |
| <span class="normal"><a href="#__codelineno-0-1485">1485</a></span> |
| <span class="normal"><a href="#__codelineno-0-1486">1486</a></span> |
| <span class="normal"><a href="#__codelineno-0-1487">1487</a></span> |
| <span class="normal"><a href="#__codelineno-0-1488">1488</a></span> |
| <span class="normal"><a href="#__codelineno-0-1489">1489</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1467" name="__codelineno-0-1467"></a><span class="k">def</span><span class="w"> </span><span class="nf">add_files</span><span class="p">(</span> |
| <a id="__codelineno-0-1468" name="__codelineno-0-1468"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1469" name="__codelineno-0-1469"></a> <span class="n">file_paths</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <a id="__codelineno-0-1470" name="__codelineno-0-1470"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1471" name="__codelineno-0-1471"></a> <span class="n">check_duplicate_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1472" name="__codelineno-0-1472"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1473" name="__codelineno-0-1473"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1474" name="__codelineno-0-1474"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1475" name="__codelineno-0-1475"></a><span class="sd"> Shorthand API for adding files as data files to the table.</span> |
| <a id="__codelineno-0-1476" name="__codelineno-0-1476"></a> |
| <a id="__codelineno-0-1477" name="__codelineno-0-1477"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1478" name="__codelineno-0-1478"></a><span class="sd"> file_paths: The list of full file paths to be added as data files to the table</span> |
| <a id="__codelineno-0-1479" name="__codelineno-0-1479"></a> |
| <a id="__codelineno-0-1480" name="__codelineno-0-1480"></a><span class="sd"> Raises:</span> |
| <a id="__codelineno-0-1481" name="__codelineno-0-1481"></a><span class="sd"> FileNotFoundError: If the file does not exist.</span> |
| <a id="__codelineno-0-1482" name="__codelineno-0-1482"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1483" name="__codelineno-0-1483"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1484" name="__codelineno-0-1484"></a> <span class="n">tx</span><span class="o">.</span><span class="n">add_files</span><span class="p">(</span> |
| <a id="__codelineno-0-1485" name="__codelineno-0-1485"></a> <span class="n">file_paths</span><span class="o">=</span><span class="n">file_paths</span><span class="p">,</span> |
| <a id="__codelineno-0-1486" name="__codelineno-0-1486"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> |
| <a id="__codelineno-0-1487" name="__codelineno-0-1487"></a> <span class="n">check_duplicate_files</span><span class="o">=</span><span class="n">check_duplicate_files</span><span class="p">,</span> |
| <a id="__codelineno-0-1488" name="__codelineno-0-1488"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-1489" name="__codelineno-0-1489"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.append" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">append</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.append" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand API for appending a PyArrow table to the table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>df</code> |
| </td> |
| <td> |
| <code><span title="pyarrow.Table">Table</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The Arrow dataframe that will be appended to overwrite the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the append operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1386">1386</a></span> |
| <span class="normal"><a href="#__codelineno-0-1387">1387</a></span> |
| <span class="normal"><a href="#__codelineno-0-1388">1388</a></span> |
| <span class="normal"><a href="#__codelineno-0-1389">1389</a></span> |
| <span class="normal"><a href="#__codelineno-0-1390">1390</a></span> |
| <span class="normal"><a href="#__codelineno-0-1391">1391</a></span> |
| <span class="normal"><a href="#__codelineno-0-1392">1392</a></span> |
| <span class="normal"><a href="#__codelineno-0-1393">1393</a></span> |
| <span class="normal"><a href="#__codelineno-0-1394">1394</a></span> |
| <span class="normal"><a href="#__codelineno-0-1395">1395</a></span> |
| <span class="normal"><a href="#__codelineno-0-1396">1396</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1386" name="__codelineno-0-1386"></a><span class="k">def</span><span class="w"> </span><span class="nf">append</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1387" name="__codelineno-0-1387"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1388" name="__codelineno-0-1388"></a><span class="sd"> Shorthand API for appending a PyArrow table to the table.</span> |
| <a id="__codelineno-0-1389" name="__codelineno-0-1389"></a> |
| <a id="__codelineno-0-1390" name="__codelineno-0-1390"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1391" name="__codelineno-0-1391"></a><span class="sd"> df: The Arrow dataframe that will be appended to overwrite the table</span> |
| <a id="__codelineno-0-1392" name="__codelineno-0-1392"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1393" name="__codelineno-0-1393"></a><span class="sd"> branch: Branch Reference to run the append operation</span> |
| <a id="__codelineno-0-1394" name="__codelineno-0-1394"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1395" name="__codelineno-0-1395"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1396" name="__codelineno-0-1396"></a> <span class="n">tx</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.current_snapshot" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">current_snapshot</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.current_snapshot" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Get the current snapshot for this table, or None if there is no current snapshot.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1236">1236</a></span> |
| <span class="normal"><a href="#__codelineno-0-1237">1237</a></span> |
| <span class="normal"><a href="#__codelineno-0-1238">1238</a></span> |
| <span class="normal"><a href="#__codelineno-0-1239">1239</a></span> |
| <span class="normal"><a href="#__codelineno-0-1240">1240</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1236" name="__codelineno-0-1236"></a><span class="k">def</span><span class="w"> </span><span class="nf">current_snapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1237" name="__codelineno-0-1237"></a><span class="w"> </span><span class="sd">"""Get the current snapshot for this table, or None if there is no current snapshot."""</span> |
| <a id="__codelineno-0-1238" name="__codelineno-0-1238"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">current_snapshot_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1239" name="__codelineno-0-1239"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">current_snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1240" name="__codelineno-0-1240"></a> <span class="k">return</span> <span class="kc">None</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.delete" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">delete</span><span class="p">(</span><span class="n">delete_filter</span><span class="o">=</span><span class="n">ALWAYS_TRUE</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.delete" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for deleting rows from the table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>delete_filter</code> |
| </td> |
| <td> |
| <code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="BooleanExpression (pyiceberg.expressions.BooleanExpression)" href="../expressions/#pyiceberg.expressions.BooleanExpression">BooleanExpression</a>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The predicate that used to remove rows</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.ALWAYS_TRUE">ALWAYS_TRUE</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A bool determine if the provided <code>delete_filter</code> is case-sensitive</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the delete operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1446">1446</a></span> |
| <span class="normal"><a href="#__codelineno-0-1447">1447</a></span> |
| <span class="normal"><a href="#__codelineno-0-1448">1448</a></span> |
| <span class="normal"><a href="#__codelineno-0-1449">1449</a></span> |
| <span class="normal"><a href="#__codelineno-0-1450">1450</a></span> |
| <span class="normal"><a href="#__codelineno-0-1451">1451</a></span> |
| <span class="normal"><a href="#__codelineno-0-1452">1452</a></span> |
| <span class="normal"><a href="#__codelineno-0-1453">1453</a></span> |
| <span class="normal"><a href="#__codelineno-0-1454">1454</a></span> |
| <span class="normal"><a href="#__codelineno-0-1455">1455</a></span> |
| <span class="normal"><a href="#__codelineno-0-1456">1456</a></span> |
| <span class="normal"><a href="#__codelineno-0-1457">1457</a></span> |
| <span class="normal"><a href="#__codelineno-0-1458">1458</a></span> |
| <span class="normal"><a href="#__codelineno-0-1459">1459</a></span> |
| <span class="normal"><a href="#__codelineno-0-1460">1460</a></span> |
| <span class="normal"><a href="#__codelineno-0-1461">1461</a></span> |
| <span class="normal"><a href="#__codelineno-0-1462">1462</a></span> |
| <span class="normal"><a href="#__codelineno-0-1463">1463</a></span> |
| <span class="normal"><a href="#__codelineno-0-1464">1464</a></span> |
| <span class="normal"><a href="#__codelineno-0-1465">1465</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1446" name="__codelineno-0-1446"></a><span class="k">def</span><span class="w"> </span><span class="nf">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-1447" name="__codelineno-0-1447"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1448" name="__codelineno-0-1448"></a> <span class="n">delete_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BooleanExpression</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1449" name="__codelineno-0-1449"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1450" name="__codelineno-0-1450"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1451" name="__codelineno-0-1451"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1452" name="__codelineno-0-1452"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1453" name="__codelineno-0-1453"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1454" name="__codelineno-0-1454"></a><span class="sd"> Shorthand for deleting rows from the table.</span> |
| <a id="__codelineno-0-1455" name="__codelineno-0-1455"></a> |
| <a id="__codelineno-0-1456" name="__codelineno-0-1456"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1457" name="__codelineno-0-1457"></a><span class="sd"> delete_filter: The predicate that used to remove rows</span> |
| <a id="__codelineno-0-1458" name="__codelineno-0-1458"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1459" name="__codelineno-0-1459"></a><span class="sd"> case_sensitive: A bool determine if the provided `delete_filter` is case-sensitive</span> |
| <a id="__codelineno-0-1460" name="__codelineno-0-1460"></a><span class="sd"> branch: Branch Reference to run the delete operation</span> |
| <a id="__codelineno-0-1461" name="__codelineno-0-1461"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1462" name="__codelineno-0-1462"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1463" name="__codelineno-0-1463"></a> <span class="n">tx</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-1464" name="__codelineno-0-1464"></a> <span class="n">delete_filter</span><span class="o">=</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span> |
| <a id="__codelineno-0-1465" name="__codelineno-0-1465"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.dynamic_partition_overwrite" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">dynamic_partition_overwrite</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.dynamic_partition_overwrite" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for dynamic overwriting the table with a PyArrow table.</p> |
| <p>Old partitions are auto detected and replaced with data files created for input arrow table. |
| Args: |
| df: The Arrow dataframe that will be used to overwrite the table |
| snapshot_properties: Custom properties to be added to the snapshot summary |
| branch: Branch Reference to run the dynamic partition overwrite operation</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1398">1398</a></span> |
| <span class="normal"><a href="#__codelineno-0-1399">1399</a></span> |
| <span class="normal"><a href="#__codelineno-0-1400">1400</a></span> |
| <span class="normal"><a href="#__codelineno-0-1401">1401</a></span> |
| <span class="normal"><a href="#__codelineno-0-1402">1402</a></span> |
| <span class="normal"><a href="#__codelineno-0-1403">1403</a></span> |
| <span class="normal"><a href="#__codelineno-0-1404">1404</a></span> |
| <span class="normal"><a href="#__codelineno-0-1405">1405</a></span> |
| <span class="normal"><a href="#__codelineno-0-1406">1406</a></span> |
| <span class="normal"><a href="#__codelineno-0-1407">1407</a></span> |
| <span class="normal"><a href="#__codelineno-0-1408">1408</a></span> |
| <span class="normal"><a href="#__codelineno-0-1409">1409</a></span> |
| <span class="normal"><a href="#__codelineno-0-1410">1410</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1398" name="__codelineno-0-1398"></a><span class="k">def</span><span class="w"> </span><span class="nf">dynamic_partition_overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-1399" name="__codelineno-0-1399"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-1400" name="__codelineno-0-1400"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1401" name="__codelineno-0-1401"></a><span class="w"> </span><span class="sd">"""Shorthand for dynamic overwriting the table with a PyArrow table.</span> |
| <a id="__codelineno-0-1402" name="__codelineno-0-1402"></a> |
| <a id="__codelineno-0-1403" name="__codelineno-0-1403"></a><span class="sd"> Old partitions are auto detected and replaced with data files created for input arrow table.</span> |
| <a id="__codelineno-0-1404" name="__codelineno-0-1404"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1405" name="__codelineno-0-1405"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-1406" name="__codelineno-0-1406"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1407" name="__codelineno-0-1407"></a><span class="sd"> branch: Branch Reference to run the dynamic partition overwrite operation</span> |
| <a id="__codelineno-0-1408" name="__codelineno-0-1408"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1409" name="__codelineno-0-1409"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1410" name="__codelineno-0-1410"></a> <span class="n">tx</span><span class="o">.</span><span class="n">dynamic_partition_overwrite</span><span class="p">(</span><span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.history" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">history</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.history" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Get the snapshot history of this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1267">1267</a></span> |
| <span class="normal"><a href="#__codelineno-0-1268">1268</a></span> |
| <span class="normal"><a href="#__codelineno-0-1269">1269</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1267" name="__codelineno-0-1267"></a><span class="k">def</span><span class="w"> </span><span class="nf">history</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">SnapshotLogEntry</span><span class="p">]:</span> |
| <a id="__codelineno-0-1268" name="__codelineno-0-1268"></a><span class="w"> </span><span class="sd">"""Get the snapshot history of this table."""</span> |
| <a id="__codelineno-0-1269" name="__codelineno-0-1269"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">snapshot_log</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.last_partition_id" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">last_partition_id</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.last_partition_id" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the highest assigned partition field ID across all specs or 999 if only the unpartitioned spec exists.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1213">1213</a></span> |
| <span class="normal"><a href="#__codelineno-0-1214">1214</a></span> |
| <span class="normal"><a href="#__codelineno-0-1215">1215</a></span> |
| <span class="normal"><a href="#__codelineno-0-1216">1216</a></span> |
| <span class="normal"><a href="#__codelineno-0-1217">1217</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1213" name="__codelineno-0-1213"></a><span class="k">def</span><span class="w"> </span><span class="nf">last_partition_id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> |
| <a id="__codelineno-0-1214" name="__codelineno-0-1214"></a><span class="w"> </span><span class="sd">"""Return the highest assigned partition field ID across all specs or 999 if only the unpartitioned spec exists."""</span> |
| <a id="__codelineno-0-1215" name="__codelineno-0-1215"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">last_partition_id</span><span class="p">:</span> |
| <a id="__codelineno-0-1216" name="__codelineno-0-1216"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">last_partition_id</span> |
| <a id="__codelineno-0-1217" name="__codelineno-0-1217"></a> <span class="k">return</span> <span class="n">PARTITION_FIELD_ID_START</span> <span class="o">-</span> <span class="mi">1</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.location" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">location</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.location" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the table's base location.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1224">1224</a></span> |
| <span class="normal"><a href="#__codelineno-0-1225">1225</a></span> |
| <span class="normal"><a href="#__codelineno-0-1226">1226</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1224" name="__codelineno-0-1224"></a><span class="k">def</span><span class="w"> </span><span class="nf">location</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <a id="__codelineno-0-1225" name="__codelineno-0-1225"></a><span class="w"> </span><span class="sd">"""Return the table's base location."""</span> |
| <a id="__codelineno-0-1226" name="__codelineno-0-1226"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">location</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.location_provider" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">location_provider</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.location_provider" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the table's location provider.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1228">1228</a></span> |
| <span class="normal"><a href="#__codelineno-0-1229">1229</a></span> |
| <span class="normal"><a href="#__codelineno-0-1230">1230</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1228" name="__codelineno-0-1228"></a><span class="k">def</span><span class="w"> </span><span class="nf">location_provider</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">LocationProvider</span><span class="p">:</span> |
| <a id="__codelineno-0-1229" name="__codelineno-0-1229"></a><span class="w"> </span><span class="sd">"""Return the table's location provider."""</span> |
| <a id="__codelineno-0-1230" name="__codelineno-0-1230"></a> <span class="k">return</span> <span class="n">load_location_provider</span><span class="p">(</span><span class="n">table_location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">location</span><span class="p">,</span> <span class="n">table_properties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">properties</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.manage_snapshots" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">manage_snapshots</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.manage_snapshots" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand to run snapshot management operations like create branch, create tag, etc.</p> |
| <p>Use table.manage_snapshots().<operation>().commit() to run a specific operation. |
| Use table.manage_snapshots().<operation-one>().<operation-two>().commit() to run multiple operations. |
| Pending changes are applied on commit.</p> |
| <p>We can also use context managers to make more changes. For example,</p> |
| <p>with table.manage_snapshots() as ms: |
| ms.create_tag(snapshot_id1, "Tag_A").create_tag(snapshot_id2, "Tag_B")</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1271">1271</a></span> |
| <span class="normal"><a href="#__codelineno-0-1272">1272</a></span> |
| <span class="normal"><a href="#__codelineno-0-1273">1273</a></span> |
| <span class="normal"><a href="#__codelineno-0-1274">1274</a></span> |
| <span class="normal"><a href="#__codelineno-0-1275">1275</a></span> |
| <span class="normal"><a href="#__codelineno-0-1276">1276</a></span> |
| <span class="normal"><a href="#__codelineno-0-1277">1277</a></span> |
| <span class="normal"><a href="#__codelineno-0-1278">1278</a></span> |
| <span class="normal"><a href="#__codelineno-0-1279">1279</a></span> |
| <span class="normal"><a href="#__codelineno-0-1280">1280</a></span> |
| <span class="normal"><a href="#__codelineno-0-1281">1281</a></span> |
| <span class="normal"><a href="#__codelineno-0-1282">1282</a></span> |
| <span class="normal"><a href="#__codelineno-0-1283">1283</a></span> |
| <span class="normal"><a href="#__codelineno-0-1284">1284</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1271" name="__codelineno-0-1271"></a><span class="k">def</span><span class="w"> </span><span class="nf">manage_snapshots</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ManageSnapshots</span><span class="p">:</span> |
| <a id="__codelineno-0-1272" name="__codelineno-0-1272"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1273" name="__codelineno-0-1273"></a><span class="sd"> Shorthand to run snapshot management operations like create branch, create tag, etc.</span> |
| <a id="__codelineno-0-1274" name="__codelineno-0-1274"></a> |
| <a id="__codelineno-0-1275" name="__codelineno-0-1275"></a><span class="sd"> Use table.manage_snapshots().<operation>().commit() to run a specific operation.</span> |
| <a id="__codelineno-0-1276" name="__codelineno-0-1276"></a><span class="sd"> Use table.manage_snapshots().<operation-one>().<operation-two>().commit() to run multiple operations.</span> |
| <a id="__codelineno-0-1277" name="__codelineno-0-1277"></a><span class="sd"> Pending changes are applied on commit.</span> |
| <a id="__codelineno-0-1278" name="__codelineno-0-1278"></a> |
| <a id="__codelineno-0-1279" name="__codelineno-0-1279"></a><span class="sd"> We can also use context managers to make more changes. For example,</span> |
| <a id="__codelineno-0-1280" name="__codelineno-0-1280"></a> |
| <a id="__codelineno-0-1281" name="__codelineno-0-1281"></a><span class="sd"> with table.manage_snapshots() as ms:</span> |
| <a id="__codelineno-0-1282" name="__codelineno-0-1282"></a><span class="sd"> ms.create_tag(snapshot_id1, "Tag_A").create_tag(snapshot_id2, "Tag_B")</span> |
| <a id="__codelineno-0-1283" name="__codelineno-0-1283"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1284" name="__codelineno-0-1284"></a> <span class="k">return</span> <span class="n">ManageSnapshots</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.name" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">name</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.name" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the identifier of this table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Identifier = Tuple[str, ...] |
| |
| |
| module-attribute |
| (pyiceberg.typedef.Identifier)" href="../typedef/#pyiceberg.typedef.Identifier">Identifier</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>An Identifier tuple of the table name</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1126">1126</a></span> |
| <span class="normal"><a href="#__codelineno-0-1127">1127</a></span> |
| <span class="normal"><a href="#__codelineno-0-1128">1128</a></span> |
| <span class="normal"><a href="#__codelineno-0-1129">1129</a></span> |
| <span class="normal"><a href="#__codelineno-0-1130">1130</a></span> |
| <span class="normal"><a href="#__codelineno-0-1131">1131</a></span> |
| <span class="normal"><a href="#__codelineno-0-1132">1132</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1126" name="__codelineno-0-1126"></a><span class="k">def</span><span class="w"> </span><span class="nf">name</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Identifier</span><span class="p">:</span> |
| <a id="__codelineno-0-1127" name="__codelineno-0-1127"></a><span class="w"> </span><span class="sd">"""Return the identifier of this table.</span> |
| <a id="__codelineno-0-1128" name="__codelineno-0-1128"></a> |
| <a id="__codelineno-0-1129" name="__codelineno-0-1129"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1130" name="__codelineno-0-1130"></a><span class="sd"> An Identifier tuple of the table name</span> |
| <a id="__codelineno-0-1131" name="__codelineno-0-1131"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1132" name="__codelineno-0-1132"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.name_mapping" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">name_mapping</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.name_mapping" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the table's field-id NameMapping.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1328">1328</a></span> |
| <span class="normal"><a href="#__codelineno-0-1329">1329</a></span> |
| <span class="normal"><a href="#__codelineno-0-1330">1330</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1328" name="__codelineno-0-1328"></a><span class="k">def</span><span class="w"> </span><span class="nf">name_mapping</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">NameMapping</span><span class="p">]:</span> |
| <a id="__codelineno-0-1329" name="__codelineno-0-1329"></a><span class="w"> </span><span class="sd">"""Return the table's field-id NameMapping."""</span> |
| <a id="__codelineno-0-1330" name="__codelineno-0-1330"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">()</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.overwrite" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">overwrite</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">overwrite_filter</span><span class="o">=</span><span class="n">ALWAYS_TRUE</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.overwrite" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for overwriting the table with a PyArrow table.</p> |
| <p>An overwrite may produce zero or more snapshots based on the operation:</p> |
| <div class="highlight"><pre><span></span><code>- DELETE: In case existing Parquet files can be dropped completely. |
| - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the overwrite filter.. |
| - APPEND: In case new data is being inserted into the table. |
| </code></pre></div> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>df</code> |
| </td> |
| <td> |
| <code><span title="pyarrow.Table">Table</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The Arrow dataframe that will be used to overwrite the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>overwrite_filter</code> |
| </td> |
| <td> |
| <code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="BooleanExpression (pyiceberg.expressions.BooleanExpression)" href="../expressions/#pyiceberg.expressions.BooleanExpression">BooleanExpression</a>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>ALWAYS_TRUE when you overwrite all the data, |
| or a boolean expression in case of a partial overwrite</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.ALWAYS_TRUE">ALWAYS_TRUE</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A bool determine if the provided <code>overwrite_filter</code> is case-sensitive</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the overwrite operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1412">1412</a></span> |
| <span class="normal"><a href="#__codelineno-0-1413">1413</a></span> |
| <span class="normal"><a href="#__codelineno-0-1414">1414</a></span> |
| <span class="normal"><a href="#__codelineno-0-1415">1415</a></span> |
| <span class="normal"><a href="#__codelineno-0-1416">1416</a></span> |
| <span class="normal"><a href="#__codelineno-0-1417">1417</a></span> |
| <span class="normal"><a href="#__codelineno-0-1418">1418</a></span> |
| <span class="normal"><a href="#__codelineno-0-1419">1419</a></span> |
| <span class="normal"><a href="#__codelineno-0-1420">1420</a></span> |
| <span class="normal"><a href="#__codelineno-0-1421">1421</a></span> |
| <span class="normal"><a href="#__codelineno-0-1422">1422</a></span> |
| <span class="normal"><a href="#__codelineno-0-1423">1423</a></span> |
| <span class="normal"><a href="#__codelineno-0-1424">1424</a></span> |
| <span class="normal"><a href="#__codelineno-0-1425">1425</a></span> |
| <span class="normal"><a href="#__codelineno-0-1426">1426</a></span> |
| <span class="normal"><a href="#__codelineno-0-1427">1427</a></span> |
| <span class="normal"><a href="#__codelineno-0-1428">1428</a></span> |
| <span class="normal"><a href="#__codelineno-0-1429">1429</a></span> |
| <span class="normal"><a href="#__codelineno-0-1430">1430</a></span> |
| <span class="normal"><a href="#__codelineno-0-1431">1431</a></span> |
| <span class="normal"><a href="#__codelineno-0-1432">1432</a></span> |
| <span class="normal"><a href="#__codelineno-0-1433">1433</a></span> |
| <span class="normal"><a href="#__codelineno-0-1434">1434</a></span> |
| <span class="normal"><a href="#__codelineno-0-1435">1435</a></span> |
| <span class="normal"><a href="#__codelineno-0-1436">1436</a></span> |
| <span class="normal"><a href="#__codelineno-0-1437">1437</a></span> |
| <span class="normal"><a href="#__codelineno-0-1438">1438</a></span> |
| <span class="normal"><a href="#__codelineno-0-1439">1439</a></span> |
| <span class="normal"><a href="#__codelineno-0-1440">1440</a></span> |
| <span class="normal"><a href="#__codelineno-0-1441">1441</a></span> |
| <span class="normal"><a href="#__codelineno-0-1442">1442</a></span> |
| <span class="normal"><a href="#__codelineno-0-1443">1443</a></span> |
| <span class="normal"><a href="#__codelineno-0-1444">1444</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1412" name="__codelineno-0-1412"></a><span class="k">def</span><span class="w"> </span><span class="nf">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-1413" name="__codelineno-0-1413"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1414" name="__codelineno-0-1414"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-1415" name="__codelineno-0-1415"></a> <span class="n">overwrite_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BooleanExpression</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1416" name="__codelineno-0-1416"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1417" name="__codelineno-0-1417"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1418" name="__codelineno-0-1418"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1419" name="__codelineno-0-1419"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1420" name="__codelineno-0-1420"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1421" name="__codelineno-0-1421"></a><span class="sd"> Shorthand for overwriting the table with a PyArrow table.</span> |
| <a id="__codelineno-0-1422" name="__codelineno-0-1422"></a> |
| <a id="__codelineno-0-1423" name="__codelineno-0-1423"></a><span class="sd"> An overwrite may produce zero or more snapshots based on the operation:</span> |
| <a id="__codelineno-0-1424" name="__codelineno-0-1424"></a> |
| <a id="__codelineno-0-1425" name="__codelineno-0-1425"></a><span class="sd"> - DELETE: In case existing Parquet files can be dropped completely.</span> |
| <a id="__codelineno-0-1426" name="__codelineno-0-1426"></a><span class="sd"> - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the overwrite filter..</span> |
| <a id="__codelineno-0-1427" name="__codelineno-0-1427"></a><span class="sd"> - APPEND: In case new data is being inserted into the table.</span> |
| <a id="__codelineno-0-1428" name="__codelineno-0-1428"></a> |
| <a id="__codelineno-0-1429" name="__codelineno-0-1429"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1430" name="__codelineno-0-1430"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-1431" name="__codelineno-0-1431"></a><span class="sd"> overwrite_filter: ALWAYS_TRUE when you overwrite all the data,</span> |
| <a id="__codelineno-0-1432" name="__codelineno-0-1432"></a><span class="sd"> or a boolean expression in case of a partial overwrite</span> |
| <a id="__codelineno-0-1433" name="__codelineno-0-1433"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-1434" name="__codelineno-0-1434"></a><span class="sd"> case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive</span> |
| <a id="__codelineno-0-1435" name="__codelineno-0-1435"></a><span class="sd"> branch: Branch Reference to run the overwrite operation</span> |
| <a id="__codelineno-0-1436" name="__codelineno-0-1436"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1437" name="__codelineno-0-1437"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1438" name="__codelineno-0-1438"></a> <span class="n">tx</span><span class="o">.</span><span class="n">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-1439" name="__codelineno-0-1439"></a> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> |
| <a id="__codelineno-0-1440" name="__codelineno-0-1440"></a> <span class="n">overwrite_filter</span><span class="o">=</span><span class="n">overwrite_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-1441" name="__codelineno-0-1441"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1442" name="__codelineno-0-1442"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> |
| <a id="__codelineno-0-1443" name="__codelineno-0-1443"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-1444" name="__codelineno-0-1444"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.refresh" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">refresh</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.refresh" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Refresh the current table metadata.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Table (pyiceberg.table.Table)" href="#pyiceberg.table.Table">Table</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>An updated instance of the same Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1114">1114</a></span> |
| <span class="normal"><a href="#__codelineno-0-1115">1115</a></span> |
| <span class="normal"><a href="#__codelineno-0-1116">1116</a></span> |
| <span class="normal"><a href="#__codelineno-0-1117">1117</a></span> |
| <span class="normal"><a href="#__codelineno-0-1118">1118</a></span> |
| <span class="normal"><a href="#__codelineno-0-1119">1119</a></span> |
| <span class="normal"><a href="#__codelineno-0-1120">1120</a></span> |
| <span class="normal"><a href="#__codelineno-0-1121">1121</a></span> |
| <span class="normal"><a href="#__codelineno-0-1122">1122</a></span> |
| <span class="normal"><a href="#__codelineno-0-1123">1123</a></span> |
| <span class="normal"><a href="#__codelineno-0-1124">1124</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1114" name="__codelineno-0-1114"></a><span class="k">def</span><span class="w"> </span><span class="nf">refresh</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-1115" name="__codelineno-0-1115"></a><span class="w"> </span><span class="sd">"""Refresh the current table metadata.</span> |
| <a id="__codelineno-0-1116" name="__codelineno-0-1116"></a> |
| <a id="__codelineno-0-1117" name="__codelineno-0-1117"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1118" name="__codelineno-0-1118"></a><span class="sd"> An updated instance of the same Iceberg table</span> |
| <a id="__codelineno-0-1119" name="__codelineno-0-1119"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1120" name="__codelineno-0-1120"></a> <span class="n">fresh</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">catalog</span><span class="o">.</span><span class="n">load_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_identifier</span><span class="p">)</span> |
| <a id="__codelineno-0-1121" name="__codelineno-0-1121"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">fresh</span><span class="o">.</span><span class="n">metadata</span> |
| <a id="__codelineno-0-1122" name="__codelineno-0-1122"></a> <span class="bp">self</span><span class="o">.</span><span class="n">io</span> <span class="o">=</span> <span class="n">fresh</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-1123" name="__codelineno-0-1123"></a> <span class="bp">self</span><span class="o">.</span><span class="n">metadata_location</span> <span class="o">=</span> <span class="n">fresh</span><span class="o">.</span><span class="n">metadata_location</span> |
| <a id="__codelineno-0-1124" name="__codelineno-0-1124"></a> <span class="k">return</span> <span class="bp">self</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.refs" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">refs</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.refs" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the snapshot references in the table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1494">1494</a></span> |
| <span class="normal"><a href="#__codelineno-0-1495">1495</a></span> |
| <span class="normal"><a href="#__codelineno-0-1496">1496</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1494" name="__codelineno-0-1494"></a><span class="k">def</span><span class="w"> </span><span class="nf">refs</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">SnapshotRef</span><span class="p">]:</span> |
| <a id="__codelineno-0-1495" name="__codelineno-0-1495"></a><span class="w"> </span><span class="sd">"""Return the snapshot references in the table."""</span> |
| <a id="__codelineno-0-1496" name="__codelineno-0-1496"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">refs</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.scan" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">scan</span><span class="p">(</span><span class="n">row_filter</span><span class="o">=</span><span class="n">ALWAYS_TRUE</span><span class="p">,</span> <span class="n">selected_fields</span><span class="o">=</span><span class="p">(</span><span class="s1">'*'</span><span class="p">,),</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">snapshot_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">limit</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.scan" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Fetch a DataScan based on the table's current metadata.</p> |
| <div class="highlight"><pre><span></span><code>The data scan can be used to project the table's data |
| that matches the provided row_filter onto the table's |
| current schema. |
| </code></pre></div> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>row_filter</code> |
| </td> |
| <td> |
| <code><span title="typing.Union">Union</span>[<span title="str">str</span>, <a class="autorefs autorefs-internal" title="BooleanExpression (pyiceberg.expressions.BooleanExpression)" href="../expressions/#pyiceberg.expressions.BooleanExpression">BooleanExpression</a>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A string or BooleanExpression that describes the |
| desired rows</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.ALWAYS_TRUE">ALWAYS_TRUE</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>selected_fields</code> |
| </td> |
| <td> |
| <code><span title="typing.Tuple">Tuple</span>[<span title="str">str</span>, ...]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A tuple of strings representing the column names |
| to return in the output dataframe.</p> |
| </div> |
| </td> |
| <td> |
| <code>('*',)</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If True column matching is case sensitive</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_id</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="int">int</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Optional Snapshot ID to time travel to. If None, |
| scans the table as of the current snapshot ID.</p> |
| </div> |
| </td> |
| <td> |
| <code>None</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>options</code> |
| </td> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Properties = Dict[str, Any] |
| |
| |
| module-attribute |
| (pyiceberg.typedef.Properties)" href="../typedef/#pyiceberg.typedef.Properties">Properties</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Additional Table properties as a dictionary of |
| string key value pairs to use for this scan.</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>limit</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="int">int</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>An integer representing the number of rows to |
| return in the scan result. If None, fetches all |
| matching rows.</p> |
| </div> |
| </td> |
| <td> |
| <code>None</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="DataScan (pyiceberg.table.DataScan)" href="#pyiceberg.table.DataScan">DataScan</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A DataScan based on the table's current metadata.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1134">1134</a></span> |
| <span class="normal"><a href="#__codelineno-0-1135">1135</a></span> |
| <span class="normal"><a href="#__codelineno-0-1136">1136</a></span> |
| <span class="normal"><a href="#__codelineno-0-1137">1137</a></span> |
| <span class="normal"><a href="#__codelineno-0-1138">1138</a></span> |
| <span class="normal"><a href="#__codelineno-0-1139">1139</a></span> |
| <span class="normal"><a href="#__codelineno-0-1140">1140</a></span> |
| <span class="normal"><a href="#__codelineno-0-1141">1141</a></span> |
| <span class="normal"><a href="#__codelineno-0-1142">1142</a></span> |
| <span class="normal"><a href="#__codelineno-0-1143">1143</a></span> |
| <span class="normal"><a href="#__codelineno-0-1144">1144</a></span> |
| <span class="normal"><a href="#__codelineno-0-1145">1145</a></span> |
| <span class="normal"><a href="#__codelineno-0-1146">1146</a></span> |
| <span class="normal"><a href="#__codelineno-0-1147">1147</a></span> |
| <span class="normal"><a href="#__codelineno-0-1148">1148</a></span> |
| <span class="normal"><a href="#__codelineno-0-1149">1149</a></span> |
| <span class="normal"><a href="#__codelineno-0-1150">1150</a></span> |
| <span class="normal"><a href="#__codelineno-0-1151">1151</a></span> |
| <span class="normal"><a href="#__codelineno-0-1152">1152</a></span> |
| <span class="normal"><a href="#__codelineno-0-1153">1153</a></span> |
| <span class="normal"><a href="#__codelineno-0-1154">1154</a></span> |
| <span class="normal"><a href="#__codelineno-0-1155">1155</a></span> |
| <span class="normal"><a href="#__codelineno-0-1156">1156</a></span> |
| <span class="normal"><a href="#__codelineno-0-1157">1157</a></span> |
| <span class="normal"><a href="#__codelineno-0-1158">1158</a></span> |
| <span class="normal"><a href="#__codelineno-0-1159">1159</a></span> |
| <span class="normal"><a href="#__codelineno-0-1160">1160</a></span> |
| <span class="normal"><a href="#__codelineno-0-1161">1161</a></span> |
| <span class="normal"><a href="#__codelineno-0-1162">1162</a></span> |
| <span class="normal"><a href="#__codelineno-0-1163">1163</a></span> |
| <span class="normal"><a href="#__codelineno-0-1164">1164</a></span> |
| <span class="normal"><a href="#__codelineno-0-1165">1165</a></span> |
| <span class="normal"><a href="#__codelineno-0-1166">1166</a></span> |
| <span class="normal"><a href="#__codelineno-0-1167">1167</a></span> |
| <span class="normal"><a href="#__codelineno-0-1168">1168</a></span> |
| <span class="normal"><a href="#__codelineno-0-1169">1169</a></span> |
| <span class="normal"><a href="#__codelineno-0-1170">1170</a></span> |
| <span class="normal"><a href="#__codelineno-0-1171">1171</a></span> |
| <span class="normal"><a href="#__codelineno-0-1172">1172</a></span> |
| <span class="normal"><a href="#__codelineno-0-1173">1173</a></span> |
| <span class="normal"><a href="#__codelineno-0-1174">1174</a></span> |
| <span class="normal"><a href="#__codelineno-0-1175">1175</a></span> |
| <span class="normal"><a href="#__codelineno-0-1176">1176</a></span> |
| <span class="normal"><a href="#__codelineno-0-1177">1177</a></span> |
| <span class="normal"><a href="#__codelineno-0-1178">1178</a></span> |
| <span class="normal"><a href="#__codelineno-0-1179">1179</a></span> |
| <span class="normal"><a href="#__codelineno-0-1180">1180</a></span> |
| <span class="normal"><a href="#__codelineno-0-1181">1181</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1134" name="__codelineno-0-1134"></a><span class="k">def</span><span class="w"> </span><span class="nf">scan</span><span class="p">(</span> |
| <a id="__codelineno-0-1135" name="__codelineno-0-1135"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1136" name="__codelineno-0-1136"></a> <span class="n">row_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1137" name="__codelineno-0-1137"></a> <span class="n">selected_fields</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"*"</span><span class="p">,),</span> |
| <a id="__codelineno-0-1138" name="__codelineno-0-1138"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1139" name="__codelineno-0-1139"></a> <span class="n">snapshot_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1140" name="__codelineno-0-1140"></a> <span class="n">options</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1141" name="__codelineno-0-1141"></a> <span class="n">limit</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1142" name="__codelineno-0-1142"></a><span class="p">)</span> <span class="o">-></span> <span class="n">DataScan</span><span class="p">:</span> |
| <a id="__codelineno-0-1143" name="__codelineno-0-1143"></a><span class="w"> </span><span class="sd">"""Fetch a DataScan based on the table's current metadata.</span> |
| <a id="__codelineno-0-1144" name="__codelineno-0-1144"></a> |
| <a id="__codelineno-0-1145" name="__codelineno-0-1145"></a><span class="sd"> The data scan can be used to project the table's data</span> |
| <a id="__codelineno-0-1146" name="__codelineno-0-1146"></a><span class="sd"> that matches the provided row_filter onto the table's</span> |
| <a id="__codelineno-0-1147" name="__codelineno-0-1147"></a><span class="sd"> current schema.</span> |
| <a id="__codelineno-0-1148" name="__codelineno-0-1148"></a> |
| <a id="__codelineno-0-1149" name="__codelineno-0-1149"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1150" name="__codelineno-0-1150"></a><span class="sd"> row_filter:</span> |
| <a id="__codelineno-0-1151" name="__codelineno-0-1151"></a><span class="sd"> A string or BooleanExpression that describes the</span> |
| <a id="__codelineno-0-1152" name="__codelineno-0-1152"></a><span class="sd"> desired rows</span> |
| <a id="__codelineno-0-1153" name="__codelineno-0-1153"></a><span class="sd"> selected_fields:</span> |
| <a id="__codelineno-0-1154" name="__codelineno-0-1154"></a><span class="sd"> A tuple of strings representing the column names</span> |
| <a id="__codelineno-0-1155" name="__codelineno-0-1155"></a><span class="sd"> to return in the output dataframe.</span> |
| <a id="__codelineno-0-1156" name="__codelineno-0-1156"></a><span class="sd"> case_sensitive:</span> |
| <a id="__codelineno-0-1157" name="__codelineno-0-1157"></a><span class="sd"> If True column matching is case sensitive</span> |
| <a id="__codelineno-0-1158" name="__codelineno-0-1158"></a><span class="sd"> snapshot_id:</span> |
| <a id="__codelineno-0-1159" name="__codelineno-0-1159"></a><span class="sd"> Optional Snapshot ID to time travel to. If None,</span> |
| <a id="__codelineno-0-1160" name="__codelineno-0-1160"></a><span class="sd"> scans the table as of the current snapshot ID.</span> |
| <a id="__codelineno-0-1161" name="__codelineno-0-1161"></a><span class="sd"> options:</span> |
| <a id="__codelineno-0-1162" name="__codelineno-0-1162"></a><span class="sd"> Additional Table properties as a dictionary of</span> |
| <a id="__codelineno-0-1163" name="__codelineno-0-1163"></a><span class="sd"> string key value pairs to use for this scan.</span> |
| <a id="__codelineno-0-1164" name="__codelineno-0-1164"></a><span class="sd"> limit:</span> |
| <a id="__codelineno-0-1165" name="__codelineno-0-1165"></a><span class="sd"> An integer representing the number of rows to</span> |
| <a id="__codelineno-0-1166" name="__codelineno-0-1166"></a><span class="sd"> return in the scan result. If None, fetches all</span> |
| <a id="__codelineno-0-1167" name="__codelineno-0-1167"></a><span class="sd"> matching rows.</span> |
| <a id="__codelineno-0-1168" name="__codelineno-0-1168"></a> |
| <a id="__codelineno-0-1169" name="__codelineno-0-1169"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1170" name="__codelineno-0-1170"></a><span class="sd"> A DataScan based on the table's current metadata.</span> |
| <a id="__codelineno-0-1171" name="__codelineno-0-1171"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1172" name="__codelineno-0-1172"></a> <span class="k">return</span> <span class="n">DataScan</span><span class="p">(</span> |
| <a id="__codelineno-0-1173" name="__codelineno-0-1173"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-1174" name="__codelineno-0-1174"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-1175" name="__codelineno-0-1175"></a> <span class="n">row_filter</span><span class="o">=</span><span class="n">row_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-1176" name="__codelineno-0-1176"></a> <span class="n">selected_fields</span><span class="o">=</span><span class="n">selected_fields</span><span class="p">,</span> |
| <a id="__codelineno-0-1177" name="__codelineno-0-1177"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1178" name="__codelineno-0-1178"></a> <span class="n">snapshot_id</span><span class="o">=</span><span class="n">snapshot_id</span><span class="p">,</span> |
| <a id="__codelineno-0-1179" name="__codelineno-0-1179"></a> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> |
| <a id="__codelineno-0-1180" name="__codelineno-0-1180"></a> <span class="n">limit</span><span class="o">=</span><span class="n">limit</span><span class="p">,</span> |
| <a id="__codelineno-0-1181" name="__codelineno-0-1181"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.schema" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">schema</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.schema" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the schema for this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1187">1187</a></span> |
| <span class="normal"><a href="#__codelineno-0-1188">1188</a></span> |
| <span class="normal"><a href="#__codelineno-0-1189">1189</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1187" name="__codelineno-0-1187"></a><span class="k">def</span><span class="w"> </span><span class="nf">schema</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Schema</span><span class="p">:</span> |
| <a id="__codelineno-0-1188" name="__codelineno-0-1188"></a><span class="w"> </span><span class="sd">"""Return the schema for this table."""</span> |
| <a id="__codelineno-0-1189" name="__codelineno-0-1189"></a> <span class="k">return</span> <span class="nb">next</span><span class="p">(</span><span class="n">schema</span> <span class="k">for</span> <span class="n">schema</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">schemas</span> <span class="k">if</span> <span class="n">schema</span><span class="o">.</span><span class="n">schema_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">current_schema_id</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.schemas" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">schemas</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.schemas" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return a dict of the schema of this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1191">1191</a></span> |
| <span class="normal"><a href="#__codelineno-0-1192">1192</a></span> |
| <span class="normal"><a href="#__codelineno-0-1193">1193</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1191" name="__codelineno-0-1191"></a><span class="k">def</span><span class="w"> </span><span class="nf">schemas</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Schema</span><span class="p">]:</span> |
| <a id="__codelineno-0-1192" name="__codelineno-0-1192"></a><span class="w"> </span><span class="sd">"""Return a dict of the schema of this table."""</span> |
| <a id="__codelineno-0-1193" name="__codelineno-0-1193"></a> <span class="k">return</span> <span class="p">{</span><span class="n">schema</span><span class="o">.</span><span class="n">schema_id</span><span class="p">:</span> <span class="n">schema</span> <span class="k">for</span> <span class="n">schema</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">schemas</span><span class="p">}</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.snapshot_as_of_timestamp" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">snapshot_as_of_timestamp</span><span class="p">(</span><span class="n">timestamp_ms</span><span class="p">,</span> <span class="n">inclusive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.snapshot_as_of_timestamp" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Get the snapshot that was current as of or right before the given timestamp, or None if there is no matching snapshot.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>timestamp_ms</code> |
| </td> |
| <td> |
| <code><span title="int">int</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Find snapshot that was current at/before this timestamp</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>inclusive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Includes timestamp_ms in search when True. Excludes timestamp_ms when False</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1255">1255</a></span> |
| <span class="normal"><a href="#__codelineno-0-1256">1256</a></span> |
| <span class="normal"><a href="#__codelineno-0-1257">1257</a></span> |
| <span class="normal"><a href="#__codelineno-0-1258">1258</a></span> |
| <span class="normal"><a href="#__codelineno-0-1259">1259</a></span> |
| <span class="normal"><a href="#__codelineno-0-1260">1260</a></span> |
| <span class="normal"><a href="#__codelineno-0-1261">1261</a></span> |
| <span class="normal"><a href="#__codelineno-0-1262">1262</a></span> |
| <span class="normal"><a href="#__codelineno-0-1263">1263</a></span> |
| <span class="normal"><a href="#__codelineno-0-1264">1264</a></span> |
| <span class="normal"><a href="#__codelineno-0-1265">1265</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1255" name="__codelineno-0-1255"></a><span class="k">def</span><span class="w"> </span><span class="nf">snapshot_as_of_timestamp</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">timestamp_ms</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inclusive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1256" name="__codelineno-0-1256"></a><span class="w"> </span><span class="sd">"""Get the snapshot that was current as of or right before the given timestamp, or None if there is no matching snapshot.</span> |
| <a id="__codelineno-0-1257" name="__codelineno-0-1257"></a> |
| <a id="__codelineno-0-1258" name="__codelineno-0-1258"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1259" name="__codelineno-0-1259"></a><span class="sd"> timestamp_ms: Find snapshot that was current at/before this timestamp</span> |
| <a id="__codelineno-0-1260" name="__codelineno-0-1260"></a><span class="sd"> inclusive: Includes timestamp_ms in search when True. Excludes timestamp_ms when False</span> |
| <a id="__codelineno-0-1261" name="__codelineno-0-1261"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1262" name="__codelineno-0-1262"></a> <span class="k">for</span> <span class="n">log_entry</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">history</span><span class="p">()):</span> |
| <a id="__codelineno-0-1263" name="__codelineno-0-1263"></a> <span class="k">if</span> <span class="p">(</span><span class="n">inclusive</span> <span class="ow">and</span> <span class="n">log_entry</span><span class="o">.</span><span class="n">timestamp_ms</span> <span class="o"><=</span> <span class="n">timestamp_ms</span><span class="p">)</span> <span class="ow">or</span> <span class="n">log_entry</span><span class="o">.</span><span class="n">timestamp_ms</span> <span class="o"><</span> <span class="n">timestamp_ms</span><span class="p">:</span> |
| <a id="__codelineno-0-1264" name="__codelineno-0-1264"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">log_entry</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1265" name="__codelineno-0-1265"></a> <span class="k">return</span> <span class="kc">None</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.snapshot_by_id" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">snapshot_id</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.snapshot_by_id" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Get the snapshot of this table with the given id, or None if there is no matching snapshot.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1245">1245</a></span> |
| <span class="normal"><a href="#__codelineno-0-1246">1246</a></span> |
| <span class="normal"><a href="#__codelineno-0-1247">1247</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1245" name="__codelineno-0-1245"></a><span class="k">def</span><span class="w"> </span><span class="nf">snapshot_by_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">snapshot_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1246" name="__codelineno-0-1246"></a><span class="w"> </span><span class="sd">"""Get the snapshot of this table with the given id, or None if there is no matching snapshot."""</span> |
| <a id="__codelineno-0-1247" name="__codelineno-0-1247"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">snapshot_id</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.snapshot_by_name" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">snapshot_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.snapshot_by_name" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the snapshot referenced by the given name or null if no such reference exists.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1249">1249</a></span> |
| <span class="normal"><a href="#__codelineno-0-1250">1250</a></span> |
| <span class="normal"><a href="#__codelineno-0-1251">1251</a></span> |
| <span class="normal"><a href="#__codelineno-0-1252">1252</a></span> |
| <span class="normal"><a href="#__codelineno-0-1253">1253</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1249" name="__codelineno-0-1249"></a><span class="k">def</span><span class="w"> </span><span class="nf">snapshot_by_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1250" name="__codelineno-0-1250"></a><span class="w"> </span><span class="sd">"""Return the snapshot referenced by the given name or null if no such reference exists."""</span> |
| <a id="__codelineno-0-1251" name="__codelineno-0-1251"></a> <span class="k">if</span> <span class="n">ref</span> <span class="o">:=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">refs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">):</span> |
| <a id="__codelineno-0-1252" name="__codelineno-0-1252"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="n">ref</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1253" name="__codelineno-0-1253"></a> <span class="k">return</span> <span class="kc">None</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.sort_order" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">sort_order</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.sort_order" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the sort order of this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1203">1203</a></span> |
| <span class="normal"><a href="#__codelineno-0-1204">1204</a></span> |
| <span class="normal"><a href="#__codelineno-0-1205">1205</a></span> |
| <span class="normal"><a href="#__codelineno-0-1206">1206</a></span> |
| <span class="normal"><a href="#__codelineno-0-1207">1207</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1203" name="__codelineno-0-1203"></a><span class="k">def</span><span class="w"> </span><span class="nf">sort_order</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">SortOrder</span><span class="p">:</span> |
| <a id="__codelineno-0-1204" name="__codelineno-0-1204"></a><span class="w"> </span><span class="sd">"""Return the sort order of this table."""</span> |
| <a id="__codelineno-0-1205" name="__codelineno-0-1205"></a> <span class="k">return</span> <span class="nb">next</span><span class="p">(</span> |
| <a id="__codelineno-0-1206" name="__codelineno-0-1206"></a> <span class="n">sort_order</span> <span class="k">for</span> <span class="n">sort_order</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">sort_orders</span> <span class="k">if</span> <span class="n">sort_order</span><span class="o">.</span><span class="n">order_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">default_sort_order_id</span> |
| <a id="__codelineno-0-1207" name="__codelineno-0-1207"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.sort_orders" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">sort_orders</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.sort_orders" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return a dict of the sort orders of this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1209">1209</a></span> |
| <span class="normal"><a href="#__codelineno-0-1210">1210</a></span> |
| <span class="normal"><a href="#__codelineno-0-1211">1211</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1209" name="__codelineno-0-1209"></a><span class="k">def</span><span class="w"> </span><span class="nf">sort_orders</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">SortOrder</span><span class="p">]:</span> |
| <a id="__codelineno-0-1210" name="__codelineno-0-1210"></a><span class="w"> </span><span class="sd">"""Return a dict of the sort orders of this table."""</span> |
| <a id="__codelineno-0-1211" name="__codelineno-0-1211"></a> <span class="k">return</span> <span class="p">{</span><span class="n">sort_order</span><span class="o">.</span><span class="n">order_id</span><span class="p">:</span> <span class="n">sort_order</span> <span class="k">for</span> <span class="n">sort_order</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">sort_orders</span><span class="p">}</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.spec" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">spec</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.spec" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return the partition spec of this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1195">1195</a></span> |
| <span class="normal"><a href="#__codelineno-0-1196">1196</a></span> |
| <span class="normal"><a href="#__codelineno-0-1197">1197</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1195" name="__codelineno-0-1195"></a><span class="k">def</span><span class="w"> </span><span class="nf">spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">PartitionSpec</span><span class="p">:</span> |
| <a id="__codelineno-0-1196" name="__codelineno-0-1196"></a><span class="w"> </span><span class="sd">"""Return the partition spec of this table."""</span> |
| <a id="__codelineno-0-1197" name="__codelineno-0-1197"></a> <span class="k">return</span> <span class="nb">next</span><span class="p">(</span><span class="n">spec</span> <span class="k">for</span> <span class="n">spec</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">partition_specs</span> <span class="k">if</span> <span class="n">spec</span><span class="o">.</span><span class="n">spec_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">default_spec_id</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.specs" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">specs</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.specs" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Return a dict the partition specs this table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1199">1199</a></span> |
| <span class="normal"><a href="#__codelineno-0-1200">1200</a></span> |
| <span class="normal"><a href="#__codelineno-0-1201">1201</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1199" name="__codelineno-0-1199"></a><span class="k">def</span><span class="w"> </span><span class="nf">specs</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">PartitionSpec</span><span class="p">]:</span> |
| <a id="__codelineno-0-1200" name="__codelineno-0-1200"></a><span class="w"> </span><span class="sd">"""Return a dict the partition specs this table."""</span> |
| <a id="__codelineno-0-1201" name="__codelineno-0-1201"></a> <span class="k">return</span> <span class="p">{</span><span class="n">spec</span><span class="o">.</span><span class="n">spec_id</span><span class="p">:</span> <span class="n">spec</span> <span class="k">for</span> <span class="n">spec</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">partition_specs</span><span class="p">}</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.to_bodo" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_bodo</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.to_bodo" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Read a bodo DataFrame lazily from this Iceberg table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="bodo.pandas.DataFrame">DataFrame</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>bd.DataFrame: Unmaterialized Bodo Dataframe created from the Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1541">1541</a></span> |
| <span class="normal"><a href="#__codelineno-0-1542">1542</a></span> |
| <span class="normal"><a href="#__codelineno-0-1543">1543</a></span> |
| <span class="normal"><a href="#__codelineno-0-1544">1544</a></span> |
| <span class="normal"><a href="#__codelineno-0-1545">1545</a></span> |
| <span class="normal"><a href="#__codelineno-0-1546">1546</a></span> |
| <span class="normal"><a href="#__codelineno-0-1547">1547</a></span> |
| <span class="normal"><a href="#__codelineno-0-1548">1548</a></span> |
| <span class="normal"><a href="#__codelineno-0-1549">1549</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1541" name="__codelineno-0-1541"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_bodo</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">bd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-1542" name="__codelineno-0-1542"></a><span class="w"> </span><span class="sd">"""Read a bodo DataFrame lazily from this Iceberg table.</span> |
| <a id="__codelineno-0-1543" name="__codelineno-0-1543"></a> |
| <a id="__codelineno-0-1544" name="__codelineno-0-1544"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1545" name="__codelineno-0-1545"></a><span class="sd"> bd.DataFrame: Unmaterialized Bodo Dataframe created from the Iceberg table</span> |
| <a id="__codelineno-0-1546" name="__codelineno-0-1546"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1547" name="__codelineno-0-1547"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">bodo.pandas</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">bd</span> |
| <a id="__codelineno-0-1548" name="__codelineno-0-1548"></a> |
| <a id="__codelineno-0-1549" name="__codelineno-0-1549"></a> <span class="k">return</span> <span class="n">bd</span><span class="o">.</span><span class="n">read_iceberg_table</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.to_daft" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_daft</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.to_daft" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Read a Daft DataFrame lazily from this Iceberg table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="daft.DataFrame">DataFrame</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>daft.DataFrame: Unmaterialized Daft Dataframe created from the Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1531">1531</a></span> |
| <span class="normal"><a href="#__codelineno-0-1532">1532</a></span> |
| <span class="normal"><a href="#__codelineno-0-1533">1533</a></span> |
| <span class="normal"><a href="#__codelineno-0-1534">1534</a></span> |
| <span class="normal"><a href="#__codelineno-0-1535">1535</a></span> |
| <span class="normal"><a href="#__codelineno-0-1536">1536</a></span> |
| <span class="normal"><a href="#__codelineno-0-1537">1537</a></span> |
| <span class="normal"><a href="#__codelineno-0-1538">1538</a></span> |
| <span class="normal"><a href="#__codelineno-0-1539">1539</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1531" name="__codelineno-0-1531"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_daft</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">daft</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-1532" name="__codelineno-0-1532"></a><span class="w"> </span><span class="sd">"""Read a Daft DataFrame lazily from this Iceberg table.</span> |
| <a id="__codelineno-0-1533" name="__codelineno-0-1533"></a> |
| <a id="__codelineno-0-1534" name="__codelineno-0-1534"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1535" name="__codelineno-0-1535"></a><span class="sd"> daft.DataFrame: Unmaterialized Daft Dataframe created from the Iceberg table</span> |
| <a id="__codelineno-0-1536" name="__codelineno-0-1536"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1537" name="__codelineno-0-1537"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">daft</span> |
| <a id="__codelineno-0-1538" name="__codelineno-0-1538"></a> |
| <a id="__codelineno-0-1539" name="__codelineno-0-1539"></a> <span class="k">return</span> <span class="n">daft</span><span class="o">.</span><span class="n">read_iceberg</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.to_polars" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">to_polars</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.to_polars" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Lazily read from this Apache Iceberg table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="polars.LazyFrame">LazyFrame</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>pl.LazyFrame: Unmaterialized Polars LazyFrame created from the Iceberg table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1551">1551</a></span> |
| <span class="normal"><a href="#__codelineno-0-1552">1552</a></span> |
| <span class="normal"><a href="#__codelineno-0-1553">1553</a></span> |
| <span class="normal"><a href="#__codelineno-0-1554">1554</a></span> |
| <span class="normal"><a href="#__codelineno-0-1555">1555</a></span> |
| <span class="normal"><a href="#__codelineno-0-1556">1556</a></span> |
| <span class="normal"><a href="#__codelineno-0-1557">1557</a></span> |
| <span class="normal"><a href="#__codelineno-0-1558">1558</a></span> |
| <span class="normal"><a href="#__codelineno-0-1559">1559</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1551" name="__codelineno-0-1551"></a><span class="k">def</span><span class="w"> </span><span class="nf">to_polars</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pl</span><span class="o">.</span><span class="n">LazyFrame</span><span class="p">:</span> |
| <a id="__codelineno-0-1552" name="__codelineno-0-1552"></a><span class="w"> </span><span class="sd">"""Lazily read from this Apache Iceberg table.</span> |
| <a id="__codelineno-0-1553" name="__codelineno-0-1553"></a> |
| <a id="__codelineno-0-1554" name="__codelineno-0-1554"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1555" name="__codelineno-0-1555"></a><span class="sd"> pl.LazyFrame: Unmaterialized Polars LazyFrame created from the Iceberg table</span> |
| <a id="__codelineno-0-1556" name="__codelineno-0-1556"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1557" name="__codelineno-0-1557"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">polars</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pl</span> |
| <a id="__codelineno-0-1558" name="__codelineno-0-1558"></a> |
| <a id="__codelineno-0-1559" name="__codelineno-0-1559"></a> <span class="k">return</span> <span class="n">pl</span><span class="o">.</span><span class="n">scan_iceberg</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.transaction" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">transaction</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.transaction" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new transaction object to first stage the changes, and then commit them to the catalog.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Transaction (pyiceberg.table.Transaction)" href="#pyiceberg.table.Transaction">Transaction</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The transaction object</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1088">1088</a></span> |
| <span class="normal"><a href="#__codelineno-0-1089">1089</a></span> |
| <span class="normal"><a href="#__codelineno-0-1090">1090</a></span> |
| <span class="normal"><a href="#__codelineno-0-1091">1091</a></span> |
| <span class="normal"><a href="#__codelineno-0-1092">1092</a></span> |
| <span class="normal"><a href="#__codelineno-0-1093">1093</a></span> |
| <span class="normal"><a href="#__codelineno-0-1094">1094</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1088" name="__codelineno-0-1088"></a><span class="k">def</span><span class="w"> </span><span class="nf">transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-1089" name="__codelineno-0-1089"></a><span class="w"> </span><span class="sd">"""Create a new transaction object to first stage the changes, and then commit them to the catalog.</span> |
| <a id="__codelineno-0-1090" name="__codelineno-0-1090"></a> |
| <a id="__codelineno-0-1091" name="__codelineno-0-1091"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1092" name="__codelineno-0-1092"></a><span class="sd"> The transaction object</span> |
| <a id="__codelineno-0-1093" name="__codelineno-0-1093"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1094" name="__codelineno-0-1094"></a> <span class="k">return</span> <span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.update_schema" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_schema</span><span class="p">(</span><span class="n">allow_incompatible_changes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.update_schema" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateSchema to alter the columns of this table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>allow_incompatible_changes</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If changes are allowed that might break downstream consumers.</p> |
| </div> |
| </td> |
| <td> |
| <code>False</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If field names are case-sensitive.</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpdateSchema (pyiceberg.table.update.schema.UpdateSchema)" href="update/schema/#pyiceberg.table.update.schema.UpdateSchema">UpdateSchema</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateSchema.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1303">1303</a></span> |
| <span class="normal"><a href="#__codelineno-0-1304">1304</a></span> |
| <span class="normal"><a href="#__codelineno-0-1305">1305</a></span> |
| <span class="normal"><a href="#__codelineno-0-1306">1306</a></span> |
| <span class="normal"><a href="#__codelineno-0-1307">1307</a></span> |
| <span class="normal"><a href="#__codelineno-0-1308">1308</a></span> |
| <span class="normal"><a href="#__codelineno-0-1309">1309</a></span> |
| <span class="normal"><a href="#__codelineno-0-1310">1310</a></span> |
| <span class="normal"><a href="#__codelineno-0-1311">1311</a></span> |
| <span class="normal"><a href="#__codelineno-0-1312">1312</a></span> |
| <span class="normal"><a href="#__codelineno-0-1313">1313</a></span> |
| <span class="normal"><a href="#__codelineno-0-1314">1314</a></span> |
| <span class="normal"><a href="#__codelineno-0-1315">1315</a></span> |
| <span class="normal"><a href="#__codelineno-0-1316">1316</a></span> |
| <span class="normal"><a href="#__codelineno-0-1317">1317</a></span> |
| <span class="normal"><a href="#__codelineno-0-1318">1318</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1303" name="__codelineno-0-1303"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">allow_incompatible_changes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSchema</span><span class="p">:</span> |
| <a id="__codelineno-0-1304" name="__codelineno-0-1304"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSchema to alter the columns of this table.</span> |
| <a id="__codelineno-0-1305" name="__codelineno-0-1305"></a> |
| <a id="__codelineno-0-1306" name="__codelineno-0-1306"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1307" name="__codelineno-0-1307"></a><span class="sd"> allow_incompatible_changes: If changes are allowed that might break downstream consumers.</span> |
| <a id="__codelineno-0-1308" name="__codelineno-0-1308"></a><span class="sd"> case_sensitive: If field names are case-sensitive.</span> |
| <a id="__codelineno-0-1309" name="__codelineno-0-1309"></a> |
| <a id="__codelineno-0-1310" name="__codelineno-0-1310"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1311" name="__codelineno-0-1311"></a><span class="sd"> A new UpdateSchema.</span> |
| <a id="__codelineno-0-1312" name="__codelineno-0-1312"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1313" name="__codelineno-0-1313"></a> <span class="k">return</span> <span class="n">UpdateSchema</span><span class="p">(</span> |
| <a id="__codelineno-0-1314" name="__codelineno-0-1314"></a> <span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> |
| <a id="__codelineno-0-1315" name="__codelineno-0-1315"></a> <span class="n">allow_incompatible_changes</span><span class="o">=</span><span class="n">allow_incompatible_changes</span><span class="p">,</span> |
| <a id="__codelineno-0-1316" name="__codelineno-0-1316"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1317" name="__codelineno-0-1317"></a> <span class="n">name_mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">(),</span> |
| <a id="__codelineno-0-1318" name="__codelineno-0-1318"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.update_sort_order" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_sort_order</span><span class="p">(</span><span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.update_sort_order" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateSortOrder to update the sort order of this table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpdateSortOrder (pyiceberg.table.update.sorting.UpdateSortOrder)" href="update/sorting/#pyiceberg.table.update.sorting.UpdateSortOrder">UpdateSortOrder</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateSortOrder.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1320">1320</a></span> |
| <span class="normal"><a href="#__codelineno-0-1321">1321</a></span> |
| <span class="normal"><a href="#__codelineno-0-1322">1322</a></span> |
| <span class="normal"><a href="#__codelineno-0-1323">1323</a></span> |
| <span class="normal"><a href="#__codelineno-0-1324">1324</a></span> |
| <span class="normal"><a href="#__codelineno-0-1325">1325</a></span> |
| <span class="normal"><a href="#__codelineno-0-1326">1326</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1320" name="__codelineno-0-1320"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_sort_order</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSortOrder</span><span class="p">:</span> |
| <a id="__codelineno-0-1321" name="__codelineno-0-1321"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSortOrder to update the sort order of this table.</span> |
| <a id="__codelineno-0-1322" name="__codelineno-0-1322"></a> |
| <a id="__codelineno-0-1323" name="__codelineno-0-1323"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1324" name="__codelineno-0-1324"></a><span class="sd"> A new UpdateSortOrder.</span> |
| <a id="__codelineno-0-1325" name="__codelineno-0-1325"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1326" name="__codelineno-0-1326"></a> <span class="k">return</span> <span class="n">UpdateSortOrder</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.update_statistics" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_statistics</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Table.update_statistics" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand to run statistics management operations like add statistics and remove statistics.</p> |
| <p>Use table.update_statistics().<operation>().commit() to run a specific operation. |
| Use table.update_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.</p> |
| <p>Pending changes are applied on commit.</p> |
| <p>We can also use context managers to make more changes. For example:</p> |
| <p>with table.update_statistics() as update: |
| update.set_statistics(statistics_file=statistics_file) |
| update.remove_statistics(snapshot_id=2)</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1286">1286</a></span> |
| <span class="normal"><a href="#__codelineno-0-1287">1287</a></span> |
| <span class="normal"><a href="#__codelineno-0-1288">1288</a></span> |
| <span class="normal"><a href="#__codelineno-0-1289">1289</a></span> |
| <span class="normal"><a href="#__codelineno-0-1290">1290</a></span> |
| <span class="normal"><a href="#__codelineno-0-1291">1291</a></span> |
| <span class="normal"><a href="#__codelineno-0-1292">1292</a></span> |
| <span class="normal"><a href="#__codelineno-0-1293">1293</a></span> |
| <span class="normal"><a href="#__codelineno-0-1294">1294</a></span> |
| <span class="normal"><a href="#__codelineno-0-1295">1295</a></span> |
| <span class="normal"><a href="#__codelineno-0-1296">1296</a></span> |
| <span class="normal"><a href="#__codelineno-0-1297">1297</a></span> |
| <span class="normal"><a href="#__codelineno-0-1298">1298</a></span> |
| <span class="normal"><a href="#__codelineno-0-1299">1299</a></span> |
| <span class="normal"><a href="#__codelineno-0-1300">1300</a></span> |
| <span class="normal"><a href="#__codelineno-0-1301">1301</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1286" name="__codelineno-0-1286"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_statistics</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateStatistics</span><span class="p">:</span> |
| <a id="__codelineno-0-1287" name="__codelineno-0-1287"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-1288" name="__codelineno-0-1288"></a><span class="sd"> Shorthand to run statistics management operations like add statistics and remove statistics.</span> |
| <a id="__codelineno-0-1289" name="__codelineno-0-1289"></a> |
| <a id="__codelineno-0-1290" name="__codelineno-0-1290"></a><span class="sd"> Use table.update_statistics().<operation>().commit() to run a specific operation.</span> |
| <a id="__codelineno-0-1291" name="__codelineno-0-1291"></a><span class="sd"> Use table.update_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.</span> |
| <a id="__codelineno-0-1292" name="__codelineno-0-1292"></a> |
| <a id="__codelineno-0-1293" name="__codelineno-0-1293"></a><span class="sd"> Pending changes are applied on commit.</span> |
| <a id="__codelineno-0-1294" name="__codelineno-0-1294"></a> |
| <a id="__codelineno-0-1295" name="__codelineno-0-1295"></a><span class="sd"> We can also use context managers to make more changes. For example:</span> |
| <a id="__codelineno-0-1296" name="__codelineno-0-1296"></a> |
| <a id="__codelineno-0-1297" name="__codelineno-0-1297"></a><span class="sd"> with table.update_statistics() as update:</span> |
| <a id="__codelineno-0-1298" name="__codelineno-0-1298"></a><span class="sd"> update.set_statistics(statistics_file=statistics_file)</span> |
| <a id="__codelineno-0-1299" name="__codelineno-0-1299"></a><span class="sd"> update.remove_statistics(snapshot_id=2)</span> |
| <a id="__codelineno-0-1300" name="__codelineno-0-1300"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1301" name="__codelineno-0-1301"></a> <span class="k">return</span> <span class="n">UpdateStatistics</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="n">Transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Table.upsert" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">upsert</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">join_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">when_matched_update_all</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">when_not_matched_insert_all</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Table.upsert" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand API for performing an upsert to an iceberg table.</p> |
| <p>Args:</p> |
| <div class="highlight"><pre><span></span><code>df: The input dataframe to upsert with the table's data. |
| join_cols: Columns to join on, if not provided, it will use the identifier-field-ids. |
| when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing |
| when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any existing rows in the table |
| case_sensitive: Bool indicating if the match should be case-sensitive |
| branch: Branch Reference to run the upsert operation |
| |
| To learn more about the identifier-field-ids: https://iceberg.apache.org/spec/#identifier-field-ids |
| |
| Example Use Cases: |
| Case 1: Both Parameters = True (Full Upsert) |
| Existing row found → Update it |
| New row found → Insert it |
| |
| Case 2: when_matched_update_all = False, when_not_matched_insert_all = True |
| Existing row found → Do nothing (no updates) |
| New row found → Insert it |
| |
| Case 3: when_matched_update_all = True, when_not_matched_insert_all = False |
| Existing row found → Update it |
| New row found → Do nothing (no inserts) |
| |
| Case 4: Both Parameters = False (No Merge Effect) |
| Existing row found → Do nothing |
| New row found → Do nothing |
| (Function effectively does nothing) |
| </code></pre></div> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpsertResult |
| |
| |
| |
| dataclass |
| (pyiceberg.table.UpsertResult)" href="#pyiceberg.table.UpsertResult">UpsertResult</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>An UpsertResult class (contains details of rows updated and inserted)</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1332">1332</a></span> |
| <span class="normal"><a href="#__codelineno-0-1333">1333</a></span> |
| <span class="normal"><a href="#__codelineno-0-1334">1334</a></span> |
| <span class="normal"><a href="#__codelineno-0-1335">1335</a></span> |
| <span class="normal"><a href="#__codelineno-0-1336">1336</a></span> |
| <span class="normal"><a href="#__codelineno-0-1337">1337</a></span> |
| <span class="normal"><a href="#__codelineno-0-1338">1338</a></span> |
| <span class="normal"><a href="#__codelineno-0-1339">1339</a></span> |
| <span class="normal"><a href="#__codelineno-0-1340">1340</a></span> |
| <span class="normal"><a href="#__codelineno-0-1341">1341</a></span> |
| <span class="normal"><a href="#__codelineno-0-1342">1342</a></span> |
| <span class="normal"><a href="#__codelineno-0-1343">1343</a></span> |
| <span class="normal"><a href="#__codelineno-0-1344">1344</a></span> |
| <span class="normal"><a href="#__codelineno-0-1345">1345</a></span> |
| <span class="normal"><a href="#__codelineno-0-1346">1346</a></span> |
| <span class="normal"><a href="#__codelineno-0-1347">1347</a></span> |
| <span class="normal"><a href="#__codelineno-0-1348">1348</a></span> |
| <span class="normal"><a href="#__codelineno-0-1349">1349</a></span> |
| <span class="normal"><a href="#__codelineno-0-1350">1350</a></span> |
| <span class="normal"><a href="#__codelineno-0-1351">1351</a></span> |
| <span class="normal"><a href="#__codelineno-0-1352">1352</a></span> |
| <span class="normal"><a href="#__codelineno-0-1353">1353</a></span> |
| <span class="normal"><a href="#__codelineno-0-1354">1354</a></span> |
| <span class="normal"><a href="#__codelineno-0-1355">1355</a></span> |
| <span class="normal"><a href="#__codelineno-0-1356">1356</a></span> |
| <span class="normal"><a href="#__codelineno-0-1357">1357</a></span> |
| <span class="normal"><a href="#__codelineno-0-1358">1358</a></span> |
| <span class="normal"><a href="#__codelineno-0-1359">1359</a></span> |
| <span class="normal"><a href="#__codelineno-0-1360">1360</a></span> |
| <span class="normal"><a href="#__codelineno-0-1361">1361</a></span> |
| <span class="normal"><a href="#__codelineno-0-1362">1362</a></span> |
| <span class="normal"><a href="#__codelineno-0-1363">1363</a></span> |
| <span class="normal"><a href="#__codelineno-0-1364">1364</a></span> |
| <span class="normal"><a href="#__codelineno-0-1365">1365</a></span> |
| <span class="normal"><a href="#__codelineno-0-1366">1366</a></span> |
| <span class="normal"><a href="#__codelineno-0-1367">1367</a></span> |
| <span class="normal"><a href="#__codelineno-0-1368">1368</a></span> |
| <span class="normal"><a href="#__codelineno-0-1369">1369</a></span> |
| <span class="normal"><a href="#__codelineno-0-1370">1370</a></span> |
| <span class="normal"><a href="#__codelineno-0-1371">1371</a></span> |
| <span class="normal"><a href="#__codelineno-0-1372">1372</a></span> |
| <span class="normal"><a href="#__codelineno-0-1373">1373</a></span> |
| <span class="normal"><a href="#__codelineno-0-1374">1374</a></span> |
| <span class="normal"><a href="#__codelineno-0-1375">1375</a></span> |
| <span class="normal"><a href="#__codelineno-0-1376">1376</a></span> |
| <span class="normal"><a href="#__codelineno-0-1377">1377</a></span> |
| <span class="normal"><a href="#__codelineno-0-1378">1378</a></span> |
| <span class="normal"><a href="#__codelineno-0-1379">1379</a></span> |
| <span class="normal"><a href="#__codelineno-0-1380">1380</a></span> |
| <span class="normal"><a href="#__codelineno-0-1381">1381</a></span> |
| <span class="normal"><a href="#__codelineno-0-1382">1382</a></span> |
| <span class="normal"><a href="#__codelineno-0-1383">1383</a></span> |
| <span class="normal"><a href="#__codelineno-0-1384">1384</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1332" name="__codelineno-0-1332"></a><span class="k">def</span><span class="w"> </span><span class="nf">upsert</span><span class="p">(</span> |
| <a id="__codelineno-0-1333" name="__codelineno-0-1333"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1334" name="__codelineno-0-1334"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-1335" name="__codelineno-0-1335"></a> <span class="n">join_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1336" name="__codelineno-0-1336"></a> <span class="n">when_matched_update_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1337" name="__codelineno-0-1337"></a> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1338" name="__codelineno-0-1338"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1339" name="__codelineno-0-1339"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-1340" name="__codelineno-0-1340"></a><span class="p">)</span> <span class="o">-></span> <span class="n">UpsertResult</span><span class="p">:</span> |
| <a id="__codelineno-0-1341" name="__codelineno-0-1341"></a><span class="w"> </span><span class="sd">"""Shorthand API for performing an upsert to an iceberg table.</span> |
| <a id="__codelineno-0-1342" name="__codelineno-0-1342"></a> |
| <a id="__codelineno-0-1343" name="__codelineno-0-1343"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-1344" name="__codelineno-0-1344"></a> |
| <a id="__codelineno-0-1345" name="__codelineno-0-1345"></a><span class="sd"> df: The input dataframe to upsert with the table's data.</span> |
| <a id="__codelineno-0-1346" name="__codelineno-0-1346"></a><span class="sd"> join_cols: Columns to join on, if not provided, it will use the identifier-field-ids.</span> |
| <a id="__codelineno-0-1347" name="__codelineno-0-1347"></a><span class="sd"> when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing</span> |
| <a id="__codelineno-0-1348" name="__codelineno-0-1348"></a><span class="sd"> when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any existing rows in the table</span> |
| <a id="__codelineno-0-1349" name="__codelineno-0-1349"></a><span class="sd"> case_sensitive: Bool indicating if the match should be case-sensitive</span> |
| <a id="__codelineno-0-1350" name="__codelineno-0-1350"></a><span class="sd"> branch: Branch Reference to run the upsert operation</span> |
| <a id="__codelineno-0-1351" name="__codelineno-0-1351"></a> |
| <a id="__codelineno-0-1352" name="__codelineno-0-1352"></a><span class="sd"> To learn more about the identifier-field-ids: https://iceberg.apache.org/spec/#identifier-field-ids</span> |
| <a id="__codelineno-0-1353" name="__codelineno-0-1353"></a> |
| <a id="__codelineno-0-1354" name="__codelineno-0-1354"></a><span class="sd"> Example Use Cases:</span> |
| <a id="__codelineno-0-1355" name="__codelineno-0-1355"></a><span class="sd"> Case 1: Both Parameters = True (Full Upsert)</span> |
| <a id="__codelineno-0-1356" name="__codelineno-0-1356"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-1357" name="__codelineno-0-1357"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-1358" name="__codelineno-0-1358"></a> |
| <a id="__codelineno-0-1359" name="__codelineno-0-1359"></a><span class="sd"> Case 2: when_matched_update_all = False, when_not_matched_insert_all = True</span> |
| <a id="__codelineno-0-1360" name="__codelineno-0-1360"></a><span class="sd"> Existing row found → Do nothing (no updates)</span> |
| <a id="__codelineno-0-1361" name="__codelineno-0-1361"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-1362" name="__codelineno-0-1362"></a> |
| <a id="__codelineno-0-1363" name="__codelineno-0-1363"></a><span class="sd"> Case 3: when_matched_update_all = True, when_not_matched_insert_all = False</span> |
| <a id="__codelineno-0-1364" name="__codelineno-0-1364"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-1365" name="__codelineno-0-1365"></a><span class="sd"> New row found → Do nothing (no inserts)</span> |
| <a id="__codelineno-0-1366" name="__codelineno-0-1366"></a> |
| <a id="__codelineno-0-1367" name="__codelineno-0-1367"></a><span class="sd"> Case 4: Both Parameters = False (No Merge Effect)</span> |
| <a id="__codelineno-0-1368" name="__codelineno-0-1368"></a><span class="sd"> Existing row found → Do nothing</span> |
| <a id="__codelineno-0-1369" name="__codelineno-0-1369"></a><span class="sd"> New row found → Do nothing</span> |
| <a id="__codelineno-0-1370" name="__codelineno-0-1370"></a><span class="sd"> (Function effectively does nothing)</span> |
| <a id="__codelineno-0-1371" name="__codelineno-0-1371"></a> |
| <a id="__codelineno-0-1372" name="__codelineno-0-1372"></a> |
| <a id="__codelineno-0-1373" name="__codelineno-0-1373"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-1374" name="__codelineno-0-1374"></a><span class="sd"> An UpsertResult class (contains details of rows updated and inserted)</span> |
| <a id="__codelineno-0-1375" name="__codelineno-0-1375"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-1376" name="__codelineno-0-1376"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">transaction</span><span class="p">()</span> <span class="k">as</span> <span class="n">tx</span><span class="p">:</span> |
| <a id="__codelineno-0-1377" name="__codelineno-0-1377"></a> <span class="k">return</span> <span class="n">tx</span><span class="o">.</span><span class="n">upsert</span><span class="p">(</span> |
| <a id="__codelineno-0-1378" name="__codelineno-0-1378"></a> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> |
| <a id="__codelineno-0-1379" name="__codelineno-0-1379"></a> <span class="n">join_cols</span><span class="o">=</span><span class="n">join_cols</span><span class="p">,</span> |
| <a id="__codelineno-0-1380" name="__codelineno-0-1380"></a> <span class="n">when_matched_update_all</span><span class="o">=</span><span class="n">when_matched_update_all</span><span class="p">,</span> |
| <a id="__codelineno-0-1381" name="__codelineno-0-1381"></a> <span class="n">when_not_matched_insert_all</span><span class="o">=</span><span class="n">when_not_matched_insert_all</span><span class="p">,</span> |
| <a id="__codelineno-0-1382" name="__codelineno-0-1382"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-1383" name="__codelineno-0-1383"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-1384" name="__codelineno-0-1384"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.TableIdentifier" class="doc doc-heading"> |
| <code>TableIdentifier</code> |
| |
| |
| <a href="#pyiceberg.table.TableIdentifier" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><a class="autorefs autorefs-internal" title="IcebergBaseModel (pyiceberg.typedef.IcebergBaseModel)" href="../typedef/#pyiceberg.typedef.IcebergBaseModel">IcebergBaseModel</a></code></p> |
| |
| |
| <p>Fully Qualified identifier to a table.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1040">1040</a></span> |
| <span class="normal"><a href="#__codelineno-0-1041">1041</a></span> |
| <span class="normal"><a href="#__codelineno-0-1042">1042</a></span> |
| <span class="normal"><a href="#__codelineno-0-1043">1043</a></span> |
| <span class="normal"><a href="#__codelineno-0-1044">1044</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1040" name="__codelineno-0-1040"></a><span class="k">class</span><span class="w"> </span><span class="nc">TableIdentifier</span><span class="p">(</span><span class="n">IcebergBaseModel</span><span class="p">):</span> |
| <a id="__codelineno-0-1041" name="__codelineno-0-1041"></a><span class="w"> </span><span class="sd">"""Fully Qualified identifier to a table."""</span> |
| <a id="__codelineno-0-1042" name="__codelineno-0-1042"></a> |
| <a id="__codelineno-0-1043" name="__codelineno-0-1043"></a> <span class="n">namespace</span><span class="p">:</span> <span class="n">Namespace</span> |
| <a id="__codelineno-0-1044" name="__codelineno-0-1044"></a> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.TableScan" class="doc doc-heading"> |
| <code>TableScan</code> |
| |
| |
| <a href="#pyiceberg.table.TableScan" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| <p class="doc doc-class-bases"> |
| Bases: <code><span title="abc.ABC">ABC</span></code></p> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1688">1688</a></span> |
| <span class="normal"><a href="#__codelineno-0-1689">1689</a></span> |
| <span class="normal"><a href="#__codelineno-0-1690">1690</a></span> |
| <span class="normal"><a href="#__codelineno-0-1691">1691</a></span> |
| <span class="normal"><a href="#__codelineno-0-1692">1692</a></span> |
| <span class="normal"><a href="#__codelineno-0-1693">1693</a></span> |
| <span class="normal"><a href="#__codelineno-0-1694">1694</a></span> |
| <span class="normal"><a href="#__codelineno-0-1695">1695</a></span> |
| <span class="normal"><a href="#__codelineno-0-1696">1696</a></span> |
| <span class="normal"><a href="#__codelineno-0-1697">1697</a></span> |
| <span class="normal"><a href="#__codelineno-0-1698">1698</a></span> |
| <span class="normal"><a href="#__codelineno-0-1699">1699</a></span> |
| <span class="normal"><a href="#__codelineno-0-1700">1700</a></span> |
| <span class="normal"><a href="#__codelineno-0-1701">1701</a></span> |
| <span class="normal"><a href="#__codelineno-0-1702">1702</a></span> |
| <span class="normal"><a href="#__codelineno-0-1703">1703</a></span> |
| <span class="normal"><a href="#__codelineno-0-1704">1704</a></span> |
| <span class="normal"><a href="#__codelineno-0-1705">1705</a></span> |
| <span class="normal"><a href="#__codelineno-0-1706">1706</a></span> |
| <span class="normal"><a href="#__codelineno-0-1707">1707</a></span> |
| <span class="normal"><a href="#__codelineno-0-1708">1708</a></span> |
| <span class="normal"><a href="#__codelineno-0-1709">1709</a></span> |
| <span class="normal"><a href="#__codelineno-0-1710">1710</a></span> |
| <span class="normal"><a href="#__codelineno-0-1711">1711</a></span> |
| <span class="normal"><a href="#__codelineno-0-1712">1712</a></span> |
| <span class="normal"><a href="#__codelineno-0-1713">1713</a></span> |
| <span class="normal"><a href="#__codelineno-0-1714">1714</a></span> |
| <span class="normal"><a href="#__codelineno-0-1715">1715</a></span> |
| <span class="normal"><a href="#__codelineno-0-1716">1716</a></span> |
| <span class="normal"><a href="#__codelineno-0-1717">1717</a></span> |
| <span class="normal"><a href="#__codelineno-0-1718">1718</a></span> |
| <span class="normal"><a href="#__codelineno-0-1719">1719</a></span> |
| <span class="normal"><a href="#__codelineno-0-1720">1720</a></span> |
| <span class="normal"><a href="#__codelineno-0-1721">1721</a></span> |
| <span class="normal"><a href="#__codelineno-0-1722">1722</a></span> |
| <span class="normal"><a href="#__codelineno-0-1723">1723</a></span> |
| <span class="normal"><a href="#__codelineno-0-1724">1724</a></span> |
| <span class="normal"><a href="#__codelineno-0-1725">1725</a></span> |
| <span class="normal"><a href="#__codelineno-0-1726">1726</a></span> |
| <span class="normal"><a href="#__codelineno-0-1727">1727</a></span> |
| <span class="normal"><a href="#__codelineno-0-1728">1728</a></span> |
| <span class="normal"><a href="#__codelineno-0-1729">1729</a></span> |
| <span class="normal"><a href="#__codelineno-0-1730">1730</a></span> |
| <span class="normal"><a href="#__codelineno-0-1731">1731</a></span> |
| <span class="normal"><a href="#__codelineno-0-1732">1732</a></span> |
| <span class="normal"><a href="#__codelineno-0-1733">1733</a></span> |
| <span class="normal"><a href="#__codelineno-0-1734">1734</a></span> |
| <span class="normal"><a href="#__codelineno-0-1735">1735</a></span> |
| <span class="normal"><a href="#__codelineno-0-1736">1736</a></span> |
| <span class="normal"><a href="#__codelineno-0-1737">1737</a></span> |
| <span class="normal"><a href="#__codelineno-0-1738">1738</a></span> |
| <span class="normal"><a href="#__codelineno-0-1739">1739</a></span> |
| <span class="normal"><a href="#__codelineno-0-1740">1740</a></span> |
| <span class="normal"><a href="#__codelineno-0-1741">1741</a></span> |
| <span class="normal"><a href="#__codelineno-0-1742">1742</a></span> |
| <span class="normal"><a href="#__codelineno-0-1743">1743</a></span> |
| <span class="normal"><a href="#__codelineno-0-1744">1744</a></span> |
| <span class="normal"><a href="#__codelineno-0-1745">1745</a></span> |
| <span class="normal"><a href="#__codelineno-0-1746">1746</a></span> |
| <span class="normal"><a href="#__codelineno-0-1747">1747</a></span> |
| <span class="normal"><a href="#__codelineno-0-1748">1748</a></span> |
| <span class="normal"><a href="#__codelineno-0-1749">1749</a></span> |
| <span class="normal"><a href="#__codelineno-0-1750">1750</a></span> |
| <span class="normal"><a href="#__codelineno-0-1751">1751</a></span> |
| <span class="normal"><a href="#__codelineno-0-1752">1752</a></span> |
| <span class="normal"><a href="#__codelineno-0-1753">1753</a></span> |
| <span class="normal"><a href="#__codelineno-0-1754">1754</a></span> |
| <span class="normal"><a href="#__codelineno-0-1755">1755</a></span> |
| <span class="normal"><a href="#__codelineno-0-1756">1756</a></span> |
| <span class="normal"><a href="#__codelineno-0-1757">1757</a></span> |
| <span class="normal"><a href="#__codelineno-0-1758">1758</a></span> |
| <span class="normal"><a href="#__codelineno-0-1759">1759</a></span> |
| <span class="normal"><a href="#__codelineno-0-1760">1760</a></span> |
| <span class="normal"><a href="#__codelineno-0-1761">1761</a></span> |
| <span class="normal"><a href="#__codelineno-0-1762">1762</a></span> |
| <span class="normal"><a href="#__codelineno-0-1763">1763</a></span> |
| <span class="normal"><a href="#__codelineno-0-1764">1764</a></span> |
| <span class="normal"><a href="#__codelineno-0-1765">1765</a></span> |
| <span class="normal"><a href="#__codelineno-0-1766">1766</a></span> |
| <span class="normal"><a href="#__codelineno-0-1767">1767</a></span> |
| <span class="normal"><a href="#__codelineno-0-1768">1768</a></span> |
| <span class="normal"><a href="#__codelineno-0-1769">1769</a></span> |
| <span class="normal"><a href="#__codelineno-0-1770">1770</a></span> |
| <span class="normal"><a href="#__codelineno-0-1771">1771</a></span> |
| <span class="normal"><a href="#__codelineno-0-1772">1772</a></span> |
| <span class="normal"><a href="#__codelineno-0-1773">1773</a></span> |
| <span class="normal"><a href="#__codelineno-0-1774">1774</a></span> |
| <span class="normal"><a href="#__codelineno-0-1775">1775</a></span> |
| <span class="normal"><a href="#__codelineno-0-1776">1776</a></span> |
| <span class="normal"><a href="#__codelineno-0-1777">1777</a></span> |
| <span class="normal"><a href="#__codelineno-0-1778">1778</a></span> |
| <span class="normal"><a href="#__codelineno-0-1779">1779</a></span> |
| <span class="normal"><a href="#__codelineno-0-1780">1780</a></span> |
| <span class="normal"><a href="#__codelineno-0-1781">1781</a></span> |
| <span class="normal"><a href="#__codelineno-0-1782">1782</a></span> |
| <span class="normal"><a href="#__codelineno-0-1783">1783</a></span> |
| <span class="normal"><a href="#__codelineno-0-1784">1784</a></span> |
| <span class="normal"><a href="#__codelineno-0-1785">1785</a></span> |
| <span class="normal"><a href="#__codelineno-0-1786">1786</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1688" name="__codelineno-0-1688"></a><span class="k">class</span><span class="w"> </span><span class="nc">TableScan</span><span class="p">(</span><span class="n">ABC</span><span class="p">):</span> |
| <a id="__codelineno-0-1689" name="__codelineno-0-1689"></a> <span class="n">table_metadata</span><span class="p">:</span> <span class="n">TableMetadata</span> |
| <a id="__codelineno-0-1690" name="__codelineno-0-1690"></a> <span class="n">io</span><span class="p">:</span> <span class="n">FileIO</span> |
| <a id="__codelineno-0-1691" name="__codelineno-0-1691"></a> <span class="n">row_filter</span><span class="p">:</span> <span class="n">BooleanExpression</span> |
| <a id="__codelineno-0-1692" name="__codelineno-0-1692"></a> <span class="n">selected_fields</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> |
| <a id="__codelineno-0-1693" name="__codelineno-0-1693"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> |
| <a id="__codelineno-0-1694" name="__codelineno-0-1694"></a> <span class="n">snapshot_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> |
| <a id="__codelineno-0-1695" name="__codelineno-0-1695"></a> <span class="n">options</span><span class="p">:</span> <span class="n">Properties</span> |
| <a id="__codelineno-0-1696" name="__codelineno-0-1696"></a> <span class="n">limit</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> |
| <a id="__codelineno-0-1697" name="__codelineno-0-1697"></a> |
| <a id="__codelineno-0-1698" name="__codelineno-0-1698"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span> |
| <a id="__codelineno-0-1699" name="__codelineno-0-1699"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-1700" name="__codelineno-0-1700"></a> <span class="n">table_metadata</span><span class="p">:</span> <span class="n">TableMetadata</span><span class="p">,</span> |
| <a id="__codelineno-0-1701" name="__codelineno-0-1701"></a> <span class="n">io</span><span class="p">:</span> <span class="n">FileIO</span><span class="p">,</span> |
| <a id="__codelineno-0-1702" name="__codelineno-0-1702"></a> <span class="n">row_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-1703" name="__codelineno-0-1703"></a> <span class="n">selected_fields</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"*"</span><span class="p">,),</span> |
| <a id="__codelineno-0-1704" name="__codelineno-0-1704"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-1705" name="__codelineno-0-1705"></a> <span class="n">snapshot_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1706" name="__codelineno-0-1706"></a> <span class="n">options</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-1707" name="__codelineno-0-1707"></a> <span class="n">limit</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-1708" name="__codelineno-0-1708"></a> <span class="p">):</span> |
| <a id="__codelineno-0-1709" name="__codelineno-0-1709"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span> <span class="o">=</span> <span class="n">table_metadata</span> |
| <a id="__codelineno-0-1710" name="__codelineno-0-1710"></a> <span class="bp">self</span><span class="o">.</span><span class="n">io</span> <span class="o">=</span> <span class="n">io</span> |
| <a id="__codelineno-0-1711" name="__codelineno-0-1711"></a> <span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span> <span class="o">=</span> <span class="n">_parse_row_filter</span><span class="p">(</span><span class="n">row_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-1712" name="__codelineno-0-1712"></a> <span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span> <span class="o">=</span> <span class="n">selected_fields</span> |
| <a id="__codelineno-0-1713" name="__codelineno-0-1713"></a> <span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span> <span class="o">=</span> <span class="n">case_sensitive</span> |
| <a id="__codelineno-0-1714" name="__codelineno-0-1714"></a> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span> <span class="o">=</span> <span class="n">snapshot_id</span> |
| <a id="__codelineno-0-1715" name="__codelineno-0-1715"></a> <span class="bp">self</span><span class="o">.</span><span class="n">options</span> <span class="o">=</span> <span class="n">options</span> |
| <a id="__codelineno-0-1716" name="__codelineno-0-1716"></a> <span class="bp">self</span><span class="o">.</span><span class="n">limit</span> <span class="o">=</span> <span class="n">limit</span> |
| <a id="__codelineno-0-1717" name="__codelineno-0-1717"></a> |
| <a id="__codelineno-0-1718" name="__codelineno-0-1718"></a> <span class="k">def</span><span class="w"> </span><span class="nf">snapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">Snapshot</span><span class="p">]:</span> |
| <a id="__codelineno-0-1719" name="__codelineno-0-1719"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">:</span> |
| <a id="__codelineno-0-1720" name="__codelineno-0-1720"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1721" name="__codelineno-0-1721"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">current_snapshot</span><span class="p">()</span> |
| <a id="__codelineno-0-1722" name="__codelineno-0-1722"></a> |
| <a id="__codelineno-0-1723" name="__codelineno-0-1723"></a> <span class="k">def</span><span class="w"> </span><span class="nf">projection</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Schema</span><span class="p">:</span> |
| <a id="__codelineno-0-1724" name="__codelineno-0-1724"></a> <span class="n">current_schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span> |
| <a id="__codelineno-0-1725" name="__codelineno-0-1725"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1726" name="__codelineno-0-1726"></a> <span class="n">snapshot</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">snapshot_by_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1727" name="__codelineno-0-1727"></a> <span class="k">if</span> <span class="n">snapshot</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1728" name="__codelineno-0-1728"></a> <span class="k">if</span> <span class="n">snapshot</span><span class="o">.</span><span class="n">schema_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-1729" name="__codelineno-0-1729"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-1730" name="__codelineno-0-1730"></a> <span class="n">current_schema</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span> |
| <a id="__codelineno-0-1731" name="__codelineno-0-1731"></a> <span class="n">schema</span> <span class="k">for</span> <span class="n">schema</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schemas</span> <span class="k">if</span> <span class="n">schema</span><span class="o">.</span><span class="n">schema_id</span> <span class="o">==</span> <span class="n">snapshot</span><span class="o">.</span><span class="n">schema_id</span> |
| <a id="__codelineno-0-1732" name="__codelineno-0-1732"></a> <span class="p">)</span> |
| <a id="__codelineno-0-1733" name="__codelineno-0-1733"></a> <span class="k">except</span> <span class="ne">StopIteration</span><span class="p">:</span> |
| <a id="__codelineno-0-1734" name="__codelineno-0-1734"></a> <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Metadata does not contain schema with id: </span><span class="si">{</span><span class="n">snapshot</span><span class="o">.</span><span class="n">schema_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-1735" name="__codelineno-0-1735"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-1736" name="__codelineno-0-1736"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Snapshot not found: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-1737" name="__codelineno-0-1737"></a> |
| <a id="__codelineno-0-1738" name="__codelineno-0-1738"></a> <span class="k">if</span> <span class="s2">"*"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span><span class="p">:</span> |
| <a id="__codelineno-0-1739" name="__codelineno-0-1739"></a> <span class="k">return</span> <span class="n">current_schema</span> |
| <a id="__codelineno-0-1740" name="__codelineno-0-1740"></a> |
| <a id="__codelineno-0-1741" name="__codelineno-0-1741"></a> <span class="k">return</span> <span class="n">current_schema</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-1742" name="__codelineno-0-1742"></a> |
| <a id="__codelineno-0-1743" name="__codelineno-0-1743"></a> <span class="nd">@abstractmethod</span> |
| <a id="__codelineno-0-1744" name="__codelineno-0-1744"></a> <span class="k">def</span><span class="w"> </span><span class="nf">plan_files</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">ScanTask</span><span class="p">]:</span> <span class="o">...</span> |
| <a id="__codelineno-0-1745" name="__codelineno-0-1745"></a> |
| <a id="__codelineno-0-1746" name="__codelineno-0-1746"></a> <span class="nd">@abstractmethod</span> |
| <a id="__codelineno-0-1747" name="__codelineno-0-1747"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_arrow</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">:</span> <span class="o">...</span> |
| <a id="__codelineno-0-1748" name="__codelineno-0-1748"></a> |
| <a id="__codelineno-0-1749" name="__codelineno-0-1749"></a> <span class="nd">@abstractmethod</span> |
| <a id="__codelineno-0-1750" name="__codelineno-0-1750"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_pandas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> <span class="o">...</span> |
| <a id="__codelineno-0-1751" name="__codelineno-0-1751"></a> |
| <a id="__codelineno-0-1752" name="__codelineno-0-1752"></a> <span class="nd">@abstractmethod</span> |
| <a id="__codelineno-0-1753" name="__codelineno-0-1753"></a> <span class="k">def</span><span class="w"> </span><span class="nf">to_polars</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pl</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> <span class="o">...</span> |
| <a id="__codelineno-0-1754" name="__codelineno-0-1754"></a> |
| <a id="__codelineno-0-1755" name="__codelineno-0-1755"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">S</span><span class="p">,</span> <span class="o">**</span><span class="n">overrides</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">S</span><span class="p">:</span> |
| <a id="__codelineno-0-1756" name="__codelineno-0-1756"></a><span class="w"> </span><span class="sd">"""Create a copy of this table scan with updated fields."""</span> |
| <a id="__codelineno-0-1757" name="__codelineno-0-1757"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">inspect</span><span class="w"> </span><span class="kn">import</span> <span class="n">signature</span> |
| <a id="__codelineno-0-1758" name="__codelineno-0-1758"></a> |
| <a id="__codelineno-0-1759" name="__codelineno-0-1759"></a> <span class="c1"># Extract those attributes that are constructor parameters. We don't use self.__dict__ as the kwargs to the</span> |
| <a id="__codelineno-0-1760" name="__codelineno-0-1760"></a> <span class="c1"># constructors because it may contain additional attributes that are not part of the constructor signature.</span> |
| <a id="__codelineno-0-1761" name="__codelineno-0-1761"></a> <span class="n">params</span> <span class="o">=</span> <span class="n">signature</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">)</span><span class="o">.</span><span class="n">parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="o">-</span> <span class="p">{</span><span class="s2">"self"</span><span class="p">}</span> <span class="c1"># Skip "self" parameter</span> |
| <a id="__codelineno-0-1762" name="__codelineno-0-1762"></a> <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="n">param</span><span class="p">:</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">param</span><span class="p">)</span> <span class="k">for</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">params</span><span class="p">}</span> <span class="c1"># Assume parameters are attributes</span> |
| <a id="__codelineno-0-1763" name="__codelineno-0-1763"></a> |
| <a id="__codelineno-0-1764" name="__codelineno-0-1764"></a> <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)(</span><span class="o">**</span><span class="p">{</span><span class="o">**</span><span class="n">kwargs</span><span class="p">,</span> <span class="o">**</span><span class="n">overrides</span><span class="p">})</span> |
| <a id="__codelineno-0-1765" name="__codelineno-0-1765"></a> |
| <a id="__codelineno-0-1766" name="__codelineno-0-1766"></a> <span class="k">def</span><span class="w"> </span><span class="nf">use_ref</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">S</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">S</span><span class="p">:</span> |
| <a id="__codelineno-0-1767" name="__codelineno-0-1767"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">:</span> |
| <a id="__codelineno-0-1768" name="__codelineno-0-1768"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot override ref, already set snapshot id=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">snapshot_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-1769" name="__codelineno-0-1769"></a> <span class="k">if</span> <span class="n">snapshot</span> <span class="o">:=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">snapshot_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">):</span> |
| <a id="__codelineno-0-1770" name="__codelineno-0-1770"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">snapshot_id</span><span class="o">=</span><span class="n">snapshot</span><span class="o">.</span><span class="n">snapshot_id</span><span class="p">)</span> |
| <a id="__codelineno-0-1771" name="__codelineno-0-1771"></a> |
| <a id="__codelineno-0-1772" name="__codelineno-0-1772"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot scan unknown ref=</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-1773" name="__codelineno-0-1773"></a> |
| <a id="__codelineno-0-1774" name="__codelineno-0-1774"></a> <span class="k">def</span><span class="w"> </span><span class="nf">select</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">S</span><span class="p">,</span> <span class="o">*</span><span class="n">field_names</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">S</span><span class="p">:</span> |
| <a id="__codelineno-0-1775" name="__codelineno-0-1775"></a> <span class="k">if</span> <span class="s2">"*"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span><span class="p">:</span> |
| <a id="__codelineno-0-1776" name="__codelineno-0-1776"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">selected_fields</span><span class="o">=</span><span class="n">field_names</span><span class="p">)</span> |
| <a id="__codelineno-0-1777" name="__codelineno-0-1777"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">selected_fields</span><span class="o">=</span><span class="nb">tuple</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">selected_fields</span><span class="p">)</span><span class="o">.</span><span class="n">intersection</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">field_names</span><span class="p">))))</span> |
| <a id="__codelineno-0-1778" name="__codelineno-0-1778"></a> |
| <a id="__codelineno-0-1779" name="__codelineno-0-1779"></a> <span class="k">def</span><span class="w"> </span><span class="nf">filter</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">S</span><span class="p">,</span> <span class="n">expr</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">])</span> <span class="o">-></span> <span class="n">S</span><span class="p">:</span> |
| <a id="__codelineno-0-1780" name="__codelineno-0-1780"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">row_filter</span><span class="o">=</span><span class="n">And</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">row_filter</span><span class="p">,</span> <span class="n">_parse_row_filter</span><span class="p">(</span><span class="n">expr</span><span class="p">)))</span> |
| <a id="__codelineno-0-1781" name="__codelineno-0-1781"></a> |
| <a id="__codelineno-0-1782" name="__codelineno-0-1782"></a> <span class="k">def</span><span class="w"> </span><span class="nf">with_case_sensitive</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">S</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">S</span><span class="p">:</span> |
| <a id="__codelineno-0-1783" name="__codelineno-0-1783"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-1784" name="__codelineno-0-1784"></a> |
| <a id="__codelineno-0-1785" name="__codelineno-0-1785"></a> <span class="nd">@abstractmethod</span> |
| <a id="__codelineno-0-1786" name="__codelineno-0-1786"></a> <span class="k">def</span><span class="w"> </span><span class="nf">count</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> <span class="o">...</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.TableScan.update" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update</span><span class="p">(</span><span class="o">**</span><span class="n">overrides</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.TableScan.update" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a copy of this table scan with updated fields.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1755">1755</a></span> |
| <span class="normal"><a href="#__codelineno-0-1756">1756</a></span> |
| <span class="normal"><a href="#__codelineno-0-1757">1757</a></span> |
| <span class="normal"><a href="#__codelineno-0-1758">1758</a></span> |
| <span class="normal"><a href="#__codelineno-0-1759">1759</a></span> |
| <span class="normal"><a href="#__codelineno-0-1760">1760</a></span> |
| <span class="normal"><a href="#__codelineno-0-1761">1761</a></span> |
| <span class="normal"><a href="#__codelineno-0-1762">1762</a></span> |
| <span class="normal"><a href="#__codelineno-0-1763">1763</a></span> |
| <span class="normal"><a href="#__codelineno-0-1764">1764</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1755" name="__codelineno-0-1755"></a><span class="k">def</span><span class="w"> </span><span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">:</span> <span class="n">S</span><span class="p">,</span> <span class="o">**</span><span class="n">overrides</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">S</span><span class="p">:</span> |
| <a id="__codelineno-0-1756" name="__codelineno-0-1756"></a><span class="w"> </span><span class="sd">"""Create a copy of this table scan with updated fields."""</span> |
| <a id="__codelineno-0-1757" name="__codelineno-0-1757"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">inspect</span><span class="w"> </span><span class="kn">import</span> <span class="n">signature</span> |
| <a id="__codelineno-0-1758" name="__codelineno-0-1758"></a> |
| <a id="__codelineno-0-1759" name="__codelineno-0-1759"></a> <span class="c1"># Extract those attributes that are constructor parameters. We don't use self.__dict__ as the kwargs to the</span> |
| <a id="__codelineno-0-1760" name="__codelineno-0-1760"></a> <span class="c1"># constructors because it may contain additional attributes that are not part of the constructor signature.</span> |
| <a id="__codelineno-0-1761" name="__codelineno-0-1761"></a> <span class="n">params</span> <span class="o">=</span> <span class="n">signature</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">)</span><span class="o">.</span><span class="n">parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="o">-</span> <span class="p">{</span><span class="s2">"self"</span><span class="p">}</span> <span class="c1"># Skip "self" parameter</span> |
| <a id="__codelineno-0-1762" name="__codelineno-0-1762"></a> <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="n">param</span><span class="p">:</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">param</span><span class="p">)</span> <span class="k">for</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">params</span><span class="p">}</span> <span class="c1"># Assume parameters are attributes</span> |
| <a id="__codelineno-0-1763" name="__codelineno-0-1763"></a> |
| <a id="__codelineno-0-1764" name="__codelineno-0-1764"></a> <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)(</span><span class="o">**</span><span class="p">{</span><span class="o">**</span><span class="n">kwargs</span><span class="p">,</span> <span class="o">**</span><span class="n">overrides</span><span class="p">})</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.Transaction" class="doc doc-heading"> |
| <code>Transaction</code> |
| |
| |
| <a href="#pyiceberg.table.Transaction" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-255">255</a></span> |
| <span class="normal"><a href="#__codelineno-0-256">256</a></span> |
| <span class="normal"><a href="#__codelineno-0-257">257</a></span> |
| <span class="normal"><a href="#__codelineno-0-258">258</a></span> |
| <span class="normal"><a href="#__codelineno-0-259">259</a></span> |
| <span class="normal"><a href="#__codelineno-0-260">260</a></span> |
| <span class="normal"><a href="#__codelineno-0-261">261</a></span> |
| <span class="normal"><a href="#__codelineno-0-262">262</a></span> |
| <span class="normal"><a href="#__codelineno-0-263">263</a></span> |
| <span class="normal"><a href="#__codelineno-0-264">264</a></span> |
| <span class="normal"><a href="#__codelineno-0-265">265</a></span> |
| <span class="normal"><a href="#__codelineno-0-266">266</a></span> |
| <span class="normal"><a href="#__codelineno-0-267">267</a></span> |
| <span class="normal"><a href="#__codelineno-0-268">268</a></span> |
| <span class="normal"><a href="#__codelineno-0-269">269</a></span> |
| <span class="normal"><a href="#__codelineno-0-270">270</a></span> |
| <span class="normal"><a href="#__codelineno-0-271">271</a></span> |
| <span class="normal"><a href="#__codelineno-0-272">272</a></span> |
| <span class="normal"><a href="#__codelineno-0-273">273</a></span> |
| <span class="normal"><a href="#__codelineno-0-274">274</a></span> |
| <span class="normal"><a href="#__codelineno-0-275">275</a></span> |
| <span class="normal"><a href="#__codelineno-0-276">276</a></span> |
| <span class="normal"><a href="#__codelineno-0-277">277</a></span> |
| <span class="normal"><a href="#__codelineno-0-278">278</a></span> |
| <span class="normal"><a href="#__codelineno-0-279">279</a></span> |
| <span class="normal"><a href="#__codelineno-0-280">280</a></span> |
| <span class="normal"><a href="#__codelineno-0-281">281</a></span> |
| <span class="normal"><a href="#__codelineno-0-282">282</a></span> |
| <span class="normal"><a href="#__codelineno-0-283">283</a></span> |
| <span class="normal"><a href="#__codelineno-0-284">284</a></span> |
| <span class="normal"><a href="#__codelineno-0-285">285</a></span> |
| <span class="normal"><a href="#__codelineno-0-286">286</a></span> |
| <span class="normal"><a href="#__codelineno-0-287">287</a></span> |
| <span class="normal"><a href="#__codelineno-0-288">288</a></span> |
| <span class="normal"><a href="#__codelineno-0-289">289</a></span> |
| <span class="normal"><a href="#__codelineno-0-290">290</a></span> |
| <span class="normal"><a href="#__codelineno-0-291">291</a></span> |
| <span class="normal"><a href="#__codelineno-0-292">292</a></span> |
| <span class="normal"><a href="#__codelineno-0-293">293</a></span> |
| <span class="normal"><a href="#__codelineno-0-294">294</a></span> |
| <span class="normal"><a href="#__codelineno-0-295">295</a></span> |
| <span class="normal"><a href="#__codelineno-0-296">296</a></span> |
| <span class="normal"><a href="#__codelineno-0-297">297</a></span> |
| <span class="normal"><a href="#__codelineno-0-298">298</a></span> |
| <span class="normal"><a href="#__codelineno-0-299">299</a></span> |
| <span class="normal"><a href="#__codelineno-0-300">300</a></span> |
| <span class="normal"><a href="#__codelineno-0-301">301</a></span> |
| <span class="normal"><a href="#__codelineno-0-302">302</a></span> |
| <span class="normal"><a href="#__codelineno-0-303">303</a></span> |
| <span class="normal"><a href="#__codelineno-0-304">304</a></span> |
| <span class="normal"><a href="#__codelineno-0-305">305</a></span> |
| <span class="normal"><a href="#__codelineno-0-306">306</a></span> |
| <span class="normal"><a href="#__codelineno-0-307">307</a></span> |
| <span class="normal"><a href="#__codelineno-0-308">308</a></span> |
| <span class="normal"><a href="#__codelineno-0-309">309</a></span> |
| <span class="normal"><a href="#__codelineno-0-310">310</a></span> |
| <span class="normal"><a href="#__codelineno-0-311">311</a></span> |
| <span class="normal"><a href="#__codelineno-0-312">312</a></span> |
| <span class="normal"><a href="#__codelineno-0-313">313</a></span> |
| <span class="normal"><a href="#__codelineno-0-314">314</a></span> |
| <span class="normal"><a href="#__codelineno-0-315">315</a></span> |
| <span class="normal"><a href="#__codelineno-0-316">316</a></span> |
| <span class="normal"><a href="#__codelineno-0-317">317</a></span> |
| <span class="normal"><a href="#__codelineno-0-318">318</a></span> |
| <span class="normal"><a href="#__codelineno-0-319">319</a></span> |
| <span class="normal"><a href="#__codelineno-0-320">320</a></span> |
| <span class="normal"><a href="#__codelineno-0-321">321</a></span> |
| <span class="normal"><a href="#__codelineno-0-322">322</a></span> |
| <span class="normal"><a href="#__codelineno-0-323">323</a></span> |
| <span class="normal"><a href="#__codelineno-0-324">324</a></span> |
| <span class="normal"><a href="#__codelineno-0-325">325</a></span> |
| <span class="normal"><a href="#__codelineno-0-326">326</a></span> |
| <span class="normal"><a href="#__codelineno-0-327">327</a></span> |
| <span class="normal"><a href="#__codelineno-0-328">328</a></span> |
| <span class="normal"><a href="#__codelineno-0-329">329</a></span> |
| <span class="normal"><a href="#__codelineno-0-330">330</a></span> |
| <span class="normal"><a href="#__codelineno-0-331">331</a></span> |
| <span class="normal"><a href="#__codelineno-0-332">332</a></span> |
| <span class="normal"><a href="#__codelineno-0-333">333</a></span> |
| <span class="normal"><a href="#__codelineno-0-334">334</a></span> |
| <span class="normal"><a href="#__codelineno-0-335">335</a></span> |
| <span class="normal"><a href="#__codelineno-0-336">336</a></span> |
| <span class="normal"><a href="#__codelineno-0-337">337</a></span> |
| <span class="normal"><a href="#__codelineno-0-338">338</a></span> |
| <span class="normal"><a href="#__codelineno-0-339">339</a></span> |
| <span class="normal"><a href="#__codelineno-0-340">340</a></span> |
| <span class="normal"><a href="#__codelineno-0-341">341</a></span> |
| <span class="normal"><a href="#__codelineno-0-342">342</a></span> |
| <span class="normal"><a href="#__codelineno-0-343">343</a></span> |
| <span class="normal"><a href="#__codelineno-0-344">344</a></span> |
| <span class="normal"><a href="#__codelineno-0-345">345</a></span> |
| <span class="normal"><a href="#__codelineno-0-346">346</a></span> |
| <span class="normal"><a href="#__codelineno-0-347">347</a></span> |
| <span class="normal"><a href="#__codelineno-0-348">348</a></span> |
| <span class="normal"><a href="#__codelineno-0-349">349</a></span> |
| <span class="normal"><a href="#__codelineno-0-350">350</a></span> |
| <span class="normal"><a href="#__codelineno-0-351">351</a></span> |
| <span class="normal"><a href="#__codelineno-0-352">352</a></span> |
| <span class="normal"><a href="#__codelineno-0-353">353</a></span> |
| <span class="normal"><a href="#__codelineno-0-354">354</a></span> |
| <span class="normal"><a href="#__codelineno-0-355">355</a></span> |
| <span class="normal"><a href="#__codelineno-0-356">356</a></span> |
| <span class="normal"><a href="#__codelineno-0-357">357</a></span> |
| <span class="normal"><a href="#__codelineno-0-358">358</a></span> |
| <span class="normal"><a href="#__codelineno-0-359">359</a></span> |
| <span class="normal"><a href="#__codelineno-0-360">360</a></span> |
| <span class="normal"><a href="#__codelineno-0-361">361</a></span> |
| <span class="normal"><a href="#__codelineno-0-362">362</a></span> |
| <span class="normal"><a href="#__codelineno-0-363">363</a></span> |
| <span class="normal"><a href="#__codelineno-0-364">364</a></span> |
| <span class="normal"><a href="#__codelineno-0-365">365</a></span> |
| <span class="normal"><a href="#__codelineno-0-366">366</a></span> |
| <span class="normal"><a href="#__codelineno-0-367">367</a></span> |
| <span class="normal"><a href="#__codelineno-0-368">368</a></span> |
| <span class="normal"><a href="#__codelineno-0-369">369</a></span> |
| <span class="normal"><a href="#__codelineno-0-370">370</a></span> |
| <span class="normal"><a href="#__codelineno-0-371">371</a></span> |
| <span class="normal"><a href="#__codelineno-0-372">372</a></span> |
| <span class="normal"><a href="#__codelineno-0-373">373</a></span> |
| <span class="normal"><a href="#__codelineno-0-374">374</a></span> |
| <span class="normal"><a href="#__codelineno-0-375">375</a></span> |
| <span class="normal"><a href="#__codelineno-0-376">376</a></span> |
| <span class="normal"><a href="#__codelineno-0-377">377</a></span> |
| <span class="normal"><a href="#__codelineno-0-378">378</a></span> |
| <span class="normal"><a href="#__codelineno-0-379">379</a></span> |
| <span class="normal"><a href="#__codelineno-0-380">380</a></span> |
| <span class="normal"><a href="#__codelineno-0-381">381</a></span> |
| <span class="normal"><a href="#__codelineno-0-382">382</a></span> |
| <span class="normal"><a href="#__codelineno-0-383">383</a></span> |
| <span class="normal"><a href="#__codelineno-0-384">384</a></span> |
| <span class="normal"><a href="#__codelineno-0-385">385</a></span> |
| <span class="normal"><a href="#__codelineno-0-386">386</a></span> |
| <span class="normal"><a href="#__codelineno-0-387">387</a></span> |
| <span class="normal"><a href="#__codelineno-0-388">388</a></span> |
| <span class="normal"><a href="#__codelineno-0-389">389</a></span> |
| <span class="normal"><a href="#__codelineno-0-390">390</a></span> |
| <span class="normal"><a href="#__codelineno-0-391">391</a></span> |
| <span class="normal"><a href="#__codelineno-0-392">392</a></span> |
| <span class="normal"><a href="#__codelineno-0-393">393</a></span> |
| <span class="normal"><a href="#__codelineno-0-394">394</a></span> |
| <span class="normal"><a href="#__codelineno-0-395">395</a></span> |
| <span class="normal"><a href="#__codelineno-0-396">396</a></span> |
| <span class="normal"><a href="#__codelineno-0-397">397</a></span> |
| <span class="normal"><a href="#__codelineno-0-398">398</a></span> |
| <span class="normal"><a href="#__codelineno-0-399">399</a></span> |
| <span class="normal"><a href="#__codelineno-0-400">400</a></span> |
| <span class="normal"><a href="#__codelineno-0-401">401</a></span> |
| <span class="normal"><a href="#__codelineno-0-402">402</a></span> |
| <span class="normal"><a href="#__codelineno-0-403">403</a></span> |
| <span class="normal"><a href="#__codelineno-0-404">404</a></span> |
| <span class="normal"><a href="#__codelineno-0-405">405</a></span> |
| <span class="normal"><a href="#__codelineno-0-406">406</a></span> |
| <span class="normal"><a href="#__codelineno-0-407">407</a></span> |
| <span class="normal"><a href="#__codelineno-0-408">408</a></span> |
| <span class="normal"><a href="#__codelineno-0-409">409</a></span> |
| <span class="normal"><a href="#__codelineno-0-410">410</a></span> |
| <span class="normal"><a href="#__codelineno-0-411">411</a></span> |
| <span class="normal"><a href="#__codelineno-0-412">412</a></span> |
| <span class="normal"><a href="#__codelineno-0-413">413</a></span> |
| <span class="normal"><a href="#__codelineno-0-414">414</a></span> |
| <span class="normal"><a href="#__codelineno-0-415">415</a></span> |
| <span class="normal"><a href="#__codelineno-0-416">416</a></span> |
| <span class="normal"><a href="#__codelineno-0-417">417</a></span> |
| <span class="normal"><a href="#__codelineno-0-418">418</a></span> |
| <span class="normal"><a href="#__codelineno-0-419">419</a></span> |
| <span class="normal"><a href="#__codelineno-0-420">420</a></span> |
| <span class="normal"><a href="#__codelineno-0-421">421</a></span> |
| <span class="normal"><a href="#__codelineno-0-422">422</a></span> |
| <span class="normal"><a href="#__codelineno-0-423">423</a></span> |
| <span class="normal"><a href="#__codelineno-0-424">424</a></span> |
| <span class="normal"><a href="#__codelineno-0-425">425</a></span> |
| <span class="normal"><a href="#__codelineno-0-426">426</a></span> |
| <span class="normal"><a href="#__codelineno-0-427">427</a></span> |
| <span class="normal"><a href="#__codelineno-0-428">428</a></span> |
| <span class="normal"><a href="#__codelineno-0-429">429</a></span> |
| <span class="normal"><a href="#__codelineno-0-430">430</a></span> |
| <span class="normal"><a href="#__codelineno-0-431">431</a></span> |
| <span class="normal"><a href="#__codelineno-0-432">432</a></span> |
| <span class="normal"><a href="#__codelineno-0-433">433</a></span> |
| <span class="normal"><a href="#__codelineno-0-434">434</a></span> |
| <span class="normal"><a href="#__codelineno-0-435">435</a></span> |
| <span class="normal"><a href="#__codelineno-0-436">436</a></span> |
| <span class="normal"><a href="#__codelineno-0-437">437</a></span> |
| <span class="normal"><a href="#__codelineno-0-438">438</a></span> |
| <span class="normal"><a href="#__codelineno-0-439">439</a></span> |
| <span class="normal"><a href="#__codelineno-0-440">440</a></span> |
| <span class="normal"><a href="#__codelineno-0-441">441</a></span> |
| <span class="normal"><a href="#__codelineno-0-442">442</a></span> |
| <span class="normal"><a href="#__codelineno-0-443">443</a></span> |
| <span class="normal"><a href="#__codelineno-0-444">444</a></span> |
| <span class="normal"><a href="#__codelineno-0-445">445</a></span> |
| <span class="normal"><a href="#__codelineno-0-446">446</a></span> |
| <span class="normal"><a href="#__codelineno-0-447">447</a></span> |
| <span class="normal"><a href="#__codelineno-0-448">448</a></span> |
| <span class="normal"><a href="#__codelineno-0-449">449</a></span> |
| <span class="normal"><a href="#__codelineno-0-450">450</a></span> |
| <span class="normal"><a href="#__codelineno-0-451">451</a></span> |
| <span class="normal"><a href="#__codelineno-0-452">452</a></span> |
| <span class="normal"><a href="#__codelineno-0-453">453</a></span> |
| <span class="normal"><a href="#__codelineno-0-454">454</a></span> |
| <span class="normal"><a href="#__codelineno-0-455">455</a></span> |
| <span class="normal"><a href="#__codelineno-0-456">456</a></span> |
| <span class="normal"><a href="#__codelineno-0-457">457</a></span> |
| <span class="normal"><a href="#__codelineno-0-458">458</a></span> |
| <span class="normal"><a href="#__codelineno-0-459">459</a></span> |
| <span class="normal"><a href="#__codelineno-0-460">460</a></span> |
| <span class="normal"><a href="#__codelineno-0-461">461</a></span> |
| <span class="normal"><a href="#__codelineno-0-462">462</a></span> |
| <span class="normal"><a href="#__codelineno-0-463">463</a></span> |
| <span class="normal"><a href="#__codelineno-0-464">464</a></span> |
| <span class="normal"><a href="#__codelineno-0-465">465</a></span> |
| <span class="normal"><a href="#__codelineno-0-466">466</a></span> |
| <span class="normal"><a href="#__codelineno-0-467">467</a></span> |
| <span class="normal"><a href="#__codelineno-0-468">468</a></span> |
| <span class="normal"><a href="#__codelineno-0-469">469</a></span> |
| <span class="normal"><a href="#__codelineno-0-470">470</a></span> |
| <span class="normal"><a href="#__codelineno-0-471">471</a></span> |
| <span class="normal"><a href="#__codelineno-0-472">472</a></span> |
| <span class="normal"><a href="#__codelineno-0-473">473</a></span> |
| <span class="normal"><a href="#__codelineno-0-474">474</a></span> |
| <span class="normal"><a href="#__codelineno-0-475">475</a></span> |
| <span class="normal"><a href="#__codelineno-0-476">476</a></span> |
| <span class="normal"><a href="#__codelineno-0-477">477</a></span> |
| <span class="normal"><a href="#__codelineno-0-478">478</a></span> |
| <span class="normal"><a href="#__codelineno-0-479">479</a></span> |
| <span class="normal"><a href="#__codelineno-0-480">480</a></span> |
| <span class="normal"><a href="#__codelineno-0-481">481</a></span> |
| <span class="normal"><a href="#__codelineno-0-482">482</a></span> |
| <span class="normal"><a href="#__codelineno-0-483">483</a></span> |
| <span class="normal"><a href="#__codelineno-0-484">484</a></span> |
| <span class="normal"><a href="#__codelineno-0-485">485</a></span> |
| <span class="normal"><a href="#__codelineno-0-486">486</a></span> |
| <span class="normal"><a href="#__codelineno-0-487">487</a></span> |
| <span class="normal"><a href="#__codelineno-0-488">488</a></span> |
| <span class="normal"><a href="#__codelineno-0-489">489</a></span> |
| <span class="normal"><a href="#__codelineno-0-490">490</a></span> |
| <span class="normal"><a href="#__codelineno-0-491">491</a></span> |
| <span class="normal"><a href="#__codelineno-0-492">492</a></span> |
| <span class="normal"><a href="#__codelineno-0-493">493</a></span> |
| <span class="normal"><a href="#__codelineno-0-494">494</a></span> |
| <span class="normal"><a href="#__codelineno-0-495">495</a></span> |
| <span class="normal"><a href="#__codelineno-0-496">496</a></span> |
| <span class="normal"><a href="#__codelineno-0-497">497</a></span> |
| <span class="normal"><a href="#__codelineno-0-498">498</a></span> |
| <span class="normal"><a href="#__codelineno-0-499">499</a></span> |
| <span class="normal"><a href="#__codelineno-0-500">500</a></span> |
| <span class="normal"><a href="#__codelineno-0-501">501</a></span> |
| <span class="normal"><a href="#__codelineno-0-502">502</a></span> |
| <span class="normal"><a href="#__codelineno-0-503">503</a></span> |
| <span class="normal"><a href="#__codelineno-0-504">504</a></span> |
| <span class="normal"><a href="#__codelineno-0-505">505</a></span> |
| <span class="normal"><a href="#__codelineno-0-506">506</a></span> |
| <span class="normal"><a href="#__codelineno-0-507">507</a></span> |
| <span class="normal"><a href="#__codelineno-0-508">508</a></span> |
| <span class="normal"><a href="#__codelineno-0-509">509</a></span> |
| <span class="normal"><a href="#__codelineno-0-510">510</a></span> |
| <span class="normal"><a href="#__codelineno-0-511">511</a></span> |
| <span class="normal"><a href="#__codelineno-0-512">512</a></span> |
| <span class="normal"><a href="#__codelineno-0-513">513</a></span> |
| <span class="normal"><a href="#__codelineno-0-514">514</a></span> |
| <span class="normal"><a href="#__codelineno-0-515">515</a></span> |
| <span class="normal"><a href="#__codelineno-0-516">516</a></span> |
| <span class="normal"><a href="#__codelineno-0-517">517</a></span> |
| <span class="normal"><a href="#__codelineno-0-518">518</a></span> |
| <span class="normal"><a href="#__codelineno-0-519">519</a></span> |
| <span class="normal"><a href="#__codelineno-0-520">520</a></span> |
| <span class="normal"><a href="#__codelineno-0-521">521</a></span> |
| <span class="normal"><a href="#__codelineno-0-522">522</a></span> |
| <span class="normal"><a href="#__codelineno-0-523">523</a></span> |
| <span class="normal"><a href="#__codelineno-0-524">524</a></span> |
| <span class="normal"><a href="#__codelineno-0-525">525</a></span> |
| <span class="normal"><a href="#__codelineno-0-526">526</a></span> |
| <span class="normal"><a href="#__codelineno-0-527">527</a></span> |
| <span class="normal"><a href="#__codelineno-0-528">528</a></span> |
| <span class="normal"><a href="#__codelineno-0-529">529</a></span> |
| <span class="normal"><a href="#__codelineno-0-530">530</a></span> |
| <span class="normal"><a href="#__codelineno-0-531">531</a></span> |
| <span class="normal"><a href="#__codelineno-0-532">532</a></span> |
| <span class="normal"><a href="#__codelineno-0-533">533</a></span> |
| <span class="normal"><a href="#__codelineno-0-534">534</a></span> |
| <span class="normal"><a href="#__codelineno-0-535">535</a></span> |
| <span class="normal"><a href="#__codelineno-0-536">536</a></span> |
| <span class="normal"><a href="#__codelineno-0-537">537</a></span> |
| <span class="normal"><a href="#__codelineno-0-538">538</a></span> |
| <span class="normal"><a href="#__codelineno-0-539">539</a></span> |
| <span class="normal"><a href="#__codelineno-0-540">540</a></span> |
| <span class="normal"><a href="#__codelineno-0-541">541</a></span> |
| <span class="normal"><a href="#__codelineno-0-542">542</a></span> |
| <span class="normal"><a href="#__codelineno-0-543">543</a></span> |
| <span class="normal"><a href="#__codelineno-0-544">544</a></span> |
| <span class="normal"><a href="#__codelineno-0-545">545</a></span> |
| <span class="normal"><a href="#__codelineno-0-546">546</a></span> |
| <span class="normal"><a href="#__codelineno-0-547">547</a></span> |
| <span class="normal"><a href="#__codelineno-0-548">548</a></span> |
| <span class="normal"><a href="#__codelineno-0-549">549</a></span> |
| <span class="normal"><a href="#__codelineno-0-550">550</a></span> |
| <span class="normal"><a href="#__codelineno-0-551">551</a></span> |
| <span class="normal"><a href="#__codelineno-0-552">552</a></span> |
| <span class="normal"><a href="#__codelineno-0-553">553</a></span> |
| <span class="normal"><a href="#__codelineno-0-554">554</a></span> |
| <span class="normal"><a href="#__codelineno-0-555">555</a></span> |
| <span class="normal"><a href="#__codelineno-0-556">556</a></span> |
| <span class="normal"><a href="#__codelineno-0-557">557</a></span> |
| <span class="normal"><a href="#__codelineno-0-558">558</a></span> |
| <span class="normal"><a href="#__codelineno-0-559">559</a></span> |
| <span class="normal"><a href="#__codelineno-0-560">560</a></span> |
| <span class="normal"><a href="#__codelineno-0-561">561</a></span> |
| <span class="normal"><a href="#__codelineno-0-562">562</a></span> |
| <span class="normal"><a href="#__codelineno-0-563">563</a></span> |
| <span class="normal"><a href="#__codelineno-0-564">564</a></span> |
| <span class="normal"><a href="#__codelineno-0-565">565</a></span> |
| <span class="normal"><a href="#__codelineno-0-566">566</a></span> |
| <span class="normal"><a href="#__codelineno-0-567">567</a></span> |
| <span class="normal"><a href="#__codelineno-0-568">568</a></span> |
| <span class="normal"><a href="#__codelineno-0-569">569</a></span> |
| <span class="normal"><a href="#__codelineno-0-570">570</a></span> |
| <span class="normal"><a href="#__codelineno-0-571">571</a></span> |
| <span class="normal"><a href="#__codelineno-0-572">572</a></span> |
| <span class="normal"><a href="#__codelineno-0-573">573</a></span> |
| <span class="normal"><a href="#__codelineno-0-574">574</a></span> |
| <span class="normal"><a href="#__codelineno-0-575">575</a></span> |
| <span class="normal"><a href="#__codelineno-0-576">576</a></span> |
| <span class="normal"><a href="#__codelineno-0-577">577</a></span> |
| <span class="normal"><a href="#__codelineno-0-578">578</a></span> |
| <span class="normal"><a href="#__codelineno-0-579">579</a></span> |
| <span class="normal"><a href="#__codelineno-0-580">580</a></span> |
| <span class="normal"><a href="#__codelineno-0-581">581</a></span> |
| <span class="normal"><a href="#__codelineno-0-582">582</a></span> |
| <span class="normal"><a href="#__codelineno-0-583">583</a></span> |
| <span class="normal"><a href="#__codelineno-0-584">584</a></span> |
| <span class="normal"><a href="#__codelineno-0-585">585</a></span> |
| <span class="normal"><a href="#__codelineno-0-586">586</a></span> |
| <span class="normal"><a href="#__codelineno-0-587">587</a></span> |
| <span class="normal"><a href="#__codelineno-0-588">588</a></span> |
| <span class="normal"><a href="#__codelineno-0-589">589</a></span> |
| <span class="normal"><a href="#__codelineno-0-590">590</a></span> |
| <span class="normal"><a href="#__codelineno-0-591">591</a></span> |
| <span class="normal"><a href="#__codelineno-0-592">592</a></span> |
| <span class="normal"><a href="#__codelineno-0-593">593</a></span> |
| <span class="normal"><a href="#__codelineno-0-594">594</a></span> |
| <span class="normal"><a href="#__codelineno-0-595">595</a></span> |
| <span class="normal"><a href="#__codelineno-0-596">596</a></span> |
| <span class="normal"><a href="#__codelineno-0-597">597</a></span> |
| <span class="normal"><a href="#__codelineno-0-598">598</a></span> |
| <span class="normal"><a href="#__codelineno-0-599">599</a></span> |
| <span class="normal"><a href="#__codelineno-0-600">600</a></span> |
| <span class="normal"><a href="#__codelineno-0-601">601</a></span> |
| <span class="normal"><a href="#__codelineno-0-602">602</a></span> |
| <span class="normal"><a href="#__codelineno-0-603">603</a></span> |
| <span class="normal"><a href="#__codelineno-0-604">604</a></span> |
| <span class="normal"><a href="#__codelineno-0-605">605</a></span> |
| <span class="normal"><a href="#__codelineno-0-606">606</a></span> |
| <span class="normal"><a href="#__codelineno-0-607">607</a></span> |
| <span class="normal"><a href="#__codelineno-0-608">608</a></span> |
| <span class="normal"><a href="#__codelineno-0-609">609</a></span> |
| <span class="normal"><a href="#__codelineno-0-610">610</a></span> |
| <span class="normal"><a href="#__codelineno-0-611">611</a></span> |
| <span class="normal"><a href="#__codelineno-0-612">612</a></span> |
| <span class="normal"><a href="#__codelineno-0-613">613</a></span> |
| <span class="normal"><a href="#__codelineno-0-614">614</a></span> |
| <span class="normal"><a href="#__codelineno-0-615">615</a></span> |
| <span class="normal"><a href="#__codelineno-0-616">616</a></span> |
| <span class="normal"><a href="#__codelineno-0-617">617</a></span> |
| <span class="normal"><a href="#__codelineno-0-618">618</a></span> |
| <span class="normal"><a href="#__codelineno-0-619">619</a></span> |
| <span class="normal"><a href="#__codelineno-0-620">620</a></span> |
| <span class="normal"><a href="#__codelineno-0-621">621</a></span> |
| <span class="normal"><a href="#__codelineno-0-622">622</a></span> |
| <span class="normal"><a href="#__codelineno-0-623">623</a></span> |
| <span class="normal"><a href="#__codelineno-0-624">624</a></span> |
| <span class="normal"><a href="#__codelineno-0-625">625</a></span> |
| <span class="normal"><a href="#__codelineno-0-626">626</a></span> |
| <span class="normal"><a href="#__codelineno-0-627">627</a></span> |
| <span class="normal"><a href="#__codelineno-0-628">628</a></span> |
| <span class="normal"><a href="#__codelineno-0-629">629</a></span> |
| <span class="normal"><a href="#__codelineno-0-630">630</a></span> |
| <span class="normal"><a href="#__codelineno-0-631">631</a></span> |
| <span class="normal"><a href="#__codelineno-0-632">632</a></span> |
| <span class="normal"><a href="#__codelineno-0-633">633</a></span> |
| <span class="normal"><a href="#__codelineno-0-634">634</a></span> |
| <span class="normal"><a href="#__codelineno-0-635">635</a></span> |
| <span class="normal"><a href="#__codelineno-0-636">636</a></span> |
| <span class="normal"><a href="#__codelineno-0-637">637</a></span> |
| <span class="normal"><a href="#__codelineno-0-638">638</a></span> |
| <span class="normal"><a href="#__codelineno-0-639">639</a></span> |
| <span class="normal"><a href="#__codelineno-0-640">640</a></span> |
| <span class="normal"><a href="#__codelineno-0-641">641</a></span> |
| <span class="normal"><a href="#__codelineno-0-642">642</a></span> |
| <span class="normal"><a href="#__codelineno-0-643">643</a></span> |
| <span class="normal"><a href="#__codelineno-0-644">644</a></span> |
| <span class="normal"><a href="#__codelineno-0-645">645</a></span> |
| <span class="normal"><a href="#__codelineno-0-646">646</a></span> |
| <span class="normal"><a href="#__codelineno-0-647">647</a></span> |
| <span class="normal"><a href="#__codelineno-0-648">648</a></span> |
| <span class="normal"><a href="#__codelineno-0-649">649</a></span> |
| <span class="normal"><a href="#__codelineno-0-650">650</a></span> |
| <span class="normal"><a href="#__codelineno-0-651">651</a></span> |
| <span class="normal"><a href="#__codelineno-0-652">652</a></span> |
| <span class="normal"><a href="#__codelineno-0-653">653</a></span> |
| <span class="normal"><a href="#__codelineno-0-654">654</a></span> |
| <span class="normal"><a href="#__codelineno-0-655">655</a></span> |
| <span class="normal"><a href="#__codelineno-0-656">656</a></span> |
| <span class="normal"><a href="#__codelineno-0-657">657</a></span> |
| <span class="normal"><a href="#__codelineno-0-658">658</a></span> |
| <span class="normal"><a href="#__codelineno-0-659">659</a></span> |
| <span class="normal"><a href="#__codelineno-0-660">660</a></span> |
| <span class="normal"><a href="#__codelineno-0-661">661</a></span> |
| <span class="normal"><a href="#__codelineno-0-662">662</a></span> |
| <span class="normal"><a href="#__codelineno-0-663">663</a></span> |
| <span class="normal"><a href="#__codelineno-0-664">664</a></span> |
| <span class="normal"><a href="#__codelineno-0-665">665</a></span> |
| <span class="normal"><a href="#__codelineno-0-666">666</a></span> |
| <span class="normal"><a href="#__codelineno-0-667">667</a></span> |
| <span class="normal"><a href="#__codelineno-0-668">668</a></span> |
| <span class="normal"><a href="#__codelineno-0-669">669</a></span> |
| <span class="normal"><a href="#__codelineno-0-670">670</a></span> |
| <span class="normal"><a href="#__codelineno-0-671">671</a></span> |
| <span class="normal"><a href="#__codelineno-0-672">672</a></span> |
| <span class="normal"><a href="#__codelineno-0-673">673</a></span> |
| <span class="normal"><a href="#__codelineno-0-674">674</a></span> |
| <span class="normal"><a href="#__codelineno-0-675">675</a></span> |
| <span class="normal"><a href="#__codelineno-0-676">676</a></span> |
| <span class="normal"><a href="#__codelineno-0-677">677</a></span> |
| <span class="normal"><a href="#__codelineno-0-678">678</a></span> |
| <span class="normal"><a href="#__codelineno-0-679">679</a></span> |
| <span class="normal"><a href="#__codelineno-0-680">680</a></span> |
| <span class="normal"><a href="#__codelineno-0-681">681</a></span> |
| <span class="normal"><a href="#__codelineno-0-682">682</a></span> |
| <span class="normal"><a href="#__codelineno-0-683">683</a></span> |
| <span class="normal"><a href="#__codelineno-0-684">684</a></span> |
| <span class="normal"><a href="#__codelineno-0-685">685</a></span> |
| <span class="normal"><a href="#__codelineno-0-686">686</a></span> |
| <span class="normal"><a href="#__codelineno-0-687">687</a></span> |
| <span class="normal"><a href="#__codelineno-0-688">688</a></span> |
| <span class="normal"><a href="#__codelineno-0-689">689</a></span> |
| <span class="normal"><a href="#__codelineno-0-690">690</a></span> |
| <span class="normal"><a href="#__codelineno-0-691">691</a></span> |
| <span class="normal"><a href="#__codelineno-0-692">692</a></span> |
| <span class="normal"><a href="#__codelineno-0-693">693</a></span> |
| <span class="normal"><a href="#__codelineno-0-694">694</a></span> |
| <span class="normal"><a href="#__codelineno-0-695">695</a></span> |
| <span class="normal"><a href="#__codelineno-0-696">696</a></span> |
| <span class="normal"><a href="#__codelineno-0-697">697</a></span> |
| <span class="normal"><a href="#__codelineno-0-698">698</a></span> |
| <span class="normal"><a href="#__codelineno-0-699">699</a></span> |
| <span class="normal"><a href="#__codelineno-0-700">700</a></span> |
| <span class="normal"><a href="#__codelineno-0-701">701</a></span> |
| <span class="normal"><a href="#__codelineno-0-702">702</a></span> |
| <span class="normal"><a href="#__codelineno-0-703">703</a></span> |
| <span class="normal"><a href="#__codelineno-0-704">704</a></span> |
| <span class="normal"><a href="#__codelineno-0-705">705</a></span> |
| <span class="normal"><a href="#__codelineno-0-706">706</a></span> |
| <span class="normal"><a href="#__codelineno-0-707">707</a></span> |
| <span class="normal"><a href="#__codelineno-0-708">708</a></span> |
| <span class="normal"><a href="#__codelineno-0-709">709</a></span> |
| <span class="normal"><a href="#__codelineno-0-710">710</a></span> |
| <span class="normal"><a href="#__codelineno-0-711">711</a></span> |
| <span class="normal"><a href="#__codelineno-0-712">712</a></span> |
| <span class="normal"><a href="#__codelineno-0-713">713</a></span> |
| <span class="normal"><a href="#__codelineno-0-714">714</a></span> |
| <span class="normal"><a href="#__codelineno-0-715">715</a></span> |
| <span class="normal"><a href="#__codelineno-0-716">716</a></span> |
| <span class="normal"><a href="#__codelineno-0-717">717</a></span> |
| <span class="normal"><a href="#__codelineno-0-718">718</a></span> |
| <span class="normal"><a href="#__codelineno-0-719">719</a></span> |
| <span class="normal"><a href="#__codelineno-0-720">720</a></span> |
| <span class="normal"><a href="#__codelineno-0-721">721</a></span> |
| <span class="normal"><a href="#__codelineno-0-722">722</a></span> |
| <span class="normal"><a href="#__codelineno-0-723">723</a></span> |
| <span class="normal"><a href="#__codelineno-0-724">724</a></span> |
| <span class="normal"><a href="#__codelineno-0-725">725</a></span> |
| <span class="normal"><a href="#__codelineno-0-726">726</a></span> |
| <span class="normal"><a href="#__codelineno-0-727">727</a></span> |
| <span class="normal"><a href="#__codelineno-0-728">728</a></span> |
| <span class="normal"><a href="#__codelineno-0-729">729</a></span> |
| <span class="normal"><a href="#__codelineno-0-730">730</a></span> |
| <span class="normal"><a href="#__codelineno-0-731">731</a></span> |
| <span class="normal"><a href="#__codelineno-0-732">732</a></span> |
| <span class="normal"><a href="#__codelineno-0-733">733</a></span> |
| <span class="normal"><a href="#__codelineno-0-734">734</a></span> |
| <span class="normal"><a href="#__codelineno-0-735">735</a></span> |
| <span class="normal"><a href="#__codelineno-0-736">736</a></span> |
| <span class="normal"><a href="#__codelineno-0-737">737</a></span> |
| <span class="normal"><a href="#__codelineno-0-738">738</a></span> |
| <span class="normal"><a href="#__codelineno-0-739">739</a></span> |
| <span class="normal"><a href="#__codelineno-0-740">740</a></span> |
| <span class="normal"><a href="#__codelineno-0-741">741</a></span> |
| <span class="normal"><a href="#__codelineno-0-742">742</a></span> |
| <span class="normal"><a href="#__codelineno-0-743">743</a></span> |
| <span class="normal"><a href="#__codelineno-0-744">744</a></span> |
| <span class="normal"><a href="#__codelineno-0-745">745</a></span> |
| <span class="normal"><a href="#__codelineno-0-746">746</a></span> |
| <span class="normal"><a href="#__codelineno-0-747">747</a></span> |
| <span class="normal"><a href="#__codelineno-0-748">748</a></span> |
| <span class="normal"><a href="#__codelineno-0-749">749</a></span> |
| <span class="normal"><a href="#__codelineno-0-750">750</a></span> |
| <span class="normal"><a href="#__codelineno-0-751">751</a></span> |
| <span class="normal"><a href="#__codelineno-0-752">752</a></span> |
| <span class="normal"><a href="#__codelineno-0-753">753</a></span> |
| <span class="normal"><a href="#__codelineno-0-754">754</a></span> |
| <span class="normal"><a href="#__codelineno-0-755">755</a></span> |
| <span class="normal"><a href="#__codelineno-0-756">756</a></span> |
| <span class="normal"><a href="#__codelineno-0-757">757</a></span> |
| <span class="normal"><a href="#__codelineno-0-758">758</a></span> |
| <span class="normal"><a href="#__codelineno-0-759">759</a></span> |
| <span class="normal"><a href="#__codelineno-0-760">760</a></span> |
| <span class="normal"><a href="#__codelineno-0-761">761</a></span> |
| <span class="normal"><a href="#__codelineno-0-762">762</a></span> |
| <span class="normal"><a href="#__codelineno-0-763">763</a></span> |
| <span class="normal"><a href="#__codelineno-0-764">764</a></span> |
| <span class="normal"><a href="#__codelineno-0-765">765</a></span> |
| <span class="normal"><a href="#__codelineno-0-766">766</a></span> |
| <span class="normal"><a href="#__codelineno-0-767">767</a></span> |
| <span class="normal"><a href="#__codelineno-0-768">768</a></span> |
| <span class="normal"><a href="#__codelineno-0-769">769</a></span> |
| <span class="normal"><a href="#__codelineno-0-770">770</a></span> |
| <span class="normal"><a href="#__codelineno-0-771">771</a></span> |
| <span class="normal"><a href="#__codelineno-0-772">772</a></span> |
| <span class="normal"><a href="#__codelineno-0-773">773</a></span> |
| <span class="normal"><a href="#__codelineno-0-774">774</a></span> |
| <span class="normal"><a href="#__codelineno-0-775">775</a></span> |
| <span class="normal"><a href="#__codelineno-0-776">776</a></span> |
| <span class="normal"><a href="#__codelineno-0-777">777</a></span> |
| <span class="normal"><a href="#__codelineno-0-778">778</a></span> |
| <span class="normal"><a href="#__codelineno-0-779">779</a></span> |
| <span class="normal"><a href="#__codelineno-0-780">780</a></span> |
| <span class="normal"><a href="#__codelineno-0-781">781</a></span> |
| <span class="normal"><a href="#__codelineno-0-782">782</a></span> |
| <span class="normal"><a href="#__codelineno-0-783">783</a></span> |
| <span class="normal"><a href="#__codelineno-0-784">784</a></span> |
| <span class="normal"><a href="#__codelineno-0-785">785</a></span> |
| <span class="normal"><a href="#__codelineno-0-786">786</a></span> |
| <span class="normal"><a href="#__codelineno-0-787">787</a></span> |
| <span class="normal"><a href="#__codelineno-0-788">788</a></span> |
| <span class="normal"><a href="#__codelineno-0-789">789</a></span> |
| <span class="normal"><a href="#__codelineno-0-790">790</a></span> |
| <span class="normal"><a href="#__codelineno-0-791">791</a></span> |
| <span class="normal"><a href="#__codelineno-0-792">792</a></span> |
| <span class="normal"><a href="#__codelineno-0-793">793</a></span> |
| <span class="normal"><a href="#__codelineno-0-794">794</a></span> |
| <span class="normal"><a href="#__codelineno-0-795">795</a></span> |
| <span class="normal"><a href="#__codelineno-0-796">796</a></span> |
| <span class="normal"><a href="#__codelineno-0-797">797</a></span> |
| <span class="normal"><a href="#__codelineno-0-798">798</a></span> |
| <span class="normal"><a href="#__codelineno-0-799">799</a></span> |
| <span class="normal"><a href="#__codelineno-0-800">800</a></span> |
| <span class="normal"><a href="#__codelineno-0-801">801</a></span> |
| <span class="normal"><a href="#__codelineno-0-802">802</a></span> |
| <span class="normal"><a href="#__codelineno-0-803">803</a></span> |
| <span class="normal"><a href="#__codelineno-0-804">804</a></span> |
| <span class="normal"><a href="#__codelineno-0-805">805</a></span> |
| <span class="normal"><a href="#__codelineno-0-806">806</a></span> |
| <span class="normal"><a href="#__codelineno-0-807">807</a></span> |
| <span class="normal"><a href="#__codelineno-0-808">808</a></span> |
| <span class="normal"><a href="#__codelineno-0-809">809</a></span> |
| <span class="normal"><a href="#__codelineno-0-810">810</a></span> |
| <span class="normal"><a href="#__codelineno-0-811">811</a></span> |
| <span class="normal"><a href="#__codelineno-0-812">812</a></span> |
| <span class="normal"><a href="#__codelineno-0-813">813</a></span> |
| <span class="normal"><a href="#__codelineno-0-814">814</a></span> |
| <span class="normal"><a href="#__codelineno-0-815">815</a></span> |
| <span class="normal"><a href="#__codelineno-0-816">816</a></span> |
| <span class="normal"><a href="#__codelineno-0-817">817</a></span> |
| <span class="normal"><a href="#__codelineno-0-818">818</a></span> |
| <span class="normal"><a href="#__codelineno-0-819">819</a></span> |
| <span class="normal"><a href="#__codelineno-0-820">820</a></span> |
| <span class="normal"><a href="#__codelineno-0-821">821</a></span> |
| <span class="normal"><a href="#__codelineno-0-822">822</a></span> |
| <span class="normal"><a href="#__codelineno-0-823">823</a></span> |
| <span class="normal"><a href="#__codelineno-0-824">824</a></span> |
| <span class="normal"><a href="#__codelineno-0-825">825</a></span> |
| <span class="normal"><a href="#__codelineno-0-826">826</a></span> |
| <span class="normal"><a href="#__codelineno-0-827">827</a></span> |
| <span class="normal"><a href="#__codelineno-0-828">828</a></span> |
| <span class="normal"><a href="#__codelineno-0-829">829</a></span> |
| <span class="normal"><a href="#__codelineno-0-830">830</a></span> |
| <span class="normal"><a href="#__codelineno-0-831">831</a></span> |
| <span class="normal"><a href="#__codelineno-0-832">832</a></span> |
| <span class="normal"><a href="#__codelineno-0-833">833</a></span> |
| <span class="normal"><a href="#__codelineno-0-834">834</a></span> |
| <span class="normal"><a href="#__codelineno-0-835">835</a></span> |
| <span class="normal"><a href="#__codelineno-0-836">836</a></span> |
| <span class="normal"><a href="#__codelineno-0-837">837</a></span> |
| <span class="normal"><a href="#__codelineno-0-838">838</a></span> |
| <span class="normal"><a href="#__codelineno-0-839">839</a></span> |
| <span class="normal"><a href="#__codelineno-0-840">840</a></span> |
| <span class="normal"><a href="#__codelineno-0-841">841</a></span> |
| <span class="normal"><a href="#__codelineno-0-842">842</a></span> |
| <span class="normal"><a href="#__codelineno-0-843">843</a></span> |
| <span class="normal"><a href="#__codelineno-0-844">844</a></span> |
| <span class="normal"><a href="#__codelineno-0-845">845</a></span> |
| <span class="normal"><a href="#__codelineno-0-846">846</a></span> |
| <span class="normal"><a href="#__codelineno-0-847">847</a></span> |
| <span class="normal"><a href="#__codelineno-0-848">848</a></span> |
| <span class="normal"><a href="#__codelineno-0-849">849</a></span> |
| <span class="normal"><a href="#__codelineno-0-850">850</a></span> |
| <span class="normal"><a href="#__codelineno-0-851">851</a></span> |
| <span class="normal"><a href="#__codelineno-0-852">852</a></span> |
| <span class="normal"><a href="#__codelineno-0-853">853</a></span> |
| <span class="normal"><a href="#__codelineno-0-854">854</a></span> |
| <span class="normal"><a href="#__codelineno-0-855">855</a></span> |
| <span class="normal"><a href="#__codelineno-0-856">856</a></span> |
| <span class="normal"><a href="#__codelineno-0-857">857</a></span> |
| <span class="normal"><a href="#__codelineno-0-858">858</a></span> |
| <span class="normal"><a href="#__codelineno-0-859">859</a></span> |
| <span class="normal"><a href="#__codelineno-0-860">860</a></span> |
| <span class="normal"><a href="#__codelineno-0-861">861</a></span> |
| <span class="normal"><a href="#__codelineno-0-862">862</a></span> |
| <span class="normal"><a href="#__codelineno-0-863">863</a></span> |
| <span class="normal"><a href="#__codelineno-0-864">864</a></span> |
| <span class="normal"><a href="#__codelineno-0-865">865</a></span> |
| <span class="normal"><a href="#__codelineno-0-866">866</a></span> |
| <span class="normal"><a href="#__codelineno-0-867">867</a></span> |
| <span class="normal"><a href="#__codelineno-0-868">868</a></span> |
| <span class="normal"><a href="#__codelineno-0-869">869</a></span> |
| <span class="normal"><a href="#__codelineno-0-870">870</a></span> |
| <span class="normal"><a href="#__codelineno-0-871">871</a></span> |
| <span class="normal"><a href="#__codelineno-0-872">872</a></span> |
| <span class="normal"><a href="#__codelineno-0-873">873</a></span> |
| <span class="normal"><a href="#__codelineno-0-874">874</a></span> |
| <span class="normal"><a href="#__codelineno-0-875">875</a></span> |
| <span class="normal"><a href="#__codelineno-0-876">876</a></span> |
| <span class="normal"><a href="#__codelineno-0-877">877</a></span> |
| <span class="normal"><a href="#__codelineno-0-878">878</a></span> |
| <span class="normal"><a href="#__codelineno-0-879">879</a></span> |
| <span class="normal"><a href="#__codelineno-0-880">880</a></span> |
| <span class="normal"><a href="#__codelineno-0-881">881</a></span> |
| <span class="normal"><a href="#__codelineno-0-882">882</a></span> |
| <span class="normal"><a href="#__codelineno-0-883">883</a></span> |
| <span class="normal"><a href="#__codelineno-0-884">884</a></span> |
| <span class="normal"><a href="#__codelineno-0-885">885</a></span> |
| <span class="normal"><a href="#__codelineno-0-886">886</a></span> |
| <span class="normal"><a href="#__codelineno-0-887">887</a></span> |
| <span class="normal"><a href="#__codelineno-0-888">888</a></span> |
| <span class="normal"><a href="#__codelineno-0-889">889</a></span> |
| <span class="normal"><a href="#__codelineno-0-890">890</a></span> |
| <span class="normal"><a href="#__codelineno-0-891">891</a></span> |
| <span class="normal"><a href="#__codelineno-0-892">892</a></span> |
| <span class="normal"><a href="#__codelineno-0-893">893</a></span> |
| <span class="normal"><a href="#__codelineno-0-894">894</a></span> |
| <span class="normal"><a href="#__codelineno-0-895">895</a></span> |
| <span class="normal"><a href="#__codelineno-0-896">896</a></span> |
| <span class="normal"><a href="#__codelineno-0-897">897</a></span> |
| <span class="normal"><a href="#__codelineno-0-898">898</a></span> |
| <span class="normal"><a href="#__codelineno-0-899">899</a></span> |
| <span class="normal"><a href="#__codelineno-0-900">900</a></span> |
| <span class="normal"><a href="#__codelineno-0-901">901</a></span> |
| <span class="normal"><a href="#__codelineno-0-902">902</a></span> |
| <span class="normal"><a href="#__codelineno-0-903">903</a></span> |
| <span class="normal"><a href="#__codelineno-0-904">904</a></span> |
| <span class="normal"><a href="#__codelineno-0-905">905</a></span> |
| <span class="normal"><a href="#__codelineno-0-906">906</a></span> |
| <span class="normal"><a href="#__codelineno-0-907">907</a></span> |
| <span class="normal"><a href="#__codelineno-0-908">908</a></span> |
| <span class="normal"><a href="#__codelineno-0-909">909</a></span> |
| <span class="normal"><a href="#__codelineno-0-910">910</a></span> |
| <span class="normal"><a href="#__codelineno-0-911">911</a></span> |
| <span class="normal"><a href="#__codelineno-0-912">912</a></span> |
| <span class="normal"><a href="#__codelineno-0-913">913</a></span> |
| <span class="normal"><a href="#__codelineno-0-914">914</a></span> |
| <span class="normal"><a href="#__codelineno-0-915">915</a></span> |
| <span class="normal"><a href="#__codelineno-0-916">916</a></span> |
| <span class="normal"><a href="#__codelineno-0-917">917</a></span> |
| <span class="normal"><a href="#__codelineno-0-918">918</a></span> |
| <span class="normal"><a href="#__codelineno-0-919">919</a></span> |
| <span class="normal"><a href="#__codelineno-0-920">920</a></span> |
| <span class="normal"><a href="#__codelineno-0-921">921</a></span> |
| <span class="normal"><a href="#__codelineno-0-922">922</a></span> |
| <span class="normal"><a href="#__codelineno-0-923">923</a></span> |
| <span class="normal"><a href="#__codelineno-0-924">924</a></span> |
| <span class="normal"><a href="#__codelineno-0-925">925</a></span> |
| <span class="normal"><a href="#__codelineno-0-926">926</a></span> |
| <span class="normal"><a href="#__codelineno-0-927">927</a></span> |
| <span class="normal"><a href="#__codelineno-0-928">928</a></span> |
| <span class="normal"><a href="#__codelineno-0-929">929</a></span> |
| <span class="normal"><a href="#__codelineno-0-930">930</a></span> |
| <span class="normal"><a href="#__codelineno-0-931">931</a></span> |
| <span class="normal"><a href="#__codelineno-0-932">932</a></span> |
| <span class="normal"><a href="#__codelineno-0-933">933</a></span> |
| <span class="normal"><a href="#__codelineno-0-934">934</a></span> |
| <span class="normal"><a href="#__codelineno-0-935">935</a></span> |
| <span class="normal"><a href="#__codelineno-0-936">936</a></span> |
| <span class="normal"><a href="#__codelineno-0-937">937</a></span> |
| <span class="normal"><a href="#__codelineno-0-938">938</a></span> |
| <span class="normal"><a href="#__codelineno-0-939">939</a></span> |
| <span class="normal"><a href="#__codelineno-0-940">940</a></span> |
| <span class="normal"><a href="#__codelineno-0-941">941</a></span> |
| <span class="normal"><a href="#__codelineno-0-942">942</a></span> |
| <span class="normal"><a href="#__codelineno-0-943">943</a></span> |
| <span class="normal"><a href="#__codelineno-0-944">944</a></span> |
| <span class="normal"><a href="#__codelineno-0-945">945</a></span> |
| <span class="normal"><a href="#__codelineno-0-946">946</a></span> |
| <span class="normal"><a href="#__codelineno-0-947">947</a></span> |
| <span class="normal"><a href="#__codelineno-0-948">948</a></span> |
| <span class="normal"><a href="#__codelineno-0-949">949</a></span> |
| <span class="normal"><a href="#__codelineno-0-950">950</a></span> |
| <span class="normal"><a href="#__codelineno-0-951">951</a></span> |
| <span class="normal"><a href="#__codelineno-0-952">952</a></span> |
| <span class="normal"><a href="#__codelineno-0-953">953</a></span> |
| <span class="normal"><a href="#__codelineno-0-954">954</a></span> |
| <span class="normal"><a href="#__codelineno-0-955">955</a></span> |
| <span class="normal"><a href="#__codelineno-0-956">956</a></span> |
| <span class="normal"><a href="#__codelineno-0-957">957</a></span> |
| <span class="normal"><a href="#__codelineno-0-958">958</a></span> |
| <span class="normal"><a href="#__codelineno-0-959">959</a></span> |
| <span class="normal"><a href="#__codelineno-0-960">960</a></span> |
| <span class="normal"><a href="#__codelineno-0-961">961</a></span> |
| <span class="normal"><a href="#__codelineno-0-962">962</a></span> |
| <span class="normal"><a href="#__codelineno-0-963">963</a></span> |
| <span class="normal"><a href="#__codelineno-0-964">964</a></span> |
| <span class="normal"><a href="#__codelineno-0-965">965</a></span> |
| <span class="normal"><a href="#__codelineno-0-966">966</a></span> |
| <span class="normal"><a href="#__codelineno-0-967">967</a></span> |
| <span class="normal"><a href="#__codelineno-0-968">968</a></span> |
| <span class="normal"><a href="#__codelineno-0-969">969</a></span> |
| <span class="normal"><a href="#__codelineno-0-970">970</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-255" name="__codelineno-0-255"></a><span class="k">class</span><span class="w"> </span><span class="nc">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-256" name="__codelineno-0-256"></a> <span class="n">_table</span><span class="p">:</span> <span class="n">Table</span> |
| <a id="__codelineno-0-257" name="__codelineno-0-257"></a> <span class="n">_autocommit</span><span class="p">:</span> <span class="nb">bool</span> |
| <a id="__codelineno-0-258" name="__codelineno-0-258"></a> <span class="n">_updates</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableUpdate</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> |
| <a id="__codelineno-0-259" name="__codelineno-0-259"></a> <span class="n">_requirements</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableRequirement</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> |
| <a id="__codelineno-0-260" name="__codelineno-0-260"></a> |
| <a id="__codelineno-0-261" name="__codelineno-0-261"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">:</span> <span class="n">Table</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span> |
| <a id="__codelineno-0-262" name="__codelineno-0-262"></a><span class="w"> </span><span class="sd">"""Open a transaction to stage and commit changes to a table.</span> |
| <a id="__codelineno-0-263" name="__codelineno-0-263"></a> |
| <a id="__codelineno-0-264" name="__codelineno-0-264"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-265" name="__codelineno-0-265"></a><span class="sd"> table: The table that will be altered.</span> |
| <a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="sd"> autocommit: Option to automatically commit the changes when they are staged.</span> |
| <a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-268" name="__codelineno-0-268"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span> <span class="o">=</span> <span class="n">table</span> |
| <a id="__codelineno-0-269" name="__codelineno-0-269"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_autocommit</span> <span class="o">=</span> <span class="n">autocommit</span> |
| <a id="__codelineno-0-270" name="__codelineno-0-270"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-271" name="__codelineno-0-271"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-272" name="__codelineno-0-272"></a> |
| <a id="__codelineno-0-273" name="__codelineno-0-273"></a> <span class="nd">@property</span> |
| <a id="__codelineno-0-274" name="__codelineno-0-274"></a> <span class="k">def</span><span class="w"> </span><span class="nf">table_metadata</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">TableMetadata</span><span class="p">:</span> |
| <a id="__codelineno-0-275" name="__codelineno-0-275"></a> <span class="k">return</span> <span class="n">update_table_metadata</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">metadata</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">)</span> |
| <a id="__codelineno-0-276" name="__codelineno-0-276"></a> |
| <a id="__codelineno-0-277" name="__codelineno-0-277"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="w"> </span><span class="sd">"""Start a transaction to update the table."""</span> |
| <a id="__codelineno-0-279" name="__codelineno-0-279"></a> <span class="k">return</span> <span class="bp">self</span> |
| <a id="__codelineno-0-280" name="__codelineno-0-280"></a> |
| <a id="__codelineno-0-281" name="__codelineno-0-281"></a> <span class="k">def</span><span class="w"> </span><span class="fm">__exit__</span><span class="p">(</span> |
| <a id="__codelineno-0-282" name="__codelineno-0-282"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">exctype</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">]],</span> <span class="n">excinst</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">],</span> <span class="n">exctb</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">TracebackType</span><span class="p">]</span> |
| <a id="__codelineno-0-283" name="__codelineno-0-283"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-284" name="__codelineno-0-284"></a><span class="w"> </span><span class="sd">"""Close and commit the transaction if no exceptions have been raised."""</span> |
| <a id="__codelineno-0-285" name="__codelineno-0-285"></a> <span class="k">if</span> <span class="n">exctype</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">excinst</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">exctb</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-286" name="__codelineno-0-286"></a> <span class="bp">self</span><span class="o">.</span><span class="n">commit_transaction</span><span class="p">()</span> |
| <a id="__codelineno-0-287" name="__codelineno-0-287"></a> |
| <a id="__codelineno-0-288" name="__codelineno-0-288"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_apply</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">updates</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableUpdate</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span> <span class="n">requirements</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TableRequirement</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">())</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="w"> </span><span class="sd">"""Check if the requirements are met, and applies the updates to the metadata."""</span> |
| <a id="__codelineno-0-290" name="__codelineno-0-290"></a> <span class="k">for</span> <span class="n">requirement</span> <span class="ow">in</span> <span class="n">requirements</span><span class="p">:</span> |
| <a id="__codelineno-0-291" name="__codelineno-0-291"></a> <span class="n">requirement</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">)</span> |
| <a id="__codelineno-0-292" name="__codelineno-0-292"></a> |
| <a id="__codelineno-0-293" name="__codelineno-0-293"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">+=</span> <span class="n">updates</span> |
| <a id="__codelineno-0-294" name="__codelineno-0-294"></a> |
| <a id="__codelineno-0-295" name="__codelineno-0-295"></a> <span class="c1"># For the requirements, it does not make sense to add a requirement more than once</span> |
| <a id="__codelineno-0-296" name="__codelineno-0-296"></a> <span class="c1"># For example, you cannot assert that the current schema has two different IDs</span> |
| <a id="__codelineno-0-297" name="__codelineno-0-297"></a> <span class="n">existing_requirements</span> <span class="o">=</span> <span class="p">{</span><span class="nb">type</span><span class="p">(</span><span class="n">requirement</span><span class="p">)</span> <span class="k">for</span> <span class="n">requirement</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span><span class="p">}</span> |
| <a id="__codelineno-0-298" name="__codelineno-0-298"></a> <span class="k">for</span> <span class="n">new_requirement</span> <span class="ow">in</span> <span class="n">requirements</span><span class="p">:</span> |
| <a id="__codelineno-0-299" name="__codelineno-0-299"></a> <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">new_requirement</span><span class="p">)</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">existing_requirements</span><span class="p">:</span> |
| <a id="__codelineno-0-300" name="__codelineno-0-300"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">+</span> <span class="p">(</span><span class="n">new_requirement</span><span class="p">,)</span> |
| <a id="__codelineno-0-301" name="__codelineno-0-301"></a> |
| <a id="__codelineno-0-302" name="__codelineno-0-302"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_autocommit</span><span class="p">:</span> |
| <a id="__codelineno-0-303" name="__codelineno-0-303"></a> <span class="bp">self</span><span class="o">.</span><span class="n">commit_transaction</span><span class="p">()</span> |
| <a id="__codelineno-0-304" name="__codelineno-0-304"></a> |
| <a id="__codelineno-0-305" name="__codelineno-0-305"></a> <span class="k">return</span> <span class="bp">self</span> |
| <a id="__codelineno-0-306" name="__codelineno-0-306"></a> |
| <a id="__codelineno-0-307" name="__codelineno-0-307"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_scan</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">row_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">DataScan</span><span class="p">:</span> |
| <a id="__codelineno-0-308" name="__codelineno-0-308"></a><span class="w"> </span><span class="sd">"""Minimal data scan of the table with the current state of the transaction."""</span> |
| <a id="__codelineno-0-309" name="__codelineno-0-309"></a> <span class="k">return</span> <span class="n">DataScan</span><span class="p">(</span> |
| <a id="__codelineno-0-310" name="__codelineno-0-310"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="n">row_filter</span><span class="o">=</span><span class="n">row_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span> |
| <a id="__codelineno-0-311" name="__codelineno-0-311"></a> <span class="p">)</span> |
| <a id="__codelineno-0-312" name="__codelineno-0-312"></a> |
| <a id="__codelineno-0-313" name="__codelineno-0-313"></a> <span class="k">def</span><span class="w"> </span><span class="nf">upgrade_table_version</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">format_version</span><span class="p">:</span> <span class="n">TableVersion</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-314" name="__codelineno-0-314"></a><span class="w"> </span><span class="sd">"""Set the table to a certain version.</span> |
| <a id="__codelineno-0-315" name="__codelineno-0-315"></a> |
| <a id="__codelineno-0-316" name="__codelineno-0-316"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="sd"> format_version: The newly set version.</span> |
| <a id="__codelineno-0-318" name="__codelineno-0-318"></a> |
| <a id="__codelineno-0-319" name="__codelineno-0-319"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-321" name="__codelineno-0-321"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-322" name="__codelineno-0-322"></a> <span class="k">if</span> <span class="n">format_version</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">}:</span> |
| <a id="__codelineno-0-323" name="__codelineno-0-323"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unsupported table format version: </span><span class="si">{</span><span class="n">format_version</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-324" name="__codelineno-0-324"></a> |
| <a id="__codelineno-0-325" name="__codelineno-0-325"></a> <span class="k">if</span> <span class="n">format_version</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">:</span> |
| <a id="__codelineno-0-326" name="__codelineno-0-326"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot downgrade v</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="si">}</span><span class="s2"> table to v</span><span class="si">{</span><span class="n">format_version</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-327" name="__codelineno-0-327"></a> |
| <a id="__codelineno-0-328" name="__codelineno-0-328"></a> <span class="k">if</span> <span class="n">format_version</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">:</span> |
| <a id="__codelineno-0-329" name="__codelineno-0-329"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">((</span><span class="n">UpgradeFormatVersionUpdate</span><span class="p">(</span><span class="n">format_version</span><span class="o">=</span><span class="n">format_version</span><span class="p">),))</span> |
| <a id="__codelineno-0-330" name="__codelineno-0-330"></a> |
| <a id="__codelineno-0-331" name="__codelineno-0-331"></a> <span class="k">return</span> <span class="bp">self</span> |
| <a id="__codelineno-0-332" name="__codelineno-0-332"></a> |
| <a id="__codelineno-0-333" name="__codelineno-0-333"></a> <span class="k">def</span><span class="w"> </span><span class="nf">set_properties</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">properties</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="w"> </span><span class="sd">"""Set properties.</span> |
| <a id="__codelineno-0-335" name="__codelineno-0-335"></a> |
| <a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd"> When a property is already set, it will be overwritten.</span> |
| <a id="__codelineno-0-337" name="__codelineno-0-337"></a> |
| <a id="__codelineno-0-338" name="__codelineno-0-338"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-339" name="__codelineno-0-339"></a><span class="sd"> properties: The properties set on the table.</span> |
| <a id="__codelineno-0-340" name="__codelineno-0-340"></a><span class="sd"> kwargs: properties can also be pass as kwargs.</span> |
| <a id="__codelineno-0-341" name="__codelineno-0-341"></a> |
| <a id="__codelineno-0-342" name="__codelineno-0-342"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-343" name="__codelineno-0-343"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-344" name="__codelineno-0-344"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-345" name="__codelineno-0-345"></a> <span class="k">if</span> <span class="n">properties</span> <span class="ow">and</span> <span class="n">kwargs</span><span class="p">:</span> |
| <a id="__codelineno-0-346" name="__codelineno-0-346"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Cannot pass both properties and kwargs"</span><span class="p">)</span> |
| <a id="__codelineno-0-347" name="__codelineno-0-347"></a> <span class="n">updates</span> <span class="o">=</span> <span class="n">properties</span> <span class="ow">or</span> <span class="n">kwargs</span> |
| <a id="__codelineno-0-348" name="__codelineno-0-348"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">((</span><span class="n">SetPropertiesUpdate</span><span class="p">(</span><span class="n">updates</span><span class="o">=</span><span class="n">updates</span><span class="p">),))</span> |
| <a id="__codelineno-0-349" name="__codelineno-0-349"></a> |
| <a id="__codelineno-0-350" name="__codelineno-0-350"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_set_ref_snapshot</span><span class="p">(</span> |
| <a id="__codelineno-0-351" name="__codelineno-0-351"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-352" name="__codelineno-0-352"></a> <span class="n">snapshot_id</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> |
| <a id="__codelineno-0-353" name="__codelineno-0-353"></a> <span class="n">ref_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <a id="__codelineno-0-354" name="__codelineno-0-354"></a> <span class="nb">type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <a id="__codelineno-0-355" name="__codelineno-0-355"></a> <span class="n">max_ref_age_ms</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-356" name="__codelineno-0-356"></a> <span class="n">max_snapshot_age_ms</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-357" name="__codelineno-0-357"></a> <span class="n">min_snapshots_to_keep</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-358" name="__codelineno-0-358"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">UpdatesAndRequirements</span><span class="p">:</span> |
| <a id="__codelineno-0-359" name="__codelineno-0-359"></a><span class="w"> </span><span class="sd">"""Update a ref to a snapshot.</span> |
| <a id="__codelineno-0-360" name="__codelineno-0-360"></a> |
| <a id="__codelineno-0-361" name="__codelineno-0-361"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-362" name="__codelineno-0-362"></a><span class="sd"> The updates and requirements for the set-snapshot-ref staged</span> |
| <a id="__codelineno-0-363" name="__codelineno-0-363"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-364" name="__codelineno-0-364"></a> <span class="n">updates</span> <span class="o">=</span> <span class="p">(</span> |
| <a id="__codelineno-0-365" name="__codelineno-0-365"></a> <span class="n">SetSnapshotRefUpdate</span><span class="p">(</span> |
| <a id="__codelineno-0-366" name="__codelineno-0-366"></a> <span class="n">snapshot_id</span><span class="o">=</span><span class="n">snapshot_id</span><span class="p">,</span> |
| <a id="__codelineno-0-367" name="__codelineno-0-367"></a> <span class="n">ref_name</span><span class="o">=</span><span class="n">ref_name</span><span class="p">,</span> |
| <a id="__codelineno-0-368" name="__codelineno-0-368"></a> <span class="nb">type</span><span class="o">=</span><span class="nb">type</span><span class="p">,</span> |
| <a id="__codelineno-0-369" name="__codelineno-0-369"></a> <span class="n">max_ref_age_ms</span><span class="o">=</span><span class="n">max_ref_age_ms</span><span class="p">,</span> |
| <a id="__codelineno-0-370" name="__codelineno-0-370"></a> <span class="n">max_snapshot_age_ms</span><span class="o">=</span><span class="n">max_snapshot_age_ms</span><span class="p">,</span> |
| <a id="__codelineno-0-371" name="__codelineno-0-371"></a> <span class="n">min_snapshots_to_keep</span><span class="o">=</span><span class="n">min_snapshots_to_keep</span><span class="p">,</span> |
| <a id="__codelineno-0-372" name="__codelineno-0-372"></a> <span class="p">),</span> |
| <a id="__codelineno-0-373" name="__codelineno-0-373"></a> <span class="p">)</span> |
| <a id="__codelineno-0-374" name="__codelineno-0-374"></a> <span class="n">requirements</span> <span class="o">=</span> <span class="p">(</span> |
| <a id="__codelineno-0-375" name="__codelineno-0-375"></a> <span class="n">AssertRefSnapshotId</span><span class="p">(</span> |
| <a id="__codelineno-0-376" name="__codelineno-0-376"></a> <span class="n">snapshot_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">refs</span><span class="p">[</span><span class="n">ref_name</span><span class="p">]</span><span class="o">.</span><span class="n">snapshot_id</span> <span class="k">if</span> <span class="n">ref_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">refs</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-377" name="__codelineno-0-377"></a> <span class="n">ref</span><span class="o">=</span><span class="n">ref_name</span><span class="p">,</span> |
| <a id="__codelineno-0-378" name="__codelineno-0-378"></a> <span class="p">),</span> |
| <a id="__codelineno-0-379" name="__codelineno-0-379"></a> <span class="p">)</span> |
| <a id="__codelineno-0-380" name="__codelineno-0-380"></a> |
| <a id="__codelineno-0-381" name="__codelineno-0-381"></a> <span class="k">return</span> <span class="n">updates</span><span class="p">,</span> <span class="n">requirements</span> |
| <a id="__codelineno-0-382" name="__codelineno-0-382"></a> |
| <a id="__codelineno-0-383" name="__codelineno-0-383"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_build_partition_predicate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition_records</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="n">Record</span><span class="p">])</span> <span class="o">-></span> <span class="n">BooleanExpression</span><span class="p">:</span> |
| <a id="__codelineno-0-384" name="__codelineno-0-384"></a><span class="w"> </span><span class="sd">"""Build a filter predicate matching any of the input partition records.</span> |
| <a id="__codelineno-0-385" name="__codelineno-0-385"></a> |
| <a id="__codelineno-0-386" name="__codelineno-0-386"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-387" name="__codelineno-0-387"></a><span class="sd"> partition_records: A set of partition records to match</span> |
| <a id="__codelineno-0-388" name="__codelineno-0-388"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-389" name="__codelineno-0-389"></a><span class="sd"> A predicate matching any of the input partition records.</span> |
| <a id="__codelineno-0-390" name="__codelineno-0-390"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-391" name="__codelineno-0-391"></a> <span class="n">partition_spec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span> |
| <a id="__codelineno-0-392" name="__codelineno-0-392"></a> <span class="n">schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span> |
| <a id="__codelineno-0-393" name="__codelineno-0-393"></a> <span class="n">partition_fields</span> <span class="o">=</span> <span class="p">[</span><span class="n">schema</span><span class="o">.</span><span class="n">find_field</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">source_id</span><span class="p">)</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">partition_spec</span><span class="o">.</span><span class="n">fields</span><span class="p">]</span> |
| <a id="__codelineno-0-394" name="__codelineno-0-394"></a> |
| <a id="__codelineno-0-395" name="__codelineno-0-395"></a> <span class="n">expr</span><span class="p">:</span> <span class="n">BooleanExpression</span> <span class="o">=</span> <span class="n">AlwaysFalse</span><span class="p">()</span> |
| <a id="__codelineno-0-396" name="__codelineno-0-396"></a> <span class="k">for</span> <span class="n">partition_record</span> <span class="ow">in</span> <span class="n">partition_records</span><span class="p">:</span> |
| <a id="__codelineno-0-397" name="__codelineno-0-397"></a> <span class="n">match_partition_expression</span><span class="p">:</span> <span class="n">BooleanExpression</span> <span class="o">=</span> <span class="n">AlwaysTrue</span><span class="p">()</span> |
| <a id="__codelineno-0-398" name="__codelineno-0-398"></a> |
| <a id="__codelineno-0-399" name="__codelineno-0-399"></a> <span class="k">for</span> <span class="n">pos</span><span class="p">,</span> <span class="n">partition_field</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">partition_fields</span><span class="p">):</span> |
| <a id="__codelineno-0-400" name="__codelineno-0-400"></a> <span class="n">predicate</span> <span class="o">=</span> <span class="p">(</span> |
| <a id="__codelineno-0-401" name="__codelineno-0-401"></a> <span class="n">EqualTo</span><span class="p">(</span><span class="n">Reference</span><span class="p">(</span><span class="n">partition_field</span><span class="p">),</span> <span class="n">partition_record</span><span class="p">[</span><span class="n">pos</span><span class="p">])</span> |
| <a id="__codelineno-0-402" name="__codelineno-0-402"></a> <span class="k">if</span> <span class="n">partition_record</span><span class="p">[</span><span class="n">pos</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> |
| <a id="__codelineno-0-403" name="__codelineno-0-403"></a> <span class="k">else</span> <span class="n">IsNull</span><span class="p">(</span><span class="n">Reference</span><span class="p">(</span><span class="n">partition_field</span><span class="p">))</span> |
| <a id="__codelineno-0-404" name="__codelineno-0-404"></a> <span class="p">)</span> |
| <a id="__codelineno-0-405" name="__codelineno-0-405"></a> <span class="n">match_partition_expression</span> <span class="o">=</span> <span class="n">And</span><span class="p">(</span><span class="n">match_partition_expression</span><span class="p">,</span> <span class="n">predicate</span><span class="p">)</span> |
| <a id="__codelineno-0-406" name="__codelineno-0-406"></a> <span class="n">expr</span> <span class="o">=</span> <span class="n">Or</span><span class="p">(</span><span class="n">expr</span><span class="p">,</span> <span class="n">match_partition_expression</span><span class="p">)</span> |
| <a id="__codelineno-0-407" name="__codelineno-0-407"></a> <span class="k">return</span> <span class="n">expr</span> |
| <a id="__codelineno-0-408" name="__codelineno-0-408"></a> |
| <a id="__codelineno-0-409" name="__codelineno-0-409"></a> <span class="k">def</span><span class="w"> </span><span class="nf">_append_snapshot_producer</span><span class="p">(</span> |
| <a id="__codelineno-0-410" name="__codelineno-0-410"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-411" name="__codelineno-0-411"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">_FastAppendFiles</span><span class="p">:</span> |
| <a id="__codelineno-0-412" name="__codelineno-0-412"></a><span class="w"> </span><span class="sd">"""Determine the append type based on table properties.</span> |
| <a id="__codelineno-0-413" name="__codelineno-0-413"></a> |
| <a id="__codelineno-0-414" name="__codelineno-0-414"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-415" name="__codelineno-0-415"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-416" name="__codelineno-0-416"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-417" name="__codelineno-0-417"></a><span class="sd"> Either a fast-append or a merge-append snapshot producer.</span> |
| <a id="__codelineno-0-418" name="__codelineno-0-418"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-419" name="__codelineno-0-419"></a> <span class="n">manifest_merge_enabled</span> <span class="o">=</span> <span class="n">property_as_bool</span><span class="p">(</span> |
| <a id="__codelineno-0-420" name="__codelineno-0-420"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">properties</span><span class="p">,</span> |
| <a id="__codelineno-0-421" name="__codelineno-0-421"></a> <span class="n">TableProperties</span><span class="o">.</span><span class="n">MANIFEST_MERGE_ENABLED</span><span class="p">,</span> |
| <a id="__codelineno-0-422" name="__codelineno-0-422"></a> <span class="n">TableProperties</span><span class="o">.</span><span class="n">MANIFEST_MERGE_ENABLED_DEFAULT</span><span class="p">,</span> |
| <a id="__codelineno-0-423" name="__codelineno-0-423"></a> <span class="p">)</span> |
| <a id="__codelineno-0-424" name="__codelineno-0-424"></a> <span class="n">update_snapshot</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">update_snapshot</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-425" name="__codelineno-0-425"></a> <span class="k">return</span> <span class="n">update_snapshot</span><span class="o">.</span><span class="n">merge_append</span><span class="p">()</span> <span class="k">if</span> <span class="n">manifest_merge_enabled</span> <span class="k">else</span> <span class="n">update_snapshot</span><span class="o">.</span><span class="n">fast_append</span><span class="p">()</span> |
| <a id="__codelineno-0-426" name="__codelineno-0-426"></a> |
| <a id="__codelineno-0-427" name="__codelineno-0-427"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">allow_incompatible_changes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSchema</span><span class="p">:</span> |
| <a id="__codelineno-0-428" name="__codelineno-0-428"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSchema to alter the columns of this table.</span> |
| <a id="__codelineno-0-429" name="__codelineno-0-429"></a> |
| <a id="__codelineno-0-430" name="__codelineno-0-430"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-431" name="__codelineno-0-431"></a><span class="sd"> allow_incompatible_changes: If changes are allowed that might break downstream consumers.</span> |
| <a id="__codelineno-0-432" name="__codelineno-0-432"></a><span class="sd"> case_sensitive: If field names are case-sensitive.</span> |
| <a id="__codelineno-0-433" name="__codelineno-0-433"></a> |
| <a id="__codelineno-0-434" name="__codelineno-0-434"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-435" name="__codelineno-0-435"></a><span class="sd"> A new UpdateSchema.</span> |
| <a id="__codelineno-0-436" name="__codelineno-0-436"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-437" name="__codelineno-0-437"></a> <span class="k">return</span> <span class="n">UpdateSchema</span><span class="p">(</span> |
| <a id="__codelineno-0-438" name="__codelineno-0-438"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-439" name="__codelineno-0-439"></a> <span class="n">allow_incompatible_changes</span><span class="o">=</span><span class="n">allow_incompatible_changes</span><span class="p">,</span> |
| <a id="__codelineno-0-440" name="__codelineno-0-440"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-441" name="__codelineno-0-441"></a> <span class="n">name_mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">(),</span> |
| <a id="__codelineno-0-442" name="__codelineno-0-442"></a> <span class="p">)</span> |
| <a id="__codelineno-0-443" name="__codelineno-0-443"></a> |
| <a id="__codelineno-0-444" name="__codelineno-0-444"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_sort_order</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSortOrder</span><span class="p">:</span> |
| <a id="__codelineno-0-445" name="__codelineno-0-445"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSortOrder to update the sort order of this table.</span> |
| <a id="__codelineno-0-446" name="__codelineno-0-446"></a> |
| <a id="__codelineno-0-447" name="__codelineno-0-447"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-448" name="__codelineno-0-448"></a><span class="sd"> case_sensitive: If field names are case-sensitive.</span> |
| <a id="__codelineno-0-449" name="__codelineno-0-449"></a> |
| <a id="__codelineno-0-450" name="__codelineno-0-450"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-451" name="__codelineno-0-451"></a><span class="sd"> A new UpdateSortOrder.</span> |
| <a id="__codelineno-0-452" name="__codelineno-0-452"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-453" name="__codelineno-0-453"></a> <span class="k">return</span> <span class="n">UpdateSortOrder</span><span class="p">(</span> |
| <a id="__codelineno-0-454" name="__codelineno-0-454"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-455" name="__codelineno-0-455"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-456" name="__codelineno-0-456"></a> <span class="p">)</span> |
| <a id="__codelineno-0-457" name="__codelineno-0-457"></a> |
| <a id="__codelineno-0-458" name="__codelineno-0-458"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_snapshot</span><span class="p">(</span> |
| <a id="__codelineno-0-459" name="__codelineno-0-459"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-460" name="__codelineno-0-460"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSnapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-461" name="__codelineno-0-461"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSnapshot to produce a new snapshot for the table.</span> |
| <a id="__codelineno-0-462" name="__codelineno-0-462"></a> |
| <a id="__codelineno-0-463" name="__codelineno-0-463"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-464" name="__codelineno-0-464"></a><span class="sd"> A new UpdateSnapshot</span> |
| <a id="__codelineno-0-465" name="__codelineno-0-465"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-466" name="__codelineno-0-466"></a> <span class="k">return</span> <span class="n">UpdateSnapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">)</span> |
| <a id="__codelineno-0-467" name="__codelineno-0-467"></a> |
| <a id="__codelineno-0-468" name="__codelineno-0-468"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_statistics</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateStatistics</span><span class="p">:</span> |
| <a id="__codelineno-0-469" name="__codelineno-0-469"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-470" name="__codelineno-0-470"></a><span class="sd"> Create a new UpdateStatistics to update the statistics of the table.</span> |
| <a id="__codelineno-0-471" name="__codelineno-0-471"></a> |
| <a id="__codelineno-0-472" name="__codelineno-0-472"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-473" name="__codelineno-0-473"></a><span class="sd"> A new UpdateStatistics</span> |
| <a id="__codelineno-0-474" name="__codelineno-0-474"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-475" name="__codelineno-0-475"></a> <span class="k">return</span> <span class="n">UpdateStatistics</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-476" name="__codelineno-0-476"></a> |
| <a id="__codelineno-0-477" name="__codelineno-0-477"></a> <span class="k">def</span><span class="w"> </span><span class="nf">append</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-478" name="__codelineno-0-478"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-479" name="__codelineno-0-479"></a><span class="sd"> Shorthand API for appending a PyArrow table to a table transaction.</span> |
| <a id="__codelineno-0-480" name="__codelineno-0-480"></a> |
| <a id="__codelineno-0-481" name="__codelineno-0-481"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-482" name="__codelineno-0-482"></a><span class="sd"> df: The Arrow dataframe that will be appended to overwrite the table</span> |
| <a id="__codelineno-0-483" name="__codelineno-0-483"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-484" name="__codelineno-0-484"></a><span class="sd"> branch: Branch Reference to run the append operation</span> |
| <a id="__codelineno-0-485" name="__codelineno-0-485"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-486" name="__codelineno-0-486"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-487" name="__codelineno-0-487"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-488" name="__codelineno-0-488"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-489" name="__codelineno-0-489"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-490" name="__codelineno-0-490"></a> |
| <a id="__codelineno-0-491" name="__codelineno-0-491"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">,</span> <span class="n">_dataframe_to_data_files</span> |
| <a id="__codelineno-0-492" name="__codelineno-0-492"></a> |
| <a id="__codelineno-0-493" name="__codelineno-0-493"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-494" name="__codelineno-0-494"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Expected PyArrow table, got: </span><span class="si">{</span><span class="n">df</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-495" name="__codelineno-0-495"></a> |
| <a id="__codelineno-0-496" name="__codelineno-0-496"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-497" name="__codelineno-0-497"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-498" name="__codelineno-0-498"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-499" name="__codelineno-0-499"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-500" name="__codelineno-0-500"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-501" name="__codelineno-0-501"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-502" name="__codelineno-0-502"></a> <span class="p">)</span> |
| <a id="__codelineno-0-503" name="__codelineno-0-503"></a> |
| <a id="__codelineno-0-504" name="__codelineno-0-504"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-505" name="__codelineno-0-505"></a> <span class="c1"># skip writing data files if the dataframe is empty</span> |
| <a id="__codelineno-0-506" name="__codelineno-0-506"></a> <span class="k">if</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-507" name="__codelineno-0-507"></a> <span class="n">data_files</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span> |
| <a id="__codelineno-0-508" name="__codelineno-0-508"></a> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-509" name="__codelineno-0-509"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">write_uuid</span><span class="o">=</span><span class="n">append_files</span><span class="o">.</span><span class="n">commit_uuid</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-510" name="__codelineno-0-510"></a> <span class="p">)</span> |
| <a id="__codelineno-0-511" name="__codelineno-0-511"></a> <span class="p">)</span> |
| <a id="__codelineno-0-512" name="__codelineno-0-512"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-513" name="__codelineno-0-513"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-514" name="__codelineno-0-514"></a> |
| <a id="__codelineno-0-515" name="__codelineno-0-515"></a> <span class="k">def</span><span class="w"> </span><span class="nf">dynamic_partition_overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-516" name="__codelineno-0-516"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-517" name="__codelineno-0-517"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-518" name="__codelineno-0-518"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-519" name="__codelineno-0-519"></a><span class="sd"> Shorthand for overwriting existing partitions with a PyArrow table.</span> |
| <a id="__codelineno-0-520" name="__codelineno-0-520"></a> |
| <a id="__codelineno-0-521" name="__codelineno-0-521"></a><span class="sd"> The function detects partition values in the provided arrow table using the current</span> |
| <a id="__codelineno-0-522" name="__codelineno-0-522"></a><span class="sd"> partition spec, and deletes existing partitions matching these values. Finally, the</span> |
| <a id="__codelineno-0-523" name="__codelineno-0-523"></a><span class="sd"> data in the table is appended to the table.</span> |
| <a id="__codelineno-0-524" name="__codelineno-0-524"></a> |
| <a id="__codelineno-0-525" name="__codelineno-0-525"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-526" name="__codelineno-0-526"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-527" name="__codelineno-0-527"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-528" name="__codelineno-0-528"></a><span class="sd"> branch: Branch Reference to run the dynamic partition overwrite operation</span> |
| <a id="__codelineno-0-529" name="__codelineno-0-529"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-530" name="__codelineno-0-530"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-531" name="__codelineno-0-531"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-532" name="__codelineno-0-532"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-533" name="__codelineno-0-533"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-534" name="__codelineno-0-534"></a> |
| <a id="__codelineno-0-535" name="__codelineno-0-535"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">,</span> <span class="n">_dataframe_to_data_files</span> |
| <a id="__codelineno-0-536" name="__codelineno-0-536"></a> |
| <a id="__codelineno-0-537" name="__codelineno-0-537"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-538" name="__codelineno-0-538"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Expected PyArrow table, got: </span><span class="si">{</span><span class="n">df</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-539" name="__codelineno-0-539"></a> |
| <a id="__codelineno-0-540" name="__codelineno-0-540"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span><span class="o">.</span><span class="n">is_unpartitioned</span><span class="p">():</span> |
| <a id="__codelineno-0-541" name="__codelineno-0-541"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Cannot apply dynamic overwrite on an unpartitioned table."</span><span class="p">)</span> |
| <a id="__codelineno-0-542" name="__codelineno-0-542"></a> |
| <a id="__codelineno-0-543" name="__codelineno-0-543"></a> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span><span class="o">.</span><span class="n">fields</span><span class="p">:</span> |
| <a id="__codelineno-0-544" name="__codelineno-0-544"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">transform</span><span class="p">,</span> <span class="n">IdentityTransform</span><span class="p">):</span> |
| <a id="__codelineno-0-545" name="__codelineno-0-545"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <a id="__codelineno-0-546" name="__codelineno-0-546"></a> <span class="sa">f</span><span class="s2">"For now dynamic overwrite does not support a table with non-identity-transform field in the latest partition spec: </span><span class="si">{</span><span class="n">field</span><span class="si">}</span><span class="s2">"</span> |
| <a id="__codelineno-0-547" name="__codelineno-0-547"></a> <span class="p">)</span> |
| <a id="__codelineno-0-548" name="__codelineno-0-548"></a> |
| <a id="__codelineno-0-549" name="__codelineno-0-549"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-550" name="__codelineno-0-550"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-551" name="__codelineno-0-551"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-552" name="__codelineno-0-552"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-553" name="__codelineno-0-553"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-554" name="__codelineno-0-554"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-555" name="__codelineno-0-555"></a> <span class="p">)</span> |
| <a id="__codelineno-0-556" name="__codelineno-0-556"></a> |
| <a id="__codelineno-0-557" name="__codelineno-0-557"></a> <span class="c1"># If dataframe does not have data, there is no need to overwrite</span> |
| <a id="__codelineno-0-558" name="__codelineno-0-558"></a> <span class="k">if</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-559" name="__codelineno-0-559"></a> <span class="k">return</span> |
| <a id="__codelineno-0-560" name="__codelineno-0-560"></a> |
| <a id="__codelineno-0-561" name="__codelineno-0-561"></a> <span class="n">append_snapshot_commit_uuid</span> <span class="o">=</span> <span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">()</span> |
| <a id="__codelineno-0-562" name="__codelineno-0-562"></a> <span class="n">data_files</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">DataFile</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span> |
| <a id="__codelineno-0-563" name="__codelineno-0-563"></a> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-564" name="__codelineno-0-564"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">metadata</span><span class="p">,</span> <span class="n">write_uuid</span><span class="o">=</span><span class="n">append_snapshot_commit_uuid</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-565" name="__codelineno-0-565"></a> <span class="p">)</span> |
| <a id="__codelineno-0-566" name="__codelineno-0-566"></a> <span class="p">)</span> |
| <a id="__codelineno-0-567" name="__codelineno-0-567"></a> |
| <a id="__codelineno-0-568" name="__codelineno-0-568"></a> <span class="n">partitions_to_overwrite</span> <span class="o">=</span> <span class="p">{</span><span class="n">data_file</span><span class="o">.</span><span class="n">partition</span> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">}</span> |
| <a id="__codelineno-0-569" name="__codelineno-0-569"></a> <span class="n">delete_filter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_partition_predicate</span><span class="p">(</span><span class="n">partition_records</span><span class="o">=</span><span class="n">partitions_to_overwrite</span><span class="p">)</span> |
| <a id="__codelineno-0-570" name="__codelineno-0-570"></a> <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">delete_filter</span><span class="o">=</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-571" name="__codelineno-0-571"></a> |
| <a id="__codelineno-0-572" name="__codelineno-0-572"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-573" name="__codelineno-0-573"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">commit_uuid</span> <span class="o">=</span> <span class="n">append_snapshot_commit_uuid</span> |
| <a id="__codelineno-0-574" name="__codelineno-0-574"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-575" name="__codelineno-0-575"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-576" name="__codelineno-0-576"></a> |
| <a id="__codelineno-0-577" name="__codelineno-0-577"></a> <span class="k">def</span><span class="w"> </span><span class="nf">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-578" name="__codelineno-0-578"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-579" name="__codelineno-0-579"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-580" name="__codelineno-0-580"></a> <span class="n">overwrite_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BooleanExpression</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-581" name="__codelineno-0-581"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-582" name="__codelineno-0-582"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-583" name="__codelineno-0-583"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-584" name="__codelineno-0-584"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-585" name="__codelineno-0-585"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-586" name="__codelineno-0-586"></a><span class="sd"> Shorthand for adding a table overwrite with a PyArrow table to the transaction.</span> |
| <a id="__codelineno-0-587" name="__codelineno-0-587"></a> |
| <a id="__codelineno-0-588" name="__codelineno-0-588"></a><span class="sd"> An overwrite may produce zero or more snapshots based on the operation:</span> |
| <a id="__codelineno-0-589" name="__codelineno-0-589"></a> |
| <a id="__codelineno-0-590" name="__codelineno-0-590"></a><span class="sd"> - DELETE: In case existing Parquet files can be dropped completely.</span> |
| <a id="__codelineno-0-591" name="__codelineno-0-591"></a><span class="sd"> - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the overwrite filter.</span> |
| <a id="__codelineno-0-592" name="__codelineno-0-592"></a><span class="sd"> - APPEND: In case new data is being inserted into the table.</span> |
| <a id="__codelineno-0-593" name="__codelineno-0-593"></a> |
| <a id="__codelineno-0-594" name="__codelineno-0-594"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-595" name="__codelineno-0-595"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-596" name="__codelineno-0-596"></a><span class="sd"> overwrite_filter: ALWAYS_TRUE when you overwrite all the data,</span> |
| <a id="__codelineno-0-597" name="__codelineno-0-597"></a><span class="sd"> or a boolean expression in case of a partial overwrite</span> |
| <a id="__codelineno-0-598" name="__codelineno-0-598"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-599" name="__codelineno-0-599"></a><span class="sd"> case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive</span> |
| <a id="__codelineno-0-600" name="__codelineno-0-600"></a><span class="sd"> branch: Branch Reference to run the overwrite operation</span> |
| <a id="__codelineno-0-601" name="__codelineno-0-601"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-602" name="__codelineno-0-602"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-603" name="__codelineno-0-603"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-604" name="__codelineno-0-604"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-605" name="__codelineno-0-605"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-606" name="__codelineno-0-606"></a> |
| <a id="__codelineno-0-607" name="__codelineno-0-607"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">,</span> <span class="n">_dataframe_to_data_files</span> |
| <a id="__codelineno-0-608" name="__codelineno-0-608"></a> |
| <a id="__codelineno-0-609" name="__codelineno-0-609"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-610" name="__codelineno-0-610"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Expected PyArrow table, got: </span><span class="si">{</span><span class="n">df</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-611" name="__codelineno-0-611"></a> |
| <a id="__codelineno-0-612" name="__codelineno-0-612"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-613" name="__codelineno-0-613"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-614" name="__codelineno-0-614"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-615" name="__codelineno-0-615"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-616" name="__codelineno-0-616"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-617" name="__codelineno-0-617"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-618" name="__codelineno-0-618"></a> <span class="p">)</span> |
| <a id="__codelineno-0-619" name="__codelineno-0-619"></a> |
| <a id="__codelineno-0-620" name="__codelineno-0-620"></a> <span class="k">if</span> <span class="n">overwrite_filter</span> <span class="o">!=</span> <span class="n">AlwaysFalse</span><span class="p">():</span> |
| <a id="__codelineno-0-621" name="__codelineno-0-621"></a> <span class="c1"># Only delete when the filter is != AlwaysFalse</span> |
| <a id="__codelineno-0-622" name="__codelineno-0-622"></a> <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-623" name="__codelineno-0-623"></a> <span class="n">delete_filter</span><span class="o">=</span><span class="n">overwrite_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-624" name="__codelineno-0-624"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-625" name="__codelineno-0-625"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> |
| <a id="__codelineno-0-626" name="__codelineno-0-626"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-627" name="__codelineno-0-627"></a> <span class="p">)</span> |
| <a id="__codelineno-0-628" name="__codelineno-0-628"></a> |
| <a id="__codelineno-0-629" name="__codelineno-0-629"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-630" name="__codelineno-0-630"></a> <span class="c1"># skip writing data files if the dataframe is empty</span> |
| <a id="__codelineno-0-631" name="__codelineno-0-631"></a> <span class="k">if</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-632" name="__codelineno-0-632"></a> <span class="n">data_files</span> <span class="o">=</span> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-633" name="__codelineno-0-633"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">write_uuid</span><span class="o">=</span><span class="n">append_files</span><span class="o">.</span><span class="n">commit_uuid</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-634" name="__codelineno-0-634"></a> <span class="p">)</span> |
| <a id="__codelineno-0-635" name="__codelineno-0-635"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-636" name="__codelineno-0-636"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-637" name="__codelineno-0-637"></a> |
| <a id="__codelineno-0-638" name="__codelineno-0-638"></a> <span class="k">def</span><span class="w"> </span><span class="nf">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-639" name="__codelineno-0-639"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-640" name="__codelineno-0-640"></a> <span class="n">delete_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">],</span> |
| <a id="__codelineno-0-641" name="__codelineno-0-641"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-642" name="__codelineno-0-642"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-643" name="__codelineno-0-643"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-644" name="__codelineno-0-644"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-645" name="__codelineno-0-645"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-646" name="__codelineno-0-646"></a><span class="sd"> Shorthand for deleting record from a table.</span> |
| <a id="__codelineno-0-647" name="__codelineno-0-647"></a> |
| <a id="__codelineno-0-648" name="__codelineno-0-648"></a><span class="sd"> A delete may produce zero or more snapshots based on the operation:</span> |
| <a id="__codelineno-0-649" name="__codelineno-0-649"></a> |
| <a id="__codelineno-0-650" name="__codelineno-0-650"></a><span class="sd"> - DELETE: In case existing Parquet files can be dropped completely.</span> |
| <a id="__codelineno-0-651" name="__codelineno-0-651"></a><span class="sd"> - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the delete filter.</span> |
| <a id="__codelineno-0-652" name="__codelineno-0-652"></a> |
| <a id="__codelineno-0-653" name="__codelineno-0-653"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-654" name="__codelineno-0-654"></a><span class="sd"> delete_filter: A boolean expression to delete rows from a table</span> |
| <a id="__codelineno-0-655" name="__codelineno-0-655"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-656" name="__codelineno-0-656"></a><span class="sd"> case_sensitive: A bool determine if the provided `delete_filter` is case-sensitive</span> |
| <a id="__codelineno-0-657" name="__codelineno-0-657"></a><span class="sd"> branch: Branch Reference to run the delete operation</span> |
| <a id="__codelineno-0-658" name="__codelineno-0-658"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-659" name="__codelineno-0-659"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span> |
| <a id="__codelineno-0-660" name="__codelineno-0-660"></a> <span class="n">ArrowScan</span><span class="p">,</span> |
| <a id="__codelineno-0-661" name="__codelineno-0-661"></a> <span class="n">_dataframe_to_data_files</span><span class="p">,</span> |
| <a id="__codelineno-0-662" name="__codelineno-0-662"></a> <span class="n">_expression_to_complementary_pyarrow</span><span class="p">,</span> |
| <a id="__codelineno-0-663" name="__codelineno-0-663"></a> <span class="p">)</span> |
| <a id="__codelineno-0-664" name="__codelineno-0-664"></a> |
| <a id="__codelineno-0-665" name="__codelineno-0-665"></a> <span class="k">if</span> <span class="p">(</span> |
| <a id="__codelineno-0-666" name="__codelineno-0-666"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">TableProperties</span><span class="o">.</span><span class="n">DELETE_MODE</span><span class="p">,</span> <span class="n">TableProperties</span><span class="o">.</span><span class="n">DELETE_MODE_DEFAULT</span><span class="p">)</span> |
| <a id="__codelineno-0-667" name="__codelineno-0-667"></a> <span class="o">==</span> <span class="n">TableProperties</span><span class="o">.</span><span class="n">DELETE_MODE_MERGE_ON_READ</span> |
| <a id="__codelineno-0-668" name="__codelineno-0-668"></a> <span class="p">):</span> |
| <a id="__codelineno-0-669" name="__codelineno-0-669"></a> <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Merge on read is not yet supported, falling back to copy-on-write"</span><span class="p">)</span> |
| <a id="__codelineno-0-670" name="__codelineno-0-670"></a> |
| <a id="__codelineno-0-671" name="__codelineno-0-671"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <a id="__codelineno-0-672" name="__codelineno-0-672"></a> <span class="n">delete_filter</span> <span class="o">=</span> <span class="n">_parse_row_filter</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-673" name="__codelineno-0-673"></a> |
| <a id="__codelineno-0-674" name="__codelineno-0-674"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">update_snapshot</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span> <span class="k">as</span> <span class="n">delete_snapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-675" name="__codelineno-0-675"></a> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">delete_by_predicate</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-676" name="__codelineno-0-676"></a> |
| <a id="__codelineno-0-677" name="__codelineno-0-677"></a> <span class="c1"># Check if there are any files that require an actual rewrite of a data file</span> |
| <a id="__codelineno-0-678" name="__codelineno-0-678"></a> <span class="k">if</span> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">rewrites_needed</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span> |
| <a id="__codelineno-0-679" name="__codelineno-0-679"></a> <span class="n">bound_delete_filter</span> <span class="o">=</span> <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> <span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-680" name="__codelineno-0-680"></a> <span class="n">preserve_row_filter</span> <span class="o">=</span> <span class="n">_expression_to_complementary_pyarrow</span><span class="p">(</span><span class="n">bound_delete_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-681" name="__codelineno-0-681"></a> |
| <a id="__codelineno-0-682" name="__codelineno-0-682"></a> <span class="n">file_scan</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan</span><span class="p">(</span><span class="n">row_filter</span><span class="o">=</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-683" name="__codelineno-0-683"></a> <span class="k">if</span> <span class="n">branch</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-684" name="__codelineno-0-684"></a> <span class="n">file_scan</span> <span class="o">=</span> <span class="n">file_scan</span><span class="o">.</span><span class="n">use_ref</span><span class="p">(</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-685" name="__codelineno-0-685"></a> <span class="n">files</span> <span class="o">=</span> <span class="n">file_scan</span><span class="o">.</span><span class="n">plan_files</span><span class="p">()</span> |
| <a id="__codelineno-0-686" name="__codelineno-0-686"></a> |
| <a id="__codelineno-0-687" name="__codelineno-0-687"></a> <span class="n">commit_uuid</span> <span class="o">=</span> <span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">()</span> |
| <a id="__codelineno-0-688" name="__codelineno-0-688"></a> <span class="n">counter</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> |
| <a id="__codelineno-0-689" name="__codelineno-0-689"></a> |
| <a id="__codelineno-0-690" name="__codelineno-0-690"></a> <span class="n">replaced_files</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">DataFile</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">DataFile</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-691" name="__codelineno-0-691"></a> <span class="c1"># This will load the Parquet file into memory, including:</span> |
| <a id="__codelineno-0-692" name="__codelineno-0-692"></a> <span class="c1"># - Filter out the rows based on the delete filter</span> |
| <a id="__codelineno-0-693" name="__codelineno-0-693"></a> <span class="c1"># - Projecting it to the current schema</span> |
| <a id="__codelineno-0-694" name="__codelineno-0-694"></a> <span class="c1"># - Applying the positional deletes if they are there</span> |
| <a id="__codelineno-0-695" name="__codelineno-0-695"></a> <span class="c1"># When writing</span> |
| <a id="__codelineno-0-696" name="__codelineno-0-696"></a> <span class="c1"># - Apply the latest partition-spec</span> |
| <a id="__codelineno-0-697" name="__codelineno-0-697"></a> <span class="c1"># - And sort order when added</span> |
| <a id="__codelineno-0-698" name="__codelineno-0-698"></a> <span class="k">for</span> <span class="n">original_file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span> |
| <a id="__codelineno-0-699" name="__codelineno-0-699"></a> <span class="n">df</span> <span class="o">=</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-700" name="__codelineno-0-700"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-701" name="__codelineno-0-701"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-702" name="__codelineno-0-702"></a> <span class="n">projected_schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-703" name="__codelineno-0-703"></a> <span class="n">row_filter</span><span class="o">=</span><span class="n">AlwaysTrue</span><span class="p">(),</span> |
| <a id="__codelineno-0-704" name="__codelineno-0-704"></a> <span class="p">)</span><span class="o">.</span><span class="n">to_table</span><span class="p">(</span><span class="n">tasks</span><span class="o">=</span><span class="p">[</span><span class="n">original_file</span><span class="p">])</span> |
| <a id="__codelineno-0-705" name="__codelineno-0-705"></a> <span class="n">filtered_df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">preserve_row_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-706" name="__codelineno-0-706"></a> |
| <a id="__codelineno-0-707" name="__codelineno-0-707"></a> <span class="c1"># Only rewrite if there are records being deleted</span> |
| <a id="__codelineno-0-708" name="__codelineno-0-708"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">filtered_df</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-709" name="__codelineno-0-709"></a> <span class="n">replaced_files</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">original_file</span><span class="o">.</span><span class="n">file</span><span class="p">,</span> <span class="p">[]))</span> |
| <a id="__codelineno-0-710" name="__codelineno-0-710"></a> <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">filtered_df</span><span class="p">):</span> |
| <a id="__codelineno-0-711" name="__codelineno-0-711"></a> <span class="n">replaced_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span> |
| <a id="__codelineno-0-712" name="__codelineno-0-712"></a> <span class="p">(</span> |
| <a id="__codelineno-0-713" name="__codelineno-0-713"></a> <span class="n">original_file</span><span class="o">.</span><span class="n">file</span><span class="p">,</span> |
| <a id="__codelineno-0-714" name="__codelineno-0-714"></a> <span class="nb">list</span><span class="p">(</span> |
| <a id="__codelineno-0-715" name="__codelineno-0-715"></a> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-716" name="__codelineno-0-716"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-717" name="__codelineno-0-717"></a> <span class="n">df</span><span class="o">=</span><span class="n">filtered_df</span><span class="p">,</span> |
| <a id="__codelineno-0-718" name="__codelineno-0-718"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-719" name="__codelineno-0-719"></a> <span class="n">write_uuid</span><span class="o">=</span><span class="n">commit_uuid</span><span class="p">,</span> |
| <a id="__codelineno-0-720" name="__codelineno-0-720"></a> <span class="n">counter</span><span class="o">=</span><span class="n">counter</span><span class="p">,</span> |
| <a id="__codelineno-0-721" name="__codelineno-0-721"></a> <span class="p">)</span> |
| <a id="__codelineno-0-722" name="__codelineno-0-722"></a> <span class="p">),</span> |
| <a id="__codelineno-0-723" name="__codelineno-0-723"></a> <span class="p">)</span> |
| <a id="__codelineno-0-724" name="__codelineno-0-724"></a> <span class="p">)</span> |
| <a id="__codelineno-0-725" name="__codelineno-0-725"></a> |
| <a id="__codelineno-0-726" name="__codelineno-0-726"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">replaced_files</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-727" name="__codelineno-0-727"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">update_snapshot</span><span class="p">(</span> |
| <a id="__codelineno-0-728" name="__codelineno-0-728"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span> |
| <a id="__codelineno-0-729" name="__codelineno-0-729"></a> <span class="p">)</span><span class="o">.</span><span class="n">overwrite</span><span class="p">()</span> <span class="k">as</span> <span class="n">overwrite_snapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-730" name="__codelineno-0-730"></a> <span class="n">overwrite_snapshot</span><span class="o">.</span><span class="n">commit_uuid</span> <span class="o">=</span> <span class="n">commit_uuid</span> |
| <a id="__codelineno-0-731" name="__codelineno-0-731"></a> <span class="k">for</span> <span class="n">original_data_file</span><span class="p">,</span> <span class="n">replaced_data_files</span> <span class="ow">in</span> <span class="n">replaced_files</span><span class="p">:</span> |
| <a id="__codelineno-0-732" name="__codelineno-0-732"></a> <span class="n">overwrite_snapshot</span><span class="o">.</span><span class="n">delete_data_file</span><span class="p">(</span><span class="n">original_data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-733" name="__codelineno-0-733"></a> <span class="k">for</span> <span class="n">replaced_data_file</span> <span class="ow">in</span> <span class="n">replaced_data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-734" name="__codelineno-0-734"></a> <span class="n">overwrite_snapshot</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">replaced_data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-735" name="__codelineno-0-735"></a> |
| <a id="__codelineno-0-736" name="__codelineno-0-736"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">files_affected</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">rewrites_needed</span><span class="p">:</span> |
| <a id="__codelineno-0-737" name="__codelineno-0-737"></a> <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Delete operation did not match any records"</span><span class="p">)</span> |
| <a id="__codelineno-0-738" name="__codelineno-0-738"></a> |
| <a id="__codelineno-0-739" name="__codelineno-0-739"></a> <span class="k">def</span><span class="w"> </span><span class="nf">upsert</span><span class="p">(</span> |
| <a id="__codelineno-0-740" name="__codelineno-0-740"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-741" name="__codelineno-0-741"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-742" name="__codelineno-0-742"></a> <span class="n">join_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-743" name="__codelineno-0-743"></a> <span class="n">when_matched_update_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-744" name="__codelineno-0-744"></a> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-745" name="__codelineno-0-745"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-746" name="__codelineno-0-746"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-747" name="__codelineno-0-747"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">UpsertResult</span><span class="p">:</span> |
| <a id="__codelineno-0-748" name="__codelineno-0-748"></a><span class="w"> </span><span class="sd">"""Shorthand API for performing an upsert to an iceberg table.</span> |
| <a id="__codelineno-0-749" name="__codelineno-0-749"></a> |
| <a id="__codelineno-0-750" name="__codelineno-0-750"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-751" name="__codelineno-0-751"></a> |
| <a id="__codelineno-0-752" name="__codelineno-0-752"></a><span class="sd"> df: The input dataframe to upsert with the table's data.</span> |
| <a id="__codelineno-0-753" name="__codelineno-0-753"></a><span class="sd"> join_cols: Columns to join on, if not provided, it will use the identifier-field-ids.</span> |
| <a id="__codelineno-0-754" name="__codelineno-0-754"></a><span class="sd"> when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing</span> |
| <a id="__codelineno-0-755" name="__codelineno-0-755"></a><span class="sd"> when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any existing rows in the table</span> |
| <a id="__codelineno-0-756" name="__codelineno-0-756"></a><span class="sd"> case_sensitive: Bool indicating if the match should be case-sensitive</span> |
| <a id="__codelineno-0-757" name="__codelineno-0-757"></a><span class="sd"> branch: Branch Reference to run the upsert operation</span> |
| <a id="__codelineno-0-758" name="__codelineno-0-758"></a> |
| <a id="__codelineno-0-759" name="__codelineno-0-759"></a><span class="sd"> To learn more about the identifier-field-ids: https://iceberg.apache.org/spec/#identifier-field-ids</span> |
| <a id="__codelineno-0-760" name="__codelineno-0-760"></a> |
| <a id="__codelineno-0-761" name="__codelineno-0-761"></a><span class="sd"> Example Use Cases:</span> |
| <a id="__codelineno-0-762" name="__codelineno-0-762"></a><span class="sd"> Case 1: Both Parameters = True (Full Upsert)</span> |
| <a id="__codelineno-0-763" name="__codelineno-0-763"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-764" name="__codelineno-0-764"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-765" name="__codelineno-0-765"></a> |
| <a id="__codelineno-0-766" name="__codelineno-0-766"></a><span class="sd"> Case 2: when_matched_update_all = False, when_not_matched_insert_all = True</span> |
| <a id="__codelineno-0-767" name="__codelineno-0-767"></a><span class="sd"> Existing row found → Do nothing (no updates)</span> |
| <a id="__codelineno-0-768" name="__codelineno-0-768"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-769" name="__codelineno-0-769"></a> |
| <a id="__codelineno-0-770" name="__codelineno-0-770"></a><span class="sd"> Case 3: when_matched_update_all = True, when_not_matched_insert_all = False</span> |
| <a id="__codelineno-0-771" name="__codelineno-0-771"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-772" name="__codelineno-0-772"></a><span class="sd"> New row found → Do nothing (no inserts)</span> |
| <a id="__codelineno-0-773" name="__codelineno-0-773"></a> |
| <a id="__codelineno-0-774" name="__codelineno-0-774"></a><span class="sd"> Case 4: Both Parameters = False (No Merge Effect)</span> |
| <a id="__codelineno-0-775" name="__codelineno-0-775"></a><span class="sd"> Existing row found → Do nothing</span> |
| <a id="__codelineno-0-776" name="__codelineno-0-776"></a><span class="sd"> New row found → Do nothing</span> |
| <a id="__codelineno-0-777" name="__codelineno-0-777"></a><span class="sd"> (Function effectively does nothing)</span> |
| <a id="__codelineno-0-778" name="__codelineno-0-778"></a> |
| <a id="__codelineno-0-779" name="__codelineno-0-779"></a> |
| <a id="__codelineno-0-780" name="__codelineno-0-780"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-781" name="__codelineno-0-781"></a><span class="sd"> An UpsertResult class (contains details of rows updated and inserted)</span> |
| <a id="__codelineno-0-782" name="__codelineno-0-782"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-783" name="__codelineno-0-783"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-784" name="__codelineno-0-784"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> <span class="c1"># noqa: F401</span> |
| <a id="__codelineno-0-785" name="__codelineno-0-785"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-786" name="__codelineno-0-786"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-787" name="__codelineno-0-787"></a> |
| <a id="__codelineno-0-788" name="__codelineno-0-788"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">expression_to_pyarrow</span> |
| <a id="__codelineno-0-789" name="__codelineno-0-789"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.table</span><span class="w"> </span><span class="kn">import</span> <span class="n">upsert_util</span> |
| <a id="__codelineno-0-790" name="__codelineno-0-790"></a> |
| <a id="__codelineno-0-791" name="__codelineno-0-791"></a> <span class="k">if</span> <span class="n">join_cols</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-792" name="__codelineno-0-792"></a> <span class="n">join_cols</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-793" name="__codelineno-0-793"></a> <span class="k">for</span> <span class="n">field_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">identifier_field_ids</span><span class="p">:</span> |
| <a id="__codelineno-0-794" name="__codelineno-0-794"></a> <span class="n">col</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">find_column_name</span><span class="p">(</span><span class="n">field_id</span><span class="p">)</span> |
| <a id="__codelineno-0-795" name="__codelineno-0-795"></a> <span class="k">if</span> <span class="n">col</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-796" name="__codelineno-0-796"></a> <span class="n">join_cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">)</span> |
| <a id="__codelineno-0-797" name="__codelineno-0-797"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-798" name="__codelineno-0-798"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Field-ID could not be found: </span><span class="si">{</span><span class="n">join_cols</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-799" name="__codelineno-0-799"></a> |
| <a id="__codelineno-0-800" name="__codelineno-0-800"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">join_cols</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-801" name="__codelineno-0-801"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Join columns could not be found, please set identifier-field-ids or pass in explicitly."</span><span class="p">)</span> |
| <a id="__codelineno-0-802" name="__codelineno-0-802"></a> |
| <a id="__codelineno-0-803" name="__codelineno-0-803"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">when_matched_update_all</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> |
| <a id="__codelineno-0-804" name="__codelineno-0-804"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"no upsert options selected...exiting"</span><span class="p">)</span> |
| <a id="__codelineno-0-805" name="__codelineno-0-805"></a> |
| <a id="__codelineno-0-806" name="__codelineno-0-806"></a> <span class="k">if</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">has_duplicate_rows</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">):</span> |
| <a id="__codelineno-0-807" name="__codelineno-0-807"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Duplicate rows found in source dataset based on the key columns. No upsert executed"</span><span class="p">)</span> |
| <a id="__codelineno-0-808" name="__codelineno-0-808"></a> |
| <a id="__codelineno-0-809" name="__codelineno-0-809"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span> |
| <a id="__codelineno-0-810" name="__codelineno-0-810"></a> |
| <a id="__codelineno-0-811" name="__codelineno-0-811"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-812" name="__codelineno-0-812"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-813" name="__codelineno-0-813"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-814" name="__codelineno-0-814"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-815" name="__codelineno-0-815"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-816" name="__codelineno-0-816"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-817" name="__codelineno-0-817"></a> <span class="p">)</span> |
| <a id="__codelineno-0-818" name="__codelineno-0-818"></a> |
| <a id="__codelineno-0-819" name="__codelineno-0-819"></a> <span class="c1"># get list of rows that exist so we don't have to load the entire target table</span> |
| <a id="__codelineno-0-820" name="__codelineno-0-820"></a> <span class="n">matched_predicate</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">create_match_filter</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-821" name="__codelineno-0-821"></a> |
| <a id="__codelineno-0-822" name="__codelineno-0-822"></a> <span class="c1"># We must use Transaction.table_metadata for the scan. This includes all uncommitted - but relevant - changes.</span> |
| <a id="__codelineno-0-823" name="__codelineno-0-823"></a> |
| <a id="__codelineno-0-824" name="__codelineno-0-824"></a> <span class="n">matched_iceberg_record_batches_scan</span> <span class="o">=</span> <span class="n">DataScan</span><span class="p">(</span> |
| <a id="__codelineno-0-825" name="__codelineno-0-825"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-826" name="__codelineno-0-826"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-827" name="__codelineno-0-827"></a> <span class="n">row_filter</span><span class="o">=</span><span class="n">matched_predicate</span><span class="p">,</span> |
| <a id="__codelineno-0-828" name="__codelineno-0-828"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-829" name="__codelineno-0-829"></a> <span class="p">)</span> |
| <a id="__codelineno-0-830" name="__codelineno-0-830"></a> |
| <a id="__codelineno-0-831" name="__codelineno-0-831"></a> <span class="k">if</span> <span class="n">branch</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">refs</span><span class="p">:</span> |
| <a id="__codelineno-0-832" name="__codelineno-0-832"></a> <span class="n">matched_iceberg_record_batches_scan</span> <span class="o">=</span> <span class="n">matched_iceberg_record_batches_scan</span><span class="o">.</span><span class="n">use_ref</span><span class="p">(</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-833" name="__codelineno-0-833"></a> |
| <a id="__codelineno-0-834" name="__codelineno-0-834"></a> <span class="n">matched_iceberg_record_batches</span> <span class="o">=</span> <span class="n">matched_iceberg_record_batches_scan</span><span class="o">.</span><span class="n">to_arrow_batch_reader</span><span class="p">()</span> |
| <a id="__codelineno-0-835" name="__codelineno-0-835"></a> |
| <a id="__codelineno-0-836" name="__codelineno-0-836"></a> <span class="n">batches_to_overwrite</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-837" name="__codelineno-0-837"></a> <span class="n">overwrite_predicates</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-838" name="__codelineno-0-838"></a> <span class="n">rows_to_insert</span> <span class="o">=</span> <span class="n">df</span> |
| <a id="__codelineno-0-839" name="__codelineno-0-839"></a> |
| <a id="__codelineno-0-840" name="__codelineno-0-840"></a> <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">matched_iceberg_record_batches</span><span class="p">:</span> |
| <a id="__codelineno-0-841" name="__codelineno-0-841"></a> <span class="n">rows</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="o">.</span><span class="n">from_batches</span><span class="p">([</span><span class="n">batch</span><span class="p">])</span> |
| <a id="__codelineno-0-842" name="__codelineno-0-842"></a> |
| <a id="__codelineno-0-843" name="__codelineno-0-843"></a> <span class="k">if</span> <span class="n">when_matched_update_all</span><span class="p">:</span> |
| <a id="__codelineno-0-844" name="__codelineno-0-844"></a> <span class="c1"># function get_rows_to_update is doing a check on non-key columns to see if any of the values have actually changed</span> |
| <a id="__codelineno-0-845" name="__codelineno-0-845"></a> <span class="c1"># we don't want to do just a blanket overwrite for matched rows if the actual non-key column data hasn't changed</span> |
| <a id="__codelineno-0-846" name="__codelineno-0-846"></a> <span class="c1"># this extra step avoids unnecessary IO and writes</span> |
| <a id="__codelineno-0-847" name="__codelineno-0-847"></a> <span class="n">rows_to_update</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">get_rows_to_update</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-848" name="__codelineno-0-848"></a> |
| <a id="__codelineno-0-849" name="__codelineno-0-849"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-850" name="__codelineno-0-850"></a> <span class="c1"># build the match predicate filter</span> |
| <a id="__codelineno-0-851" name="__codelineno-0-851"></a> <span class="n">overwrite_mask_predicate</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">create_match_filter</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-852" name="__codelineno-0-852"></a> |
| <a id="__codelineno-0-853" name="__codelineno-0-853"></a> <span class="n">batches_to_overwrite</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">)</span> |
| <a id="__codelineno-0-854" name="__codelineno-0-854"></a> <span class="n">overwrite_predicates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">overwrite_mask_predicate</span><span class="p">)</span> |
| <a id="__codelineno-0-855" name="__codelineno-0-855"></a> |
| <a id="__codelineno-0-856" name="__codelineno-0-856"></a> <span class="k">if</span> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> |
| <a id="__codelineno-0-857" name="__codelineno-0-857"></a> <span class="n">expr_match</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">create_match_filter</span><span class="p">(</span><span class="n">rows</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-858" name="__codelineno-0-858"></a> <span class="n">expr_match_bound</span> <span class="o">=</span> <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> <span class="n">expr_match</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-859" name="__codelineno-0-859"></a> <span class="n">expr_match_arrow</span> <span class="o">=</span> <span class="n">expression_to_pyarrow</span><span class="p">(</span><span class="n">expr_match_bound</span><span class="p">)</span> |
| <a id="__codelineno-0-860" name="__codelineno-0-860"></a> |
| <a id="__codelineno-0-861" name="__codelineno-0-861"></a> <span class="c1"># Filter rows per batch.</span> |
| <a id="__codelineno-0-862" name="__codelineno-0-862"></a> <span class="n">rows_to_insert</span> <span class="o">=</span> <span class="n">rows_to_insert</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">expr_match_arrow</span><span class="p">)</span> |
| <a id="__codelineno-0-863" name="__codelineno-0-863"></a> |
| <a id="__codelineno-0-864" name="__codelineno-0-864"></a> <span class="n">update_row_cnt</span> <span class="o">=</span> <span class="mi">0</span> |
| <a id="__codelineno-0-865" name="__codelineno-0-865"></a> <span class="n">insert_row_cnt</span> <span class="o">=</span> <span class="mi">0</span> |
| <a id="__codelineno-0-866" name="__codelineno-0-866"></a> |
| <a id="__codelineno-0-867" name="__codelineno-0-867"></a> <span class="k">if</span> <span class="n">batches_to_overwrite</span><span class="p">:</span> |
| <a id="__codelineno-0-868" name="__codelineno-0-868"></a> <span class="n">rows_to_update</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">concat_tables</span><span class="p">(</span><span class="n">batches_to_overwrite</span><span class="p">)</span> |
| <a id="__codelineno-0-869" name="__codelineno-0-869"></a> <span class="n">update_row_cnt</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">)</span> |
| <a id="__codelineno-0-870" name="__codelineno-0-870"></a> <span class="bp">self</span><span class="o">.</span><span class="n">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-871" name="__codelineno-0-871"></a> <span class="n">rows_to_update</span><span class="p">,</span> |
| <a id="__codelineno-0-872" name="__codelineno-0-872"></a> <span class="n">overwrite_filter</span><span class="o">=</span><span class="n">Or</span><span class="p">(</span><span class="o">*</span><span class="n">overwrite_predicates</span><span class="p">)</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">overwrite_predicates</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span> <span class="k">else</span> <span class="n">overwrite_predicates</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> |
| <a id="__codelineno-0-873" name="__codelineno-0-873"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-874" name="__codelineno-0-874"></a> <span class="p">)</span> |
| <a id="__codelineno-0-875" name="__codelineno-0-875"></a> |
| <a id="__codelineno-0-876" name="__codelineno-0-876"></a> <span class="k">if</span> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> |
| <a id="__codelineno-0-877" name="__codelineno-0-877"></a> <span class="n">insert_row_cnt</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">rows_to_insert</span><span class="p">)</span> |
| <a id="__codelineno-0-878" name="__codelineno-0-878"></a> <span class="k">if</span> <span class="n">rows_to_insert</span><span class="p">:</span> |
| <a id="__codelineno-0-879" name="__codelineno-0-879"></a> <span class="bp">self</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">rows_to_insert</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-880" name="__codelineno-0-880"></a> |
| <a id="__codelineno-0-881" name="__codelineno-0-881"></a> <span class="k">return</span> <span class="n">UpsertResult</span><span class="p">(</span><span class="n">rows_updated</span><span class="o">=</span><span class="n">update_row_cnt</span><span class="p">,</span> <span class="n">rows_inserted</span><span class="o">=</span><span class="n">insert_row_cnt</span><span class="p">)</span> |
| <a id="__codelineno-0-882" name="__codelineno-0-882"></a> |
| <a id="__codelineno-0-883" name="__codelineno-0-883"></a> <span class="k">def</span><span class="w"> </span><span class="nf">add_files</span><span class="p">(</span> |
| <a id="__codelineno-0-884" name="__codelineno-0-884"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-885" name="__codelineno-0-885"></a> <span class="n">file_paths</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <a id="__codelineno-0-886" name="__codelineno-0-886"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-887" name="__codelineno-0-887"></a> <span class="n">check_duplicate_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-888" name="__codelineno-0-888"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-889" name="__codelineno-0-889"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-890" name="__codelineno-0-890"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-891" name="__codelineno-0-891"></a><span class="sd"> Shorthand API for adding files as data files to the table transaction.</span> |
| <a id="__codelineno-0-892" name="__codelineno-0-892"></a> |
| <a id="__codelineno-0-893" name="__codelineno-0-893"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-894" name="__codelineno-0-894"></a><span class="sd"> file_paths: The list of full file paths to be added as data files to the table</span> |
| <a id="__codelineno-0-895" name="__codelineno-0-895"></a> |
| <a id="__codelineno-0-896" name="__codelineno-0-896"></a><span class="sd"> Raises:</span> |
| <a id="__codelineno-0-897" name="__codelineno-0-897"></a><span class="sd"> FileNotFoundError: If the file does not exist.</span> |
| <a id="__codelineno-0-898" name="__codelineno-0-898"></a><span class="sd"> ValueError: Raises a ValueError given file_paths contains duplicate files</span> |
| <a id="__codelineno-0-899" name="__codelineno-0-899"></a><span class="sd"> ValueError: Raises a ValueError given file_paths already referenced by table</span> |
| <a id="__codelineno-0-900" name="__codelineno-0-900"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-901" name="__codelineno-0-901"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">file_paths</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">file_paths</span><span class="p">)):</span> |
| <a id="__codelineno-0-902" name="__codelineno-0-902"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"File paths must be unique"</span><span class="p">)</span> |
| <a id="__codelineno-0-903" name="__codelineno-0-903"></a> |
| <a id="__codelineno-0-904" name="__codelineno-0-904"></a> <span class="k">if</span> <span class="n">check_duplicate_files</span><span class="p">:</span> |
| <a id="__codelineno-0-905" name="__codelineno-0-905"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.compute</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pc</span> |
| <a id="__codelineno-0-906" name="__codelineno-0-906"></a> |
| <a id="__codelineno-0-907" name="__codelineno-0-907"></a> <span class="n">expr</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"file_path"</span><span class="p">)</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">file_paths</span><span class="p">)</span> |
| <a id="__codelineno-0-908" name="__codelineno-0-908"></a> <span class="n">referenced_files</span> <span class="o">=</span> <span class="p">[</span><span class="n">file</span><span class="p">[</span><span class="s2">"file_path"</span><span class="p">]</span> <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">inspect</span><span class="o">.</span><span class="n">data_files</span><span class="p">()</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">to_pylist</span><span class="p">()]</span> |
| <a id="__codelineno-0-909" name="__codelineno-0-909"></a> |
| <a id="__codelineno-0-910" name="__codelineno-0-910"></a> <span class="k">if</span> <span class="n">referenced_files</span><span class="p">:</span> |
| <a id="__codelineno-0-911" name="__codelineno-0-911"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot add files that are already referenced by table, files: </span><span class="si">{</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">referenced_files</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-912" name="__codelineno-0-912"></a> |
| <a id="__codelineno-0-913" name="__codelineno-0-913"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-914" name="__codelineno-0-914"></a> <span class="bp">self</span><span class="o">.</span><span class="n">set_properties</span><span class="p">(</span> |
| <a id="__codelineno-0-915" name="__codelineno-0-915"></a> <span class="o">**</span><span class="p">{</span><span class="n">TableProperties</span><span class="o">.</span><span class="n">DEFAULT_NAME_MAPPING</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">name_mapping</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">()}</span> |
| <a id="__codelineno-0-916" name="__codelineno-0-916"></a> <span class="p">)</span> |
| <a id="__codelineno-0-917" name="__codelineno-0-917"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-918" name="__codelineno-0-918"></a> <span class="n">data_files</span> <span class="o">=</span> <span class="n">_parquet_files_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-919" name="__codelineno-0-919"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">file_paths</span><span class="o">=</span><span class="n">file_paths</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-920" name="__codelineno-0-920"></a> <span class="p">)</span> |
| <a id="__codelineno-0-921" name="__codelineno-0-921"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-922" name="__codelineno-0-922"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-923" name="__codelineno-0-923"></a> |
| <a id="__codelineno-0-924" name="__codelineno-0-924"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSpec</span><span class="p">:</span> |
| <a id="__codelineno-0-925" name="__codelineno-0-925"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSpec to update the partitioning of the table.</span> |
| <a id="__codelineno-0-926" name="__codelineno-0-926"></a> |
| <a id="__codelineno-0-927" name="__codelineno-0-927"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-928" name="__codelineno-0-928"></a><span class="sd"> A new UpdateSpec.</span> |
| <a id="__codelineno-0-929" name="__codelineno-0-929"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-930" name="__codelineno-0-930"></a> <span class="k">return</span> <span class="n">UpdateSpec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <a id="__codelineno-0-931" name="__codelineno-0-931"></a> |
| <a id="__codelineno-0-932" name="__codelineno-0-932"></a> <span class="k">def</span><span class="w"> </span><span class="nf">remove_properties</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">removals</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-933" name="__codelineno-0-933"></a><span class="w"> </span><span class="sd">"""Remove properties.</span> |
| <a id="__codelineno-0-934" name="__codelineno-0-934"></a> |
| <a id="__codelineno-0-935" name="__codelineno-0-935"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-936" name="__codelineno-0-936"></a><span class="sd"> removals: Properties to be removed.</span> |
| <a id="__codelineno-0-937" name="__codelineno-0-937"></a> |
| <a id="__codelineno-0-938" name="__codelineno-0-938"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-939" name="__codelineno-0-939"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-940" name="__codelineno-0-940"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-941" name="__codelineno-0-941"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">((</span><span class="n">RemovePropertiesUpdate</span><span class="p">(</span><span class="n">removals</span><span class="o">=</span><span class="n">removals</span><span class="p">),))</span> |
| <a id="__codelineno-0-942" name="__codelineno-0-942"></a> |
| <a id="__codelineno-0-943" name="__codelineno-0-943"></a> <span class="k">def</span><span class="w"> </span><span class="nf">update_location</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">location</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-944" name="__codelineno-0-944"></a><span class="w"> </span><span class="sd">"""Set the new table location.</span> |
| <a id="__codelineno-0-945" name="__codelineno-0-945"></a> |
| <a id="__codelineno-0-946" name="__codelineno-0-946"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-947" name="__codelineno-0-947"></a><span class="sd"> location: The new location of the table.</span> |
| <a id="__codelineno-0-948" name="__codelineno-0-948"></a> |
| <a id="__codelineno-0-949" name="__codelineno-0-949"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-950" name="__codelineno-0-950"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-951" name="__codelineno-0-951"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-952" name="__codelineno-0-952"></a> <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"Not yet implemented"</span><span class="p">)</span> |
| <a id="__codelineno-0-953" name="__codelineno-0-953"></a> |
| <a id="__codelineno-0-954" name="__codelineno-0-954"></a> <span class="k">def</span><span class="w"> </span><span class="nf">commit_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-955" name="__codelineno-0-955"></a><span class="w"> </span><span class="sd">"""Commit the changes to the catalog.</span> |
| <a id="__codelineno-0-956" name="__codelineno-0-956"></a> |
| <a id="__codelineno-0-957" name="__codelineno-0-957"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-958" name="__codelineno-0-958"></a><span class="sd"> The table with the updates applied.</span> |
| <a id="__codelineno-0-959" name="__codelineno-0-959"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-960" name="__codelineno-0-960"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-961" name="__codelineno-0-961"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">+=</span> <span class="p">(</span><span class="n">AssertTableUUID</span><span class="p">(</span><span class="n">uuid</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">table_uuid</span><span class="p">),)</span> |
| <a id="__codelineno-0-962" name="__codelineno-0-962"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">_do_commit</span><span class="p">(</span> <span class="c1"># pylint: disable=W0212</span> |
| <a id="__codelineno-0-963" name="__codelineno-0-963"></a> <span class="n">updates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">,</span> |
| <a id="__codelineno-0-964" name="__codelineno-0-964"></a> <span class="n">requirements</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span><span class="p">,</span> |
| <a id="__codelineno-0-965" name="__codelineno-0-965"></a> <span class="p">)</span> |
| <a id="__codelineno-0-966" name="__codelineno-0-966"></a> |
| <a id="__codelineno-0-967" name="__codelineno-0-967"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-968" name="__codelineno-0-968"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-969" name="__codelineno-0-969"></a> |
| <a id="__codelineno-0-970" name="__codelineno-0-970"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.__enter__" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="fm">__enter__</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.__enter__" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Start a transaction to update the table.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-277">277</a></span> |
| <span class="normal"><a href="#__codelineno-0-278">278</a></span> |
| <span class="normal"><a href="#__codelineno-0-279">279</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="k">def</span><span class="w"> </span><span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="w"> </span><span class="sd">"""Start a transaction to update the table."""</span> |
| <a id="__codelineno-0-279" name="__codelineno-0-279"></a> <span class="k">return</span> <span class="bp">self</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.__exit__" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="fm">__exit__</span><span class="p">(</span><span class="n">exctype</span><span class="p">,</span> <span class="n">excinst</span><span class="p">,</span> <span class="n">exctb</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.__exit__" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Close and commit the transaction if no exceptions have been raised.</p> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-281">281</a></span> |
| <span class="normal"><a href="#__codelineno-0-282">282</a></span> |
| <span class="normal"><a href="#__codelineno-0-283">283</a></span> |
| <span class="normal"><a href="#__codelineno-0-284">284</a></span> |
| <span class="normal"><a href="#__codelineno-0-285">285</a></span> |
| <span class="normal"><a href="#__codelineno-0-286">286</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-281" name="__codelineno-0-281"></a><span class="k">def</span><span class="w"> </span><span class="fm">__exit__</span><span class="p">(</span> |
| <a id="__codelineno-0-282" name="__codelineno-0-282"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">exctype</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">]],</span> <span class="n">excinst</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">],</span> <span class="n">exctb</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">TracebackType</span><span class="p">]</span> |
| <a id="__codelineno-0-283" name="__codelineno-0-283"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-284" name="__codelineno-0-284"></a><span class="w"> </span><span class="sd">"""Close and commit the transaction if no exceptions have been raised."""</span> |
| <a id="__codelineno-0-285" name="__codelineno-0-285"></a> <span class="k">if</span> <span class="n">exctype</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">excinst</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">exctb</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-286" name="__codelineno-0-286"></a> <span class="bp">self</span><span class="o">.</span><span class="n">commit_transaction</span><span class="p">()</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.__init__" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="fm">__init__</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.__init__" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Open a transaction to stage and commit changes to a table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>table</code> |
| </td> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Table (pyiceberg.table.Table)" href="#pyiceberg.table.Table">Table</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The table that will be altered.</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>autocommit</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Option to automatically commit the changes when they are staged.</p> |
| </div> |
| </td> |
| <td> |
| <code>False</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-261">261</a></span> |
| <span class="normal"><a href="#__codelineno-0-262">262</a></span> |
| <span class="normal"><a href="#__codelineno-0-263">263</a></span> |
| <span class="normal"><a href="#__codelineno-0-264">264</a></span> |
| <span class="normal"><a href="#__codelineno-0-265">265</a></span> |
| <span class="normal"><a href="#__codelineno-0-266">266</a></span> |
| <span class="normal"><a href="#__codelineno-0-267">267</a></span> |
| <span class="normal"><a href="#__codelineno-0-268">268</a></span> |
| <span class="normal"><a href="#__codelineno-0-269">269</a></span> |
| <span class="normal"><a href="#__codelineno-0-270">270</a></span> |
| <span class="normal"><a href="#__codelineno-0-271">271</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-261" name="__codelineno-0-261"></a><span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">:</span> <span class="n">Table</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span> |
| <a id="__codelineno-0-262" name="__codelineno-0-262"></a><span class="w"> </span><span class="sd">"""Open a transaction to stage and commit changes to a table.</span> |
| <a id="__codelineno-0-263" name="__codelineno-0-263"></a> |
| <a id="__codelineno-0-264" name="__codelineno-0-264"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-265" name="__codelineno-0-265"></a><span class="sd"> table: The table that will be altered.</span> |
| <a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="sd"> autocommit: Option to automatically commit the changes when they are staged.</span> |
| <a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-268" name="__codelineno-0-268"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span> <span class="o">=</span> <span class="n">table</span> |
| <a id="__codelineno-0-269" name="__codelineno-0-269"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_autocommit</span> <span class="o">=</span> <span class="n">autocommit</span> |
| <a id="__codelineno-0-270" name="__codelineno-0-270"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-271" name="__codelineno-0-271"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="p">()</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.add_files" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">add_files</span><span class="p">(</span><span class="n">file_paths</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">check_duplicate_files</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.add_files" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand API for adding files as data files to the table transaction.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>file_paths</code> |
| </td> |
| <td> |
| <code><span title="typing.List">List</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The list of full file paths to be added as data files to the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Raises:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="FileNotFoundError">FileNotFoundError</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If the file does not exist.</p> |
| </div> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="ValueError">ValueError</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Raises a ValueError given file_paths contains duplicate files</p> |
| </div> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="ValueError">ValueError</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Raises a ValueError given file_paths already referenced by table</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-883">883</a></span> |
| <span class="normal"><a href="#__codelineno-0-884">884</a></span> |
| <span class="normal"><a href="#__codelineno-0-885">885</a></span> |
| <span class="normal"><a href="#__codelineno-0-886">886</a></span> |
| <span class="normal"><a href="#__codelineno-0-887">887</a></span> |
| <span class="normal"><a href="#__codelineno-0-888">888</a></span> |
| <span class="normal"><a href="#__codelineno-0-889">889</a></span> |
| <span class="normal"><a href="#__codelineno-0-890">890</a></span> |
| <span class="normal"><a href="#__codelineno-0-891">891</a></span> |
| <span class="normal"><a href="#__codelineno-0-892">892</a></span> |
| <span class="normal"><a href="#__codelineno-0-893">893</a></span> |
| <span class="normal"><a href="#__codelineno-0-894">894</a></span> |
| <span class="normal"><a href="#__codelineno-0-895">895</a></span> |
| <span class="normal"><a href="#__codelineno-0-896">896</a></span> |
| <span class="normal"><a href="#__codelineno-0-897">897</a></span> |
| <span class="normal"><a href="#__codelineno-0-898">898</a></span> |
| <span class="normal"><a href="#__codelineno-0-899">899</a></span> |
| <span class="normal"><a href="#__codelineno-0-900">900</a></span> |
| <span class="normal"><a href="#__codelineno-0-901">901</a></span> |
| <span class="normal"><a href="#__codelineno-0-902">902</a></span> |
| <span class="normal"><a href="#__codelineno-0-903">903</a></span> |
| <span class="normal"><a href="#__codelineno-0-904">904</a></span> |
| <span class="normal"><a href="#__codelineno-0-905">905</a></span> |
| <span class="normal"><a href="#__codelineno-0-906">906</a></span> |
| <span class="normal"><a href="#__codelineno-0-907">907</a></span> |
| <span class="normal"><a href="#__codelineno-0-908">908</a></span> |
| <span class="normal"><a href="#__codelineno-0-909">909</a></span> |
| <span class="normal"><a href="#__codelineno-0-910">910</a></span> |
| <span class="normal"><a href="#__codelineno-0-911">911</a></span> |
| <span class="normal"><a href="#__codelineno-0-912">912</a></span> |
| <span class="normal"><a href="#__codelineno-0-913">913</a></span> |
| <span class="normal"><a href="#__codelineno-0-914">914</a></span> |
| <span class="normal"><a href="#__codelineno-0-915">915</a></span> |
| <span class="normal"><a href="#__codelineno-0-916">916</a></span> |
| <span class="normal"><a href="#__codelineno-0-917">917</a></span> |
| <span class="normal"><a href="#__codelineno-0-918">918</a></span> |
| <span class="normal"><a href="#__codelineno-0-919">919</a></span> |
| <span class="normal"><a href="#__codelineno-0-920">920</a></span> |
| <span class="normal"><a href="#__codelineno-0-921">921</a></span> |
| <span class="normal"><a href="#__codelineno-0-922">922</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-883" name="__codelineno-0-883"></a><span class="k">def</span><span class="w"> </span><span class="nf">add_files</span><span class="p">(</span> |
| <a id="__codelineno-0-884" name="__codelineno-0-884"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-885" name="__codelineno-0-885"></a> <span class="n">file_paths</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <a id="__codelineno-0-886" name="__codelineno-0-886"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-887" name="__codelineno-0-887"></a> <span class="n">check_duplicate_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-888" name="__codelineno-0-888"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-889" name="__codelineno-0-889"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-890" name="__codelineno-0-890"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-891" name="__codelineno-0-891"></a><span class="sd"> Shorthand API for adding files as data files to the table transaction.</span> |
| <a id="__codelineno-0-892" name="__codelineno-0-892"></a> |
| <a id="__codelineno-0-893" name="__codelineno-0-893"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-894" name="__codelineno-0-894"></a><span class="sd"> file_paths: The list of full file paths to be added as data files to the table</span> |
| <a id="__codelineno-0-895" name="__codelineno-0-895"></a> |
| <a id="__codelineno-0-896" name="__codelineno-0-896"></a><span class="sd"> Raises:</span> |
| <a id="__codelineno-0-897" name="__codelineno-0-897"></a><span class="sd"> FileNotFoundError: If the file does not exist.</span> |
| <a id="__codelineno-0-898" name="__codelineno-0-898"></a><span class="sd"> ValueError: Raises a ValueError given file_paths contains duplicate files</span> |
| <a id="__codelineno-0-899" name="__codelineno-0-899"></a><span class="sd"> ValueError: Raises a ValueError given file_paths already referenced by table</span> |
| <a id="__codelineno-0-900" name="__codelineno-0-900"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-901" name="__codelineno-0-901"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">file_paths</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">file_paths</span><span class="p">)):</span> |
| <a id="__codelineno-0-902" name="__codelineno-0-902"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"File paths must be unique"</span><span class="p">)</span> |
| <a id="__codelineno-0-903" name="__codelineno-0-903"></a> |
| <a id="__codelineno-0-904" name="__codelineno-0-904"></a> <span class="k">if</span> <span class="n">check_duplicate_files</span><span class="p">:</span> |
| <a id="__codelineno-0-905" name="__codelineno-0-905"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.compute</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pc</span> |
| <a id="__codelineno-0-906" name="__codelineno-0-906"></a> |
| <a id="__codelineno-0-907" name="__codelineno-0-907"></a> <span class="n">expr</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"file_path"</span><span class="p">)</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">file_paths</span><span class="p">)</span> |
| <a id="__codelineno-0-908" name="__codelineno-0-908"></a> <span class="n">referenced_files</span> <span class="o">=</span> <span class="p">[</span><span class="n">file</span><span class="p">[</span><span class="s2">"file_path"</span><span class="p">]</span> <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">inspect</span><span class="o">.</span><span class="n">data_files</span><span class="p">()</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">to_pylist</span><span class="p">()]</span> |
| <a id="__codelineno-0-909" name="__codelineno-0-909"></a> |
| <a id="__codelineno-0-910" name="__codelineno-0-910"></a> <span class="k">if</span> <span class="n">referenced_files</span><span class="p">:</span> |
| <a id="__codelineno-0-911" name="__codelineno-0-911"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot add files that are already referenced by table, files: </span><span class="si">{</span><span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">referenced_files</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-912" name="__codelineno-0-912"></a> |
| <a id="__codelineno-0-913" name="__codelineno-0-913"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">()</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-914" name="__codelineno-0-914"></a> <span class="bp">self</span><span class="o">.</span><span class="n">set_properties</span><span class="p">(</span> |
| <a id="__codelineno-0-915" name="__codelineno-0-915"></a> <span class="o">**</span><span class="p">{</span><span class="n">TableProperties</span><span class="o">.</span><span class="n">DEFAULT_NAME_MAPPING</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">name_mapping</span><span class="o">.</span><span class="n">model_dump_json</span><span class="p">()}</span> |
| <a id="__codelineno-0-916" name="__codelineno-0-916"></a> <span class="p">)</span> |
| <a id="__codelineno-0-917" name="__codelineno-0-917"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-918" name="__codelineno-0-918"></a> <span class="n">data_files</span> <span class="o">=</span> <span class="n">_parquet_files_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-919" name="__codelineno-0-919"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">file_paths</span><span class="o">=</span><span class="n">file_paths</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-920" name="__codelineno-0-920"></a> <span class="p">)</span> |
| <a id="__codelineno-0-921" name="__codelineno-0-921"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-922" name="__codelineno-0-922"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.append" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">append</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.append" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand API for appending a PyArrow table to a table transaction.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>df</code> |
| </td> |
| <td> |
| <code><span title="pyarrow.Table">Table</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The Arrow dataframe that will be appended to overwrite the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the append operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-477">477</a></span> |
| <span class="normal"><a href="#__codelineno-0-478">478</a></span> |
| <span class="normal"><a href="#__codelineno-0-479">479</a></span> |
| <span class="normal"><a href="#__codelineno-0-480">480</a></span> |
| <span class="normal"><a href="#__codelineno-0-481">481</a></span> |
| <span class="normal"><a href="#__codelineno-0-482">482</a></span> |
| <span class="normal"><a href="#__codelineno-0-483">483</a></span> |
| <span class="normal"><a href="#__codelineno-0-484">484</a></span> |
| <span class="normal"><a href="#__codelineno-0-485">485</a></span> |
| <span class="normal"><a href="#__codelineno-0-486">486</a></span> |
| <span class="normal"><a href="#__codelineno-0-487">487</a></span> |
| <span class="normal"><a href="#__codelineno-0-488">488</a></span> |
| <span class="normal"><a href="#__codelineno-0-489">489</a></span> |
| <span class="normal"><a href="#__codelineno-0-490">490</a></span> |
| <span class="normal"><a href="#__codelineno-0-491">491</a></span> |
| <span class="normal"><a href="#__codelineno-0-492">492</a></span> |
| <span class="normal"><a href="#__codelineno-0-493">493</a></span> |
| <span class="normal"><a href="#__codelineno-0-494">494</a></span> |
| <span class="normal"><a href="#__codelineno-0-495">495</a></span> |
| <span class="normal"><a href="#__codelineno-0-496">496</a></span> |
| <span class="normal"><a href="#__codelineno-0-497">497</a></span> |
| <span class="normal"><a href="#__codelineno-0-498">498</a></span> |
| <span class="normal"><a href="#__codelineno-0-499">499</a></span> |
| <span class="normal"><a href="#__codelineno-0-500">500</a></span> |
| <span class="normal"><a href="#__codelineno-0-501">501</a></span> |
| <span class="normal"><a href="#__codelineno-0-502">502</a></span> |
| <span class="normal"><a href="#__codelineno-0-503">503</a></span> |
| <span class="normal"><a href="#__codelineno-0-504">504</a></span> |
| <span class="normal"><a href="#__codelineno-0-505">505</a></span> |
| <span class="normal"><a href="#__codelineno-0-506">506</a></span> |
| <span class="normal"><a href="#__codelineno-0-507">507</a></span> |
| <span class="normal"><a href="#__codelineno-0-508">508</a></span> |
| <span class="normal"><a href="#__codelineno-0-509">509</a></span> |
| <span class="normal"><a href="#__codelineno-0-510">510</a></span> |
| <span class="normal"><a href="#__codelineno-0-511">511</a></span> |
| <span class="normal"><a href="#__codelineno-0-512">512</a></span> |
| <span class="normal"><a href="#__codelineno-0-513">513</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-477" name="__codelineno-0-477"></a><span class="k">def</span><span class="w"> </span><span class="nf">append</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-478" name="__codelineno-0-478"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-479" name="__codelineno-0-479"></a><span class="sd"> Shorthand API for appending a PyArrow table to a table transaction.</span> |
| <a id="__codelineno-0-480" name="__codelineno-0-480"></a> |
| <a id="__codelineno-0-481" name="__codelineno-0-481"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-482" name="__codelineno-0-482"></a><span class="sd"> df: The Arrow dataframe that will be appended to overwrite the table</span> |
| <a id="__codelineno-0-483" name="__codelineno-0-483"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-484" name="__codelineno-0-484"></a><span class="sd"> branch: Branch Reference to run the append operation</span> |
| <a id="__codelineno-0-485" name="__codelineno-0-485"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-486" name="__codelineno-0-486"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-487" name="__codelineno-0-487"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-488" name="__codelineno-0-488"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-489" name="__codelineno-0-489"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-490" name="__codelineno-0-490"></a> |
| <a id="__codelineno-0-491" name="__codelineno-0-491"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">,</span> <span class="n">_dataframe_to_data_files</span> |
| <a id="__codelineno-0-492" name="__codelineno-0-492"></a> |
| <a id="__codelineno-0-493" name="__codelineno-0-493"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-494" name="__codelineno-0-494"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Expected PyArrow table, got: </span><span class="si">{</span><span class="n">df</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-495" name="__codelineno-0-495"></a> |
| <a id="__codelineno-0-496" name="__codelineno-0-496"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-497" name="__codelineno-0-497"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-498" name="__codelineno-0-498"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-499" name="__codelineno-0-499"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-500" name="__codelineno-0-500"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-501" name="__codelineno-0-501"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-502" name="__codelineno-0-502"></a> <span class="p">)</span> |
| <a id="__codelineno-0-503" name="__codelineno-0-503"></a> |
| <a id="__codelineno-0-504" name="__codelineno-0-504"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-505" name="__codelineno-0-505"></a> <span class="c1"># skip writing data files if the dataframe is empty</span> |
| <a id="__codelineno-0-506" name="__codelineno-0-506"></a> <span class="k">if</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-507" name="__codelineno-0-507"></a> <span class="n">data_files</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span> |
| <a id="__codelineno-0-508" name="__codelineno-0-508"></a> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-509" name="__codelineno-0-509"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">write_uuid</span><span class="o">=</span><span class="n">append_files</span><span class="o">.</span><span class="n">commit_uuid</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-510" name="__codelineno-0-510"></a> <span class="p">)</span> |
| <a id="__codelineno-0-511" name="__codelineno-0-511"></a> <span class="p">)</span> |
| <a id="__codelineno-0-512" name="__codelineno-0-512"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-513" name="__codelineno-0-513"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.commit_transaction" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">commit_transaction</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.commit_transaction" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Commit the changes to the catalog.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Table (pyiceberg.table.Table)" href="#pyiceberg.table.Table">Table</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The table with the updates applied.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-954">954</a></span> |
| <span class="normal"><a href="#__codelineno-0-955">955</a></span> |
| <span class="normal"><a href="#__codelineno-0-956">956</a></span> |
| <span class="normal"><a href="#__codelineno-0-957">957</a></span> |
| <span class="normal"><a href="#__codelineno-0-958">958</a></span> |
| <span class="normal"><a href="#__codelineno-0-959">959</a></span> |
| <span class="normal"><a href="#__codelineno-0-960">960</a></span> |
| <span class="normal"><a href="#__codelineno-0-961">961</a></span> |
| <span class="normal"><a href="#__codelineno-0-962">962</a></span> |
| <span class="normal"><a href="#__codelineno-0-963">963</a></span> |
| <span class="normal"><a href="#__codelineno-0-964">964</a></span> |
| <span class="normal"><a href="#__codelineno-0-965">965</a></span> |
| <span class="normal"><a href="#__codelineno-0-966">966</a></span> |
| <span class="normal"><a href="#__codelineno-0-967">967</a></span> |
| <span class="normal"><a href="#__codelineno-0-968">968</a></span> |
| <span class="normal"><a href="#__codelineno-0-969">969</a></span> |
| <span class="normal"><a href="#__codelineno-0-970">970</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-954" name="__codelineno-0-954"></a><span class="k">def</span><span class="w"> </span><span class="nf">commit_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Table</span><span class="p">:</span> |
| <a id="__codelineno-0-955" name="__codelineno-0-955"></a><span class="w"> </span><span class="sd">"""Commit the changes to the catalog.</span> |
| <a id="__codelineno-0-956" name="__codelineno-0-956"></a> |
| <a id="__codelineno-0-957" name="__codelineno-0-957"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-958" name="__codelineno-0-958"></a><span class="sd"> The table with the updates applied.</span> |
| <a id="__codelineno-0-959" name="__codelineno-0-959"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-960" name="__codelineno-0-960"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-961" name="__codelineno-0-961"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">+=</span> <span class="p">(</span><span class="n">AssertTableUUID</span><span class="p">(</span><span class="n">uuid</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">table_uuid</span><span class="p">),)</span> |
| <a id="__codelineno-0-962" name="__codelineno-0-962"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">_do_commit</span><span class="p">(</span> <span class="c1"># pylint: disable=W0212</span> |
| <a id="__codelineno-0-963" name="__codelineno-0-963"></a> <span class="n">updates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_updates</span><span class="p">,</span> |
| <a id="__codelineno-0-964" name="__codelineno-0-964"></a> <span class="n">requirements</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span><span class="p">,</span> |
| <a id="__codelineno-0-965" name="__codelineno-0-965"></a> <span class="p">)</span> |
| <a id="__codelineno-0-966" name="__codelineno-0-966"></a> |
| <a id="__codelineno-0-967" name="__codelineno-0-967"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_updates</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-968" name="__codelineno-0-968"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_requirements</span> <span class="o">=</span> <span class="p">()</span> |
| <a id="__codelineno-0-969" name="__codelineno-0-969"></a> |
| <a id="__codelineno-0-970" name="__codelineno-0-970"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_table</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.delete" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">delete</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.delete" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for deleting record from a table.</p> |
| <p>A delete may produce zero or more snapshots based on the operation:</p> |
| <div class="highlight"><pre><span></span><code>- DELETE: In case existing Parquet files can be dropped completely. |
| - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the delete filter. |
| </code></pre></div> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>delete_filter</code> |
| </td> |
| <td> |
| <code><span title="typing.Union">Union</span>[<span title="str">str</span>, <a class="autorefs autorefs-internal" title="BooleanExpression (pyiceberg.expressions.BooleanExpression)" href="../expressions/#pyiceberg.expressions.BooleanExpression">BooleanExpression</a>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A boolean expression to delete rows from a table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A bool determine if the provided <code>delete_filter</code> is case-sensitive</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the delete operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-638">638</a></span> |
| <span class="normal"><a href="#__codelineno-0-639">639</a></span> |
| <span class="normal"><a href="#__codelineno-0-640">640</a></span> |
| <span class="normal"><a href="#__codelineno-0-641">641</a></span> |
| <span class="normal"><a href="#__codelineno-0-642">642</a></span> |
| <span class="normal"><a href="#__codelineno-0-643">643</a></span> |
| <span class="normal"><a href="#__codelineno-0-644">644</a></span> |
| <span class="normal"><a href="#__codelineno-0-645">645</a></span> |
| <span class="normal"><a href="#__codelineno-0-646">646</a></span> |
| <span class="normal"><a href="#__codelineno-0-647">647</a></span> |
| <span class="normal"><a href="#__codelineno-0-648">648</a></span> |
| <span class="normal"><a href="#__codelineno-0-649">649</a></span> |
| <span class="normal"><a href="#__codelineno-0-650">650</a></span> |
| <span class="normal"><a href="#__codelineno-0-651">651</a></span> |
| <span class="normal"><a href="#__codelineno-0-652">652</a></span> |
| <span class="normal"><a href="#__codelineno-0-653">653</a></span> |
| <span class="normal"><a href="#__codelineno-0-654">654</a></span> |
| <span class="normal"><a href="#__codelineno-0-655">655</a></span> |
| <span class="normal"><a href="#__codelineno-0-656">656</a></span> |
| <span class="normal"><a href="#__codelineno-0-657">657</a></span> |
| <span class="normal"><a href="#__codelineno-0-658">658</a></span> |
| <span class="normal"><a href="#__codelineno-0-659">659</a></span> |
| <span class="normal"><a href="#__codelineno-0-660">660</a></span> |
| <span class="normal"><a href="#__codelineno-0-661">661</a></span> |
| <span class="normal"><a href="#__codelineno-0-662">662</a></span> |
| <span class="normal"><a href="#__codelineno-0-663">663</a></span> |
| <span class="normal"><a href="#__codelineno-0-664">664</a></span> |
| <span class="normal"><a href="#__codelineno-0-665">665</a></span> |
| <span class="normal"><a href="#__codelineno-0-666">666</a></span> |
| <span class="normal"><a href="#__codelineno-0-667">667</a></span> |
| <span class="normal"><a href="#__codelineno-0-668">668</a></span> |
| <span class="normal"><a href="#__codelineno-0-669">669</a></span> |
| <span class="normal"><a href="#__codelineno-0-670">670</a></span> |
| <span class="normal"><a href="#__codelineno-0-671">671</a></span> |
| <span class="normal"><a href="#__codelineno-0-672">672</a></span> |
| <span class="normal"><a href="#__codelineno-0-673">673</a></span> |
| <span class="normal"><a href="#__codelineno-0-674">674</a></span> |
| <span class="normal"><a href="#__codelineno-0-675">675</a></span> |
| <span class="normal"><a href="#__codelineno-0-676">676</a></span> |
| <span class="normal"><a href="#__codelineno-0-677">677</a></span> |
| <span class="normal"><a href="#__codelineno-0-678">678</a></span> |
| <span class="normal"><a href="#__codelineno-0-679">679</a></span> |
| <span class="normal"><a href="#__codelineno-0-680">680</a></span> |
| <span class="normal"><a href="#__codelineno-0-681">681</a></span> |
| <span class="normal"><a href="#__codelineno-0-682">682</a></span> |
| <span class="normal"><a href="#__codelineno-0-683">683</a></span> |
| <span class="normal"><a href="#__codelineno-0-684">684</a></span> |
| <span class="normal"><a href="#__codelineno-0-685">685</a></span> |
| <span class="normal"><a href="#__codelineno-0-686">686</a></span> |
| <span class="normal"><a href="#__codelineno-0-687">687</a></span> |
| <span class="normal"><a href="#__codelineno-0-688">688</a></span> |
| <span class="normal"><a href="#__codelineno-0-689">689</a></span> |
| <span class="normal"><a href="#__codelineno-0-690">690</a></span> |
| <span class="normal"><a href="#__codelineno-0-691">691</a></span> |
| <span class="normal"><a href="#__codelineno-0-692">692</a></span> |
| <span class="normal"><a href="#__codelineno-0-693">693</a></span> |
| <span class="normal"><a href="#__codelineno-0-694">694</a></span> |
| <span class="normal"><a href="#__codelineno-0-695">695</a></span> |
| <span class="normal"><a href="#__codelineno-0-696">696</a></span> |
| <span class="normal"><a href="#__codelineno-0-697">697</a></span> |
| <span class="normal"><a href="#__codelineno-0-698">698</a></span> |
| <span class="normal"><a href="#__codelineno-0-699">699</a></span> |
| <span class="normal"><a href="#__codelineno-0-700">700</a></span> |
| <span class="normal"><a href="#__codelineno-0-701">701</a></span> |
| <span class="normal"><a href="#__codelineno-0-702">702</a></span> |
| <span class="normal"><a href="#__codelineno-0-703">703</a></span> |
| <span class="normal"><a href="#__codelineno-0-704">704</a></span> |
| <span class="normal"><a href="#__codelineno-0-705">705</a></span> |
| <span class="normal"><a href="#__codelineno-0-706">706</a></span> |
| <span class="normal"><a href="#__codelineno-0-707">707</a></span> |
| <span class="normal"><a href="#__codelineno-0-708">708</a></span> |
| <span class="normal"><a href="#__codelineno-0-709">709</a></span> |
| <span class="normal"><a href="#__codelineno-0-710">710</a></span> |
| <span class="normal"><a href="#__codelineno-0-711">711</a></span> |
| <span class="normal"><a href="#__codelineno-0-712">712</a></span> |
| <span class="normal"><a href="#__codelineno-0-713">713</a></span> |
| <span class="normal"><a href="#__codelineno-0-714">714</a></span> |
| <span class="normal"><a href="#__codelineno-0-715">715</a></span> |
| <span class="normal"><a href="#__codelineno-0-716">716</a></span> |
| <span class="normal"><a href="#__codelineno-0-717">717</a></span> |
| <span class="normal"><a href="#__codelineno-0-718">718</a></span> |
| <span class="normal"><a href="#__codelineno-0-719">719</a></span> |
| <span class="normal"><a href="#__codelineno-0-720">720</a></span> |
| <span class="normal"><a href="#__codelineno-0-721">721</a></span> |
| <span class="normal"><a href="#__codelineno-0-722">722</a></span> |
| <span class="normal"><a href="#__codelineno-0-723">723</a></span> |
| <span class="normal"><a href="#__codelineno-0-724">724</a></span> |
| <span class="normal"><a href="#__codelineno-0-725">725</a></span> |
| <span class="normal"><a href="#__codelineno-0-726">726</a></span> |
| <span class="normal"><a href="#__codelineno-0-727">727</a></span> |
| <span class="normal"><a href="#__codelineno-0-728">728</a></span> |
| <span class="normal"><a href="#__codelineno-0-729">729</a></span> |
| <span class="normal"><a href="#__codelineno-0-730">730</a></span> |
| <span class="normal"><a href="#__codelineno-0-731">731</a></span> |
| <span class="normal"><a href="#__codelineno-0-732">732</a></span> |
| <span class="normal"><a href="#__codelineno-0-733">733</a></span> |
| <span class="normal"><a href="#__codelineno-0-734">734</a></span> |
| <span class="normal"><a href="#__codelineno-0-735">735</a></span> |
| <span class="normal"><a href="#__codelineno-0-736">736</a></span> |
| <span class="normal"><a href="#__codelineno-0-737">737</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-638" name="__codelineno-0-638"></a><span class="k">def</span><span class="w"> </span><span class="nf">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-639" name="__codelineno-0-639"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-640" name="__codelineno-0-640"></a> <span class="n">delete_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">BooleanExpression</span><span class="p">],</span> |
| <a id="__codelineno-0-641" name="__codelineno-0-641"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-642" name="__codelineno-0-642"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-643" name="__codelineno-0-643"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-644" name="__codelineno-0-644"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-645" name="__codelineno-0-645"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-646" name="__codelineno-0-646"></a><span class="sd"> Shorthand for deleting record from a table.</span> |
| <a id="__codelineno-0-647" name="__codelineno-0-647"></a> |
| <a id="__codelineno-0-648" name="__codelineno-0-648"></a><span class="sd"> A delete may produce zero or more snapshots based on the operation:</span> |
| <a id="__codelineno-0-649" name="__codelineno-0-649"></a> |
| <a id="__codelineno-0-650" name="__codelineno-0-650"></a><span class="sd"> - DELETE: In case existing Parquet files can be dropped completely.</span> |
| <a id="__codelineno-0-651" name="__codelineno-0-651"></a><span class="sd"> - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the delete filter.</span> |
| <a id="__codelineno-0-652" name="__codelineno-0-652"></a> |
| <a id="__codelineno-0-653" name="__codelineno-0-653"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-654" name="__codelineno-0-654"></a><span class="sd"> delete_filter: A boolean expression to delete rows from a table</span> |
| <a id="__codelineno-0-655" name="__codelineno-0-655"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-656" name="__codelineno-0-656"></a><span class="sd"> case_sensitive: A bool determine if the provided `delete_filter` is case-sensitive</span> |
| <a id="__codelineno-0-657" name="__codelineno-0-657"></a><span class="sd"> branch: Branch Reference to run the delete operation</span> |
| <a id="__codelineno-0-658" name="__codelineno-0-658"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-659" name="__codelineno-0-659"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span> |
| <a id="__codelineno-0-660" name="__codelineno-0-660"></a> <span class="n">ArrowScan</span><span class="p">,</span> |
| <a id="__codelineno-0-661" name="__codelineno-0-661"></a> <span class="n">_dataframe_to_data_files</span><span class="p">,</span> |
| <a id="__codelineno-0-662" name="__codelineno-0-662"></a> <span class="n">_expression_to_complementary_pyarrow</span><span class="p">,</span> |
| <a id="__codelineno-0-663" name="__codelineno-0-663"></a> <span class="p">)</span> |
| <a id="__codelineno-0-664" name="__codelineno-0-664"></a> |
| <a id="__codelineno-0-665" name="__codelineno-0-665"></a> <span class="k">if</span> <span class="p">(</span> |
| <a id="__codelineno-0-666" name="__codelineno-0-666"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">TableProperties</span><span class="o">.</span><span class="n">DELETE_MODE</span><span class="p">,</span> <span class="n">TableProperties</span><span class="o">.</span><span class="n">DELETE_MODE_DEFAULT</span><span class="p">)</span> |
| <a id="__codelineno-0-667" name="__codelineno-0-667"></a> <span class="o">==</span> <span class="n">TableProperties</span><span class="o">.</span><span class="n">DELETE_MODE_MERGE_ON_READ</span> |
| <a id="__codelineno-0-668" name="__codelineno-0-668"></a> <span class="p">):</span> |
| <a id="__codelineno-0-669" name="__codelineno-0-669"></a> <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Merge on read is not yet supported, falling back to copy-on-write"</span><span class="p">)</span> |
| <a id="__codelineno-0-670" name="__codelineno-0-670"></a> |
| <a id="__codelineno-0-671" name="__codelineno-0-671"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <a id="__codelineno-0-672" name="__codelineno-0-672"></a> <span class="n">delete_filter</span> <span class="o">=</span> <span class="n">_parse_row_filter</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-673" name="__codelineno-0-673"></a> |
| <a id="__codelineno-0-674" name="__codelineno-0-674"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">update_snapshot</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span> <span class="k">as</span> <span class="n">delete_snapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-675" name="__codelineno-0-675"></a> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">delete_by_predicate</span><span class="p">(</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-676" name="__codelineno-0-676"></a> |
| <a id="__codelineno-0-677" name="__codelineno-0-677"></a> <span class="c1"># Check if there are any files that require an actual rewrite of a data file</span> |
| <a id="__codelineno-0-678" name="__codelineno-0-678"></a> <span class="k">if</span> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">rewrites_needed</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span> |
| <a id="__codelineno-0-679" name="__codelineno-0-679"></a> <span class="n">bound_delete_filter</span> <span class="o">=</span> <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> <span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-680" name="__codelineno-0-680"></a> <span class="n">preserve_row_filter</span> <span class="o">=</span> <span class="n">_expression_to_complementary_pyarrow</span><span class="p">(</span><span class="n">bound_delete_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-681" name="__codelineno-0-681"></a> |
| <a id="__codelineno-0-682" name="__codelineno-0-682"></a> <span class="n">file_scan</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_scan</span><span class="p">(</span><span class="n">row_filter</span><span class="o">=</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-683" name="__codelineno-0-683"></a> <span class="k">if</span> <span class="n">branch</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-684" name="__codelineno-0-684"></a> <span class="n">file_scan</span> <span class="o">=</span> <span class="n">file_scan</span><span class="o">.</span><span class="n">use_ref</span><span class="p">(</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-685" name="__codelineno-0-685"></a> <span class="n">files</span> <span class="o">=</span> <span class="n">file_scan</span><span class="o">.</span><span class="n">plan_files</span><span class="p">()</span> |
| <a id="__codelineno-0-686" name="__codelineno-0-686"></a> |
| <a id="__codelineno-0-687" name="__codelineno-0-687"></a> <span class="n">commit_uuid</span> <span class="o">=</span> <span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">()</span> |
| <a id="__codelineno-0-688" name="__codelineno-0-688"></a> <span class="n">counter</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> |
| <a id="__codelineno-0-689" name="__codelineno-0-689"></a> |
| <a id="__codelineno-0-690" name="__codelineno-0-690"></a> <span class="n">replaced_files</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">DataFile</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">DataFile</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-691" name="__codelineno-0-691"></a> <span class="c1"># This will load the Parquet file into memory, including:</span> |
| <a id="__codelineno-0-692" name="__codelineno-0-692"></a> <span class="c1"># - Filter out the rows based on the delete filter</span> |
| <a id="__codelineno-0-693" name="__codelineno-0-693"></a> <span class="c1"># - Projecting it to the current schema</span> |
| <a id="__codelineno-0-694" name="__codelineno-0-694"></a> <span class="c1"># - Applying the positional deletes if they are there</span> |
| <a id="__codelineno-0-695" name="__codelineno-0-695"></a> <span class="c1"># When writing</span> |
| <a id="__codelineno-0-696" name="__codelineno-0-696"></a> <span class="c1"># - Apply the latest partition-spec</span> |
| <a id="__codelineno-0-697" name="__codelineno-0-697"></a> <span class="c1"># - And sort order when added</span> |
| <a id="__codelineno-0-698" name="__codelineno-0-698"></a> <span class="k">for</span> <span class="n">original_file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span> |
| <a id="__codelineno-0-699" name="__codelineno-0-699"></a> <span class="n">df</span> <span class="o">=</span> <span class="n">ArrowScan</span><span class="p">(</span> |
| <a id="__codelineno-0-700" name="__codelineno-0-700"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-701" name="__codelineno-0-701"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-702" name="__codelineno-0-702"></a> <span class="n">projected_schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-703" name="__codelineno-0-703"></a> <span class="n">row_filter</span><span class="o">=</span><span class="n">AlwaysTrue</span><span class="p">(),</span> |
| <a id="__codelineno-0-704" name="__codelineno-0-704"></a> <span class="p">)</span><span class="o">.</span><span class="n">to_table</span><span class="p">(</span><span class="n">tasks</span><span class="o">=</span><span class="p">[</span><span class="n">original_file</span><span class="p">])</span> |
| <a id="__codelineno-0-705" name="__codelineno-0-705"></a> <span class="n">filtered_df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">preserve_row_filter</span><span class="p">)</span> |
| <a id="__codelineno-0-706" name="__codelineno-0-706"></a> |
| <a id="__codelineno-0-707" name="__codelineno-0-707"></a> <span class="c1"># Only rewrite if there are records being deleted</span> |
| <a id="__codelineno-0-708" name="__codelineno-0-708"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">filtered_df</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-709" name="__codelineno-0-709"></a> <span class="n">replaced_files</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">original_file</span><span class="o">.</span><span class="n">file</span><span class="p">,</span> <span class="p">[]))</span> |
| <a id="__codelineno-0-710" name="__codelineno-0-710"></a> <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">filtered_df</span><span class="p">):</span> |
| <a id="__codelineno-0-711" name="__codelineno-0-711"></a> <span class="n">replaced_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span> |
| <a id="__codelineno-0-712" name="__codelineno-0-712"></a> <span class="p">(</span> |
| <a id="__codelineno-0-713" name="__codelineno-0-713"></a> <span class="n">original_file</span><span class="o">.</span><span class="n">file</span><span class="p">,</span> |
| <a id="__codelineno-0-714" name="__codelineno-0-714"></a> <span class="nb">list</span><span class="p">(</span> |
| <a id="__codelineno-0-715" name="__codelineno-0-715"></a> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-716" name="__codelineno-0-716"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-717" name="__codelineno-0-717"></a> <span class="n">df</span><span class="o">=</span><span class="n">filtered_df</span><span class="p">,</span> |
| <a id="__codelineno-0-718" name="__codelineno-0-718"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-719" name="__codelineno-0-719"></a> <span class="n">write_uuid</span><span class="o">=</span><span class="n">commit_uuid</span><span class="p">,</span> |
| <a id="__codelineno-0-720" name="__codelineno-0-720"></a> <span class="n">counter</span><span class="o">=</span><span class="n">counter</span><span class="p">,</span> |
| <a id="__codelineno-0-721" name="__codelineno-0-721"></a> <span class="p">)</span> |
| <a id="__codelineno-0-722" name="__codelineno-0-722"></a> <span class="p">),</span> |
| <a id="__codelineno-0-723" name="__codelineno-0-723"></a> <span class="p">)</span> |
| <a id="__codelineno-0-724" name="__codelineno-0-724"></a> <span class="p">)</span> |
| <a id="__codelineno-0-725" name="__codelineno-0-725"></a> |
| <a id="__codelineno-0-726" name="__codelineno-0-726"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">replaced_files</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-727" name="__codelineno-0-727"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">update_snapshot</span><span class="p">(</span> |
| <a id="__codelineno-0-728" name="__codelineno-0-728"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span> |
| <a id="__codelineno-0-729" name="__codelineno-0-729"></a> <span class="p">)</span><span class="o">.</span><span class="n">overwrite</span><span class="p">()</span> <span class="k">as</span> <span class="n">overwrite_snapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-730" name="__codelineno-0-730"></a> <span class="n">overwrite_snapshot</span><span class="o">.</span><span class="n">commit_uuid</span> <span class="o">=</span> <span class="n">commit_uuid</span> |
| <a id="__codelineno-0-731" name="__codelineno-0-731"></a> <span class="k">for</span> <span class="n">original_data_file</span><span class="p">,</span> <span class="n">replaced_data_files</span> <span class="ow">in</span> <span class="n">replaced_files</span><span class="p">:</span> |
| <a id="__codelineno-0-732" name="__codelineno-0-732"></a> <span class="n">overwrite_snapshot</span><span class="o">.</span><span class="n">delete_data_file</span><span class="p">(</span><span class="n">original_data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-733" name="__codelineno-0-733"></a> <span class="k">for</span> <span class="n">replaced_data_file</span> <span class="ow">in</span> <span class="n">replaced_data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-734" name="__codelineno-0-734"></a> <span class="n">overwrite_snapshot</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">replaced_data_file</span><span class="p">)</span> |
| <a id="__codelineno-0-735" name="__codelineno-0-735"></a> |
| <a id="__codelineno-0-736" name="__codelineno-0-736"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">files_affected</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">delete_snapshot</span><span class="o">.</span><span class="n">rewrites_needed</span><span class="p">:</span> |
| <a id="__codelineno-0-737" name="__codelineno-0-737"></a> <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Delete operation did not match any records"</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.dynamic_partition_overwrite" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">dynamic_partition_overwrite</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.dynamic_partition_overwrite" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for overwriting existing partitions with a PyArrow table.</p> |
| <p>The function detects partition values in the provided arrow table using the current |
| partition spec, and deletes existing partitions matching these values. Finally, the |
| data in the table is appended to the table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>df</code> |
| </td> |
| <td> |
| <code><span title="pyarrow.Table">Table</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The Arrow dataframe that will be used to overwrite the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the dynamic partition overwrite operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-515">515</a></span> |
| <span class="normal"><a href="#__codelineno-0-516">516</a></span> |
| <span class="normal"><a href="#__codelineno-0-517">517</a></span> |
| <span class="normal"><a href="#__codelineno-0-518">518</a></span> |
| <span class="normal"><a href="#__codelineno-0-519">519</a></span> |
| <span class="normal"><a href="#__codelineno-0-520">520</a></span> |
| <span class="normal"><a href="#__codelineno-0-521">521</a></span> |
| <span class="normal"><a href="#__codelineno-0-522">522</a></span> |
| <span class="normal"><a href="#__codelineno-0-523">523</a></span> |
| <span class="normal"><a href="#__codelineno-0-524">524</a></span> |
| <span class="normal"><a href="#__codelineno-0-525">525</a></span> |
| <span class="normal"><a href="#__codelineno-0-526">526</a></span> |
| <span class="normal"><a href="#__codelineno-0-527">527</a></span> |
| <span class="normal"><a href="#__codelineno-0-528">528</a></span> |
| <span class="normal"><a href="#__codelineno-0-529">529</a></span> |
| <span class="normal"><a href="#__codelineno-0-530">530</a></span> |
| <span class="normal"><a href="#__codelineno-0-531">531</a></span> |
| <span class="normal"><a href="#__codelineno-0-532">532</a></span> |
| <span class="normal"><a href="#__codelineno-0-533">533</a></span> |
| <span class="normal"><a href="#__codelineno-0-534">534</a></span> |
| <span class="normal"><a href="#__codelineno-0-535">535</a></span> |
| <span class="normal"><a href="#__codelineno-0-536">536</a></span> |
| <span class="normal"><a href="#__codelineno-0-537">537</a></span> |
| <span class="normal"><a href="#__codelineno-0-538">538</a></span> |
| <span class="normal"><a href="#__codelineno-0-539">539</a></span> |
| <span class="normal"><a href="#__codelineno-0-540">540</a></span> |
| <span class="normal"><a href="#__codelineno-0-541">541</a></span> |
| <span class="normal"><a href="#__codelineno-0-542">542</a></span> |
| <span class="normal"><a href="#__codelineno-0-543">543</a></span> |
| <span class="normal"><a href="#__codelineno-0-544">544</a></span> |
| <span class="normal"><a href="#__codelineno-0-545">545</a></span> |
| <span class="normal"><a href="#__codelineno-0-546">546</a></span> |
| <span class="normal"><a href="#__codelineno-0-547">547</a></span> |
| <span class="normal"><a href="#__codelineno-0-548">548</a></span> |
| <span class="normal"><a href="#__codelineno-0-549">549</a></span> |
| <span class="normal"><a href="#__codelineno-0-550">550</a></span> |
| <span class="normal"><a href="#__codelineno-0-551">551</a></span> |
| <span class="normal"><a href="#__codelineno-0-552">552</a></span> |
| <span class="normal"><a href="#__codelineno-0-553">553</a></span> |
| <span class="normal"><a href="#__codelineno-0-554">554</a></span> |
| <span class="normal"><a href="#__codelineno-0-555">555</a></span> |
| <span class="normal"><a href="#__codelineno-0-556">556</a></span> |
| <span class="normal"><a href="#__codelineno-0-557">557</a></span> |
| <span class="normal"><a href="#__codelineno-0-558">558</a></span> |
| <span class="normal"><a href="#__codelineno-0-559">559</a></span> |
| <span class="normal"><a href="#__codelineno-0-560">560</a></span> |
| <span class="normal"><a href="#__codelineno-0-561">561</a></span> |
| <span class="normal"><a href="#__codelineno-0-562">562</a></span> |
| <span class="normal"><a href="#__codelineno-0-563">563</a></span> |
| <span class="normal"><a href="#__codelineno-0-564">564</a></span> |
| <span class="normal"><a href="#__codelineno-0-565">565</a></span> |
| <span class="normal"><a href="#__codelineno-0-566">566</a></span> |
| <span class="normal"><a href="#__codelineno-0-567">567</a></span> |
| <span class="normal"><a href="#__codelineno-0-568">568</a></span> |
| <span class="normal"><a href="#__codelineno-0-569">569</a></span> |
| <span class="normal"><a href="#__codelineno-0-570">570</a></span> |
| <span class="normal"><a href="#__codelineno-0-571">571</a></span> |
| <span class="normal"><a href="#__codelineno-0-572">572</a></span> |
| <span class="normal"><a href="#__codelineno-0-573">573</a></span> |
| <span class="normal"><a href="#__codelineno-0-574">574</a></span> |
| <span class="normal"><a href="#__codelineno-0-575">575</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-515" name="__codelineno-0-515"></a><span class="k">def</span><span class="w"> </span><span class="nf">dynamic_partition_overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-516" name="__codelineno-0-516"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-517" name="__codelineno-0-517"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-518" name="__codelineno-0-518"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-519" name="__codelineno-0-519"></a><span class="sd"> Shorthand for overwriting existing partitions with a PyArrow table.</span> |
| <a id="__codelineno-0-520" name="__codelineno-0-520"></a> |
| <a id="__codelineno-0-521" name="__codelineno-0-521"></a><span class="sd"> The function detects partition values in the provided arrow table using the current</span> |
| <a id="__codelineno-0-522" name="__codelineno-0-522"></a><span class="sd"> partition spec, and deletes existing partitions matching these values. Finally, the</span> |
| <a id="__codelineno-0-523" name="__codelineno-0-523"></a><span class="sd"> data in the table is appended to the table.</span> |
| <a id="__codelineno-0-524" name="__codelineno-0-524"></a> |
| <a id="__codelineno-0-525" name="__codelineno-0-525"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-526" name="__codelineno-0-526"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-527" name="__codelineno-0-527"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-528" name="__codelineno-0-528"></a><span class="sd"> branch: Branch Reference to run the dynamic partition overwrite operation</span> |
| <a id="__codelineno-0-529" name="__codelineno-0-529"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-530" name="__codelineno-0-530"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-531" name="__codelineno-0-531"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-532" name="__codelineno-0-532"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-533" name="__codelineno-0-533"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-534" name="__codelineno-0-534"></a> |
| <a id="__codelineno-0-535" name="__codelineno-0-535"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">,</span> <span class="n">_dataframe_to_data_files</span> |
| <a id="__codelineno-0-536" name="__codelineno-0-536"></a> |
| <a id="__codelineno-0-537" name="__codelineno-0-537"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-538" name="__codelineno-0-538"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Expected PyArrow table, got: </span><span class="si">{</span><span class="n">df</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-539" name="__codelineno-0-539"></a> |
| <a id="__codelineno-0-540" name="__codelineno-0-540"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span><span class="o">.</span><span class="n">is_unpartitioned</span><span class="p">():</span> |
| <a id="__codelineno-0-541" name="__codelineno-0-541"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Cannot apply dynamic overwrite on an unpartitioned table."</span><span class="p">)</span> |
| <a id="__codelineno-0-542" name="__codelineno-0-542"></a> |
| <a id="__codelineno-0-543" name="__codelineno-0-543"></a> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">spec</span><span class="p">()</span><span class="o">.</span><span class="n">fields</span><span class="p">:</span> |
| <a id="__codelineno-0-544" name="__codelineno-0-544"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">transform</span><span class="p">,</span> <span class="n">IdentityTransform</span><span class="p">):</span> |
| <a id="__codelineno-0-545" name="__codelineno-0-545"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <a id="__codelineno-0-546" name="__codelineno-0-546"></a> <span class="sa">f</span><span class="s2">"For now dynamic overwrite does not support a table with non-identity-transform field in the latest partition spec: </span><span class="si">{</span><span class="n">field</span><span class="si">}</span><span class="s2">"</span> |
| <a id="__codelineno-0-547" name="__codelineno-0-547"></a> <span class="p">)</span> |
| <a id="__codelineno-0-548" name="__codelineno-0-548"></a> |
| <a id="__codelineno-0-549" name="__codelineno-0-549"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-550" name="__codelineno-0-550"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-551" name="__codelineno-0-551"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-552" name="__codelineno-0-552"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-553" name="__codelineno-0-553"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-554" name="__codelineno-0-554"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-555" name="__codelineno-0-555"></a> <span class="p">)</span> |
| <a id="__codelineno-0-556" name="__codelineno-0-556"></a> |
| <a id="__codelineno-0-557" name="__codelineno-0-557"></a> <span class="c1"># If dataframe does not have data, there is no need to overwrite</span> |
| <a id="__codelineno-0-558" name="__codelineno-0-558"></a> <span class="k">if</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-559" name="__codelineno-0-559"></a> <span class="k">return</span> |
| <a id="__codelineno-0-560" name="__codelineno-0-560"></a> |
| <a id="__codelineno-0-561" name="__codelineno-0-561"></a> <span class="n">append_snapshot_commit_uuid</span> <span class="o">=</span> <span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">()</span> |
| <a id="__codelineno-0-562" name="__codelineno-0-562"></a> <span class="n">data_files</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">DataFile</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span> |
| <a id="__codelineno-0-563" name="__codelineno-0-563"></a> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-564" name="__codelineno-0-564"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">metadata</span><span class="p">,</span> <span class="n">write_uuid</span><span class="o">=</span><span class="n">append_snapshot_commit_uuid</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-565" name="__codelineno-0-565"></a> <span class="p">)</span> |
| <a id="__codelineno-0-566" name="__codelineno-0-566"></a> <span class="p">)</span> |
| <a id="__codelineno-0-567" name="__codelineno-0-567"></a> |
| <a id="__codelineno-0-568" name="__codelineno-0-568"></a> <span class="n">partitions_to_overwrite</span> <span class="o">=</span> <span class="p">{</span><span class="n">data_file</span><span class="o">.</span><span class="n">partition</span> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">}</span> |
| <a id="__codelineno-0-569" name="__codelineno-0-569"></a> <span class="n">delete_filter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_partition_predicate</span><span class="p">(</span><span class="n">partition_records</span><span class="o">=</span><span class="n">partitions_to_overwrite</span><span class="p">)</span> |
| <a id="__codelineno-0-570" name="__codelineno-0-570"></a> <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">delete_filter</span><span class="o">=</span><span class="n">delete_filter</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-571" name="__codelineno-0-571"></a> |
| <a id="__codelineno-0-572" name="__codelineno-0-572"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-573" name="__codelineno-0-573"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">commit_uuid</span> <span class="o">=</span> <span class="n">append_snapshot_commit_uuid</span> |
| <a id="__codelineno-0-574" name="__codelineno-0-574"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-575" name="__codelineno-0-575"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.overwrite" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">overwrite</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">overwrite_filter</span><span class="o">=</span><span class="n">ALWAYS_TRUE</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.overwrite" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand for adding a table overwrite with a PyArrow table to the transaction.</p> |
| <p>An overwrite may produce zero or more snapshots based on the operation:</p> |
| <div class="highlight"><pre><span></span><code>- DELETE: In case existing Parquet files can be dropped completely. |
| - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the overwrite filter. |
| - APPEND: In case new data is being inserted into the table. |
| </code></pre></div> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>df</code> |
| </td> |
| <td> |
| <code><span title="pyarrow.Table">Table</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The Arrow dataframe that will be used to overwrite the table</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>overwrite_filter</code> |
| </td> |
| <td> |
| <code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="BooleanExpression (pyiceberg.expressions.BooleanExpression)" href="../expressions/#pyiceberg.expressions.BooleanExpression">BooleanExpression</a>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>ALWAYS_TRUE when you overwrite all the data, |
| or a boolean expression in case of a partial overwrite</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.ALWAYS_TRUE">ALWAYS_TRUE</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>snapshot_properties</code> |
| </td> |
| <td> |
| <code><span title="typing.Dict">Dict</span>[<span title="str">str</span>, <span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Custom properties to be added to the snapshot summary</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A bool determine if the provided <code>overwrite_filter</code> is case-sensitive</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>branch</code> |
| </td> |
| <td> |
| <code><span title="typing.Optional">Optional</span>[<span title="str">str</span>]</code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Branch Reference to run the overwrite operation</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.table.refs.MAIN_BRANCH">MAIN_BRANCH</span></code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-577">577</a></span> |
| <span class="normal"><a href="#__codelineno-0-578">578</a></span> |
| <span class="normal"><a href="#__codelineno-0-579">579</a></span> |
| <span class="normal"><a href="#__codelineno-0-580">580</a></span> |
| <span class="normal"><a href="#__codelineno-0-581">581</a></span> |
| <span class="normal"><a href="#__codelineno-0-582">582</a></span> |
| <span class="normal"><a href="#__codelineno-0-583">583</a></span> |
| <span class="normal"><a href="#__codelineno-0-584">584</a></span> |
| <span class="normal"><a href="#__codelineno-0-585">585</a></span> |
| <span class="normal"><a href="#__codelineno-0-586">586</a></span> |
| <span class="normal"><a href="#__codelineno-0-587">587</a></span> |
| <span class="normal"><a href="#__codelineno-0-588">588</a></span> |
| <span class="normal"><a href="#__codelineno-0-589">589</a></span> |
| <span class="normal"><a href="#__codelineno-0-590">590</a></span> |
| <span class="normal"><a href="#__codelineno-0-591">591</a></span> |
| <span class="normal"><a href="#__codelineno-0-592">592</a></span> |
| <span class="normal"><a href="#__codelineno-0-593">593</a></span> |
| <span class="normal"><a href="#__codelineno-0-594">594</a></span> |
| <span class="normal"><a href="#__codelineno-0-595">595</a></span> |
| <span class="normal"><a href="#__codelineno-0-596">596</a></span> |
| <span class="normal"><a href="#__codelineno-0-597">597</a></span> |
| <span class="normal"><a href="#__codelineno-0-598">598</a></span> |
| <span class="normal"><a href="#__codelineno-0-599">599</a></span> |
| <span class="normal"><a href="#__codelineno-0-600">600</a></span> |
| <span class="normal"><a href="#__codelineno-0-601">601</a></span> |
| <span class="normal"><a href="#__codelineno-0-602">602</a></span> |
| <span class="normal"><a href="#__codelineno-0-603">603</a></span> |
| <span class="normal"><a href="#__codelineno-0-604">604</a></span> |
| <span class="normal"><a href="#__codelineno-0-605">605</a></span> |
| <span class="normal"><a href="#__codelineno-0-606">606</a></span> |
| <span class="normal"><a href="#__codelineno-0-607">607</a></span> |
| <span class="normal"><a href="#__codelineno-0-608">608</a></span> |
| <span class="normal"><a href="#__codelineno-0-609">609</a></span> |
| <span class="normal"><a href="#__codelineno-0-610">610</a></span> |
| <span class="normal"><a href="#__codelineno-0-611">611</a></span> |
| <span class="normal"><a href="#__codelineno-0-612">612</a></span> |
| <span class="normal"><a href="#__codelineno-0-613">613</a></span> |
| <span class="normal"><a href="#__codelineno-0-614">614</a></span> |
| <span class="normal"><a href="#__codelineno-0-615">615</a></span> |
| <span class="normal"><a href="#__codelineno-0-616">616</a></span> |
| <span class="normal"><a href="#__codelineno-0-617">617</a></span> |
| <span class="normal"><a href="#__codelineno-0-618">618</a></span> |
| <span class="normal"><a href="#__codelineno-0-619">619</a></span> |
| <span class="normal"><a href="#__codelineno-0-620">620</a></span> |
| <span class="normal"><a href="#__codelineno-0-621">621</a></span> |
| <span class="normal"><a href="#__codelineno-0-622">622</a></span> |
| <span class="normal"><a href="#__codelineno-0-623">623</a></span> |
| <span class="normal"><a href="#__codelineno-0-624">624</a></span> |
| <span class="normal"><a href="#__codelineno-0-625">625</a></span> |
| <span class="normal"><a href="#__codelineno-0-626">626</a></span> |
| <span class="normal"><a href="#__codelineno-0-627">627</a></span> |
| <span class="normal"><a href="#__codelineno-0-628">628</a></span> |
| <span class="normal"><a href="#__codelineno-0-629">629</a></span> |
| <span class="normal"><a href="#__codelineno-0-630">630</a></span> |
| <span class="normal"><a href="#__codelineno-0-631">631</a></span> |
| <span class="normal"><a href="#__codelineno-0-632">632</a></span> |
| <span class="normal"><a href="#__codelineno-0-633">633</a></span> |
| <span class="normal"><a href="#__codelineno-0-634">634</a></span> |
| <span class="normal"><a href="#__codelineno-0-635">635</a></span> |
| <span class="normal"><a href="#__codelineno-0-636">636</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-577" name="__codelineno-0-577"></a><span class="k">def</span><span class="w"> </span><span class="nf">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-578" name="__codelineno-0-578"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-579" name="__codelineno-0-579"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-580" name="__codelineno-0-580"></a> <span class="n">overwrite_filter</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BooleanExpression</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">ALWAYS_TRUE</span><span class="p">,</span> |
| <a id="__codelineno-0-581" name="__codelineno-0-581"></a> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> |
| <a id="__codelineno-0-582" name="__codelineno-0-582"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-583" name="__codelineno-0-583"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-584" name="__codelineno-0-584"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-585" name="__codelineno-0-585"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-586" name="__codelineno-0-586"></a><span class="sd"> Shorthand for adding a table overwrite with a PyArrow table to the transaction.</span> |
| <a id="__codelineno-0-587" name="__codelineno-0-587"></a> |
| <a id="__codelineno-0-588" name="__codelineno-0-588"></a><span class="sd"> An overwrite may produce zero or more snapshots based on the operation:</span> |
| <a id="__codelineno-0-589" name="__codelineno-0-589"></a> |
| <a id="__codelineno-0-590" name="__codelineno-0-590"></a><span class="sd"> - DELETE: In case existing Parquet files can be dropped completely.</span> |
| <a id="__codelineno-0-591" name="__codelineno-0-591"></a><span class="sd"> - OVERWRITE: In case existing Parquet files need to be rewritten to drop rows that match the overwrite filter.</span> |
| <a id="__codelineno-0-592" name="__codelineno-0-592"></a><span class="sd"> - APPEND: In case new data is being inserted into the table.</span> |
| <a id="__codelineno-0-593" name="__codelineno-0-593"></a> |
| <a id="__codelineno-0-594" name="__codelineno-0-594"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-595" name="__codelineno-0-595"></a><span class="sd"> df: The Arrow dataframe that will be used to overwrite the table</span> |
| <a id="__codelineno-0-596" name="__codelineno-0-596"></a><span class="sd"> overwrite_filter: ALWAYS_TRUE when you overwrite all the data,</span> |
| <a id="__codelineno-0-597" name="__codelineno-0-597"></a><span class="sd"> or a boolean expression in case of a partial overwrite</span> |
| <a id="__codelineno-0-598" name="__codelineno-0-598"></a><span class="sd"> snapshot_properties: Custom properties to be added to the snapshot summary</span> |
| <a id="__codelineno-0-599" name="__codelineno-0-599"></a><span class="sd"> case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive</span> |
| <a id="__codelineno-0-600" name="__codelineno-0-600"></a><span class="sd"> branch: Branch Reference to run the overwrite operation</span> |
| <a id="__codelineno-0-601" name="__codelineno-0-601"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-602" name="__codelineno-0-602"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-603" name="__codelineno-0-603"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <a id="__codelineno-0-604" name="__codelineno-0-604"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-605" name="__codelineno-0-605"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-606" name="__codelineno-0-606"></a> |
| <a id="__codelineno-0-607" name="__codelineno-0-607"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">,</span> <span class="n">_dataframe_to_data_files</span> |
| <a id="__codelineno-0-608" name="__codelineno-0-608"></a> |
| <a id="__codelineno-0-609" name="__codelineno-0-609"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span> |
| <a id="__codelineno-0-610" name="__codelineno-0-610"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Expected PyArrow table, got: </span><span class="si">{</span><span class="n">df</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-611" name="__codelineno-0-611"></a> |
| <a id="__codelineno-0-612" name="__codelineno-0-612"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-613" name="__codelineno-0-613"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-614" name="__codelineno-0-614"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-615" name="__codelineno-0-615"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-616" name="__codelineno-0-616"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-617" name="__codelineno-0-617"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-618" name="__codelineno-0-618"></a> <span class="p">)</span> |
| <a id="__codelineno-0-619" name="__codelineno-0-619"></a> |
| <a id="__codelineno-0-620" name="__codelineno-0-620"></a> <span class="k">if</span> <span class="n">overwrite_filter</span> <span class="o">!=</span> <span class="n">AlwaysFalse</span><span class="p">():</span> |
| <a id="__codelineno-0-621" name="__codelineno-0-621"></a> <span class="c1"># Only delete when the filter is != AlwaysFalse</span> |
| <a id="__codelineno-0-622" name="__codelineno-0-622"></a> <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span> |
| <a id="__codelineno-0-623" name="__codelineno-0-623"></a> <span class="n">delete_filter</span><span class="o">=</span><span class="n">overwrite_filter</span><span class="p">,</span> |
| <a id="__codelineno-0-624" name="__codelineno-0-624"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-625" name="__codelineno-0-625"></a> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">,</span> |
| <a id="__codelineno-0-626" name="__codelineno-0-626"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-627" name="__codelineno-0-627"></a> <span class="p">)</span> |
| <a id="__codelineno-0-628" name="__codelineno-0-628"></a> |
| <a id="__codelineno-0-629" name="__codelineno-0-629"></a> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_append_snapshot_producer</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> <span class="k">as</span> <span class="n">append_files</span><span class="p">:</span> |
| <a id="__codelineno-0-630" name="__codelineno-0-630"></a> <span class="c1"># skip writing data files if the dataframe is empty</span> |
| <a id="__codelineno-0-631" name="__codelineno-0-631"></a> <span class="k">if</span> <span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-632" name="__codelineno-0-632"></a> <span class="n">data_files</span> <span class="o">=</span> <span class="n">_dataframe_to_data_files</span><span class="p">(</span> |
| <a id="__codelineno-0-633" name="__codelineno-0-633"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> <span class="n">write_uuid</span><span class="o">=</span><span class="n">append_files</span><span class="o">.</span><span class="n">commit_uuid</span><span class="p">,</span> <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span> |
| <a id="__codelineno-0-634" name="__codelineno-0-634"></a> <span class="p">)</span> |
| <a id="__codelineno-0-635" name="__codelineno-0-635"></a> <span class="k">for</span> <span class="n">data_file</span> <span class="ow">in</span> <span class="n">data_files</span><span class="p">:</span> |
| <a id="__codelineno-0-636" name="__codelineno-0-636"></a> <span class="n">append_files</span><span class="o">.</span><span class="n">append_data_file</span><span class="p">(</span><span class="n">data_file</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.remove_properties" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">remove_properties</span><span class="p">(</span><span class="o">*</span><span class="n">removals</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.remove_properties" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Remove properties.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>removals</code> |
| </td> |
| <td> |
| <code><span title="str">str</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>Properties to be removed.</p> |
| </div> |
| </td> |
| <td> |
| <code>()</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Transaction (pyiceberg.table.Transaction)" href="#pyiceberg.table.Transaction">Transaction</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The alter table builder.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-932">932</a></span> |
| <span class="normal"><a href="#__codelineno-0-933">933</a></span> |
| <span class="normal"><a href="#__codelineno-0-934">934</a></span> |
| <span class="normal"><a href="#__codelineno-0-935">935</a></span> |
| <span class="normal"><a href="#__codelineno-0-936">936</a></span> |
| <span class="normal"><a href="#__codelineno-0-937">937</a></span> |
| <span class="normal"><a href="#__codelineno-0-938">938</a></span> |
| <span class="normal"><a href="#__codelineno-0-939">939</a></span> |
| <span class="normal"><a href="#__codelineno-0-940">940</a></span> |
| <span class="normal"><a href="#__codelineno-0-941">941</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-932" name="__codelineno-0-932"></a><span class="k">def</span><span class="w"> </span><span class="nf">remove_properties</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">removals</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-933" name="__codelineno-0-933"></a><span class="w"> </span><span class="sd">"""Remove properties.</span> |
| <a id="__codelineno-0-934" name="__codelineno-0-934"></a> |
| <a id="__codelineno-0-935" name="__codelineno-0-935"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-936" name="__codelineno-0-936"></a><span class="sd"> removals: Properties to be removed.</span> |
| <a id="__codelineno-0-937" name="__codelineno-0-937"></a> |
| <a id="__codelineno-0-938" name="__codelineno-0-938"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-939" name="__codelineno-0-939"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-940" name="__codelineno-0-940"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-941" name="__codelineno-0-941"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">((</span><span class="n">RemovePropertiesUpdate</span><span class="p">(</span><span class="n">removals</span><span class="o">=</span><span class="n">removals</span><span class="p">),))</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.set_properties" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">set_properties</span><span class="p">(</span><span class="n">properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.set_properties" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Set properties.</p> |
| <p>When a property is already set, it will be overwritten.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>properties</code> |
| </td> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Properties = Dict[str, Any] |
| |
| |
| module-attribute |
| (pyiceberg.typedef.Properties)" href="../typedef/#pyiceberg.typedef.Properties">Properties</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The properties set on the table.</p> |
| </div> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.EMPTY_DICT">EMPTY_DICT</span></code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>kwargs</code> |
| </td> |
| <td> |
| <code><span title="typing.Any">Any</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>properties can also be pass as kwargs.</p> |
| </div> |
| </td> |
| <td> |
| <code>{}</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Transaction (pyiceberg.table.Transaction)" href="#pyiceberg.table.Transaction">Transaction</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The alter table builder.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-333">333</a></span> |
| <span class="normal"><a href="#__codelineno-0-334">334</a></span> |
| <span class="normal"><a href="#__codelineno-0-335">335</a></span> |
| <span class="normal"><a href="#__codelineno-0-336">336</a></span> |
| <span class="normal"><a href="#__codelineno-0-337">337</a></span> |
| <span class="normal"><a href="#__codelineno-0-338">338</a></span> |
| <span class="normal"><a href="#__codelineno-0-339">339</a></span> |
| <span class="normal"><a href="#__codelineno-0-340">340</a></span> |
| <span class="normal"><a href="#__codelineno-0-341">341</a></span> |
| <span class="normal"><a href="#__codelineno-0-342">342</a></span> |
| <span class="normal"><a href="#__codelineno-0-343">343</a></span> |
| <span class="normal"><a href="#__codelineno-0-344">344</a></span> |
| <span class="normal"><a href="#__codelineno-0-345">345</a></span> |
| <span class="normal"><a href="#__codelineno-0-346">346</a></span> |
| <span class="normal"><a href="#__codelineno-0-347">347</a></span> |
| <span class="normal"><a href="#__codelineno-0-348">348</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-333" name="__codelineno-0-333"></a><span class="k">def</span><span class="w"> </span><span class="nf">set_properties</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">properties</span><span class="p">:</span> <span class="n">Properties</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-334" name="__codelineno-0-334"></a><span class="w"> </span><span class="sd">"""Set properties.</span> |
| <a id="__codelineno-0-335" name="__codelineno-0-335"></a> |
| <a id="__codelineno-0-336" name="__codelineno-0-336"></a><span class="sd"> When a property is already set, it will be overwritten.</span> |
| <a id="__codelineno-0-337" name="__codelineno-0-337"></a> |
| <a id="__codelineno-0-338" name="__codelineno-0-338"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-339" name="__codelineno-0-339"></a><span class="sd"> properties: The properties set on the table.</span> |
| <a id="__codelineno-0-340" name="__codelineno-0-340"></a><span class="sd"> kwargs: properties can also be pass as kwargs.</span> |
| <a id="__codelineno-0-341" name="__codelineno-0-341"></a> |
| <a id="__codelineno-0-342" name="__codelineno-0-342"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-343" name="__codelineno-0-343"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-344" name="__codelineno-0-344"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-345" name="__codelineno-0-345"></a> <span class="k">if</span> <span class="n">properties</span> <span class="ow">and</span> <span class="n">kwargs</span><span class="p">:</span> |
| <a id="__codelineno-0-346" name="__codelineno-0-346"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Cannot pass both properties and kwargs"</span><span class="p">)</span> |
| <a id="__codelineno-0-347" name="__codelineno-0-347"></a> <span class="n">updates</span> <span class="o">=</span> <span class="n">properties</span> <span class="ow">or</span> <span class="n">kwargs</span> |
| <a id="__codelineno-0-348" name="__codelineno-0-348"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">((</span><span class="n">SetPropertiesUpdate</span><span class="p">(</span><span class="n">updates</span><span class="o">=</span><span class="n">updates</span><span class="p">),))</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.update_location" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_location</span><span class="p">(</span><span class="n">location</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.update_location" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Set the new table location.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>location</code> |
| </td> |
| <td> |
| <code><span title="str">str</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The new location of the table.</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Transaction (pyiceberg.table.Transaction)" href="#pyiceberg.table.Transaction">Transaction</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The alter table builder.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-943">943</a></span> |
| <span class="normal"><a href="#__codelineno-0-944">944</a></span> |
| <span class="normal"><a href="#__codelineno-0-945">945</a></span> |
| <span class="normal"><a href="#__codelineno-0-946">946</a></span> |
| <span class="normal"><a href="#__codelineno-0-947">947</a></span> |
| <span class="normal"><a href="#__codelineno-0-948">948</a></span> |
| <span class="normal"><a href="#__codelineno-0-949">949</a></span> |
| <span class="normal"><a href="#__codelineno-0-950">950</a></span> |
| <span class="normal"><a href="#__codelineno-0-951">951</a></span> |
| <span class="normal"><a href="#__codelineno-0-952">952</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-943" name="__codelineno-0-943"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_location</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">location</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-944" name="__codelineno-0-944"></a><span class="w"> </span><span class="sd">"""Set the new table location.</span> |
| <a id="__codelineno-0-945" name="__codelineno-0-945"></a> |
| <a id="__codelineno-0-946" name="__codelineno-0-946"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-947" name="__codelineno-0-947"></a><span class="sd"> location: The new location of the table.</span> |
| <a id="__codelineno-0-948" name="__codelineno-0-948"></a> |
| <a id="__codelineno-0-949" name="__codelineno-0-949"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-950" name="__codelineno-0-950"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-951" name="__codelineno-0-951"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-952" name="__codelineno-0-952"></a> <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"Not yet implemented"</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.update_schema" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_schema</span><span class="p">(</span><span class="n">allow_incompatible_changes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.update_schema" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateSchema to alter the columns of this table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>allow_incompatible_changes</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If changes are allowed that might break downstream consumers.</p> |
| </div> |
| </td> |
| <td> |
| <code>False</code> |
| </td> |
| </tr> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If field names are case-sensitive.</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpdateSchema (pyiceberg.table.update.schema.UpdateSchema)" href="update/schema/#pyiceberg.table.update.schema.UpdateSchema">UpdateSchema</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateSchema.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-427">427</a></span> |
| <span class="normal"><a href="#__codelineno-0-428">428</a></span> |
| <span class="normal"><a href="#__codelineno-0-429">429</a></span> |
| <span class="normal"><a href="#__codelineno-0-430">430</a></span> |
| <span class="normal"><a href="#__codelineno-0-431">431</a></span> |
| <span class="normal"><a href="#__codelineno-0-432">432</a></span> |
| <span class="normal"><a href="#__codelineno-0-433">433</a></span> |
| <span class="normal"><a href="#__codelineno-0-434">434</a></span> |
| <span class="normal"><a href="#__codelineno-0-435">435</a></span> |
| <span class="normal"><a href="#__codelineno-0-436">436</a></span> |
| <span class="normal"><a href="#__codelineno-0-437">437</a></span> |
| <span class="normal"><a href="#__codelineno-0-438">438</a></span> |
| <span class="normal"><a href="#__codelineno-0-439">439</a></span> |
| <span class="normal"><a href="#__codelineno-0-440">440</a></span> |
| <span class="normal"><a href="#__codelineno-0-441">441</a></span> |
| <span class="normal"><a href="#__codelineno-0-442">442</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-427" name="__codelineno-0-427"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">allow_incompatible_changes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSchema</span><span class="p">:</span> |
| <a id="__codelineno-0-428" name="__codelineno-0-428"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSchema to alter the columns of this table.</span> |
| <a id="__codelineno-0-429" name="__codelineno-0-429"></a> |
| <a id="__codelineno-0-430" name="__codelineno-0-430"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-431" name="__codelineno-0-431"></a><span class="sd"> allow_incompatible_changes: If changes are allowed that might break downstream consumers.</span> |
| <a id="__codelineno-0-432" name="__codelineno-0-432"></a><span class="sd"> case_sensitive: If field names are case-sensitive.</span> |
| <a id="__codelineno-0-433" name="__codelineno-0-433"></a> |
| <a id="__codelineno-0-434" name="__codelineno-0-434"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-435" name="__codelineno-0-435"></a><span class="sd"> A new UpdateSchema.</span> |
| <a id="__codelineno-0-436" name="__codelineno-0-436"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-437" name="__codelineno-0-437"></a> <span class="k">return</span> <span class="n">UpdateSchema</span><span class="p">(</span> |
| <a id="__codelineno-0-438" name="__codelineno-0-438"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-439" name="__codelineno-0-439"></a> <span class="n">allow_incompatible_changes</span><span class="o">=</span><span class="n">allow_incompatible_changes</span><span class="p">,</span> |
| <a id="__codelineno-0-440" name="__codelineno-0-440"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-441" name="__codelineno-0-441"></a> <span class="n">name_mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">name_mapping</span><span class="p">(),</span> |
| <a id="__codelineno-0-442" name="__codelineno-0-442"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.update_snapshot" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_snapshot</span><span class="p">(</span><span class="n">snapshot_properties</span><span class="o">=</span><span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.update_snapshot" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateSnapshot to produce a new snapshot for the table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="pyiceberg.table.update.snapshot.UpdateSnapshot">UpdateSnapshot</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateSnapshot</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-458">458</a></span> |
| <span class="normal"><a href="#__codelineno-0-459">459</a></span> |
| <span class="normal"><a href="#__codelineno-0-460">460</a></span> |
| <span class="normal"><a href="#__codelineno-0-461">461</a></span> |
| <span class="normal"><a href="#__codelineno-0-462">462</a></span> |
| <span class="normal"><a href="#__codelineno-0-463">463</a></span> |
| <span class="normal"><a href="#__codelineno-0-464">464</a></span> |
| <span class="normal"><a href="#__codelineno-0-465">465</a></span> |
| <span class="normal"><a href="#__codelineno-0-466">466</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-458" name="__codelineno-0-458"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_snapshot</span><span class="p">(</span> |
| <a id="__codelineno-0-459" name="__codelineno-0-459"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">EMPTY_DICT</span><span class="p">,</span> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span> |
| <a id="__codelineno-0-460" name="__codelineno-0-460"></a><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSnapshot</span><span class="p">:</span> |
| <a id="__codelineno-0-461" name="__codelineno-0-461"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSnapshot to produce a new snapshot for the table.</span> |
| <a id="__codelineno-0-462" name="__codelineno-0-462"></a> |
| <a id="__codelineno-0-463" name="__codelineno-0-463"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-464" name="__codelineno-0-464"></a><span class="sd"> A new UpdateSnapshot</span> |
| <a id="__codelineno-0-465" name="__codelineno-0-465"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-466" name="__codelineno-0-466"></a> <span class="k">return</span> <span class="n">UpdateSnapshot</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> <span class="n">snapshot_properties</span><span class="o">=</span><span class="n">snapshot_properties</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.update_sort_order" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_sort_order</span><span class="p">(</span><span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.update_sort_order" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateSortOrder to update the sort order of this table.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>case_sensitive</code> |
| </td> |
| <td> |
| <code><span title="bool">bool</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>If field names are case-sensitive.</p> |
| </div> |
| </td> |
| <td> |
| <code>True</code> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpdateSortOrder (pyiceberg.table.update.sorting.UpdateSortOrder)" href="update/sorting/#pyiceberg.table.update.sorting.UpdateSortOrder">UpdateSortOrder</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateSortOrder.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-444">444</a></span> |
| <span class="normal"><a href="#__codelineno-0-445">445</a></span> |
| <span class="normal"><a href="#__codelineno-0-446">446</a></span> |
| <span class="normal"><a href="#__codelineno-0-447">447</a></span> |
| <span class="normal"><a href="#__codelineno-0-448">448</a></span> |
| <span class="normal"><a href="#__codelineno-0-449">449</a></span> |
| <span class="normal"><a href="#__codelineno-0-450">450</a></span> |
| <span class="normal"><a href="#__codelineno-0-451">451</a></span> |
| <span class="normal"><a href="#__codelineno-0-452">452</a></span> |
| <span class="normal"><a href="#__codelineno-0-453">453</a></span> |
| <span class="normal"><a href="#__codelineno-0-454">454</a></span> |
| <span class="normal"><a href="#__codelineno-0-455">455</a></span> |
| <span class="normal"><a href="#__codelineno-0-456">456</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-444" name="__codelineno-0-444"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_sort_order</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSortOrder</span><span class="p">:</span> |
| <a id="__codelineno-0-445" name="__codelineno-0-445"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSortOrder to update the sort order of this table.</span> |
| <a id="__codelineno-0-446" name="__codelineno-0-446"></a> |
| <a id="__codelineno-0-447" name="__codelineno-0-447"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-448" name="__codelineno-0-448"></a><span class="sd"> case_sensitive: If field names are case-sensitive.</span> |
| <a id="__codelineno-0-449" name="__codelineno-0-449"></a> |
| <a id="__codelineno-0-450" name="__codelineno-0-450"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-451" name="__codelineno-0-451"></a><span class="sd"> A new UpdateSortOrder.</span> |
| <a id="__codelineno-0-452" name="__codelineno-0-452"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-453" name="__codelineno-0-453"></a> <span class="k">return</span> <span class="n">UpdateSortOrder</span><span class="p">(</span> |
| <a id="__codelineno-0-454" name="__codelineno-0-454"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-455" name="__codelineno-0-455"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-456" name="__codelineno-0-456"></a> <span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.update_spec" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_spec</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.update_spec" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateSpec to update the partitioning of the table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><span title="pyiceberg.table.update.spec.UpdateSpec">UpdateSpec</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateSpec.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-924">924</a></span> |
| <span class="normal"><a href="#__codelineno-0-925">925</a></span> |
| <span class="normal"><a href="#__codelineno-0-926">926</a></span> |
| <span class="normal"><a href="#__codelineno-0-927">927</a></span> |
| <span class="normal"><a href="#__codelineno-0-928">928</a></span> |
| <span class="normal"><a href="#__codelineno-0-929">929</a></span> |
| <span class="normal"><a href="#__codelineno-0-930">930</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-924" name="__codelineno-0-924"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateSpec</span><span class="p">:</span> |
| <a id="__codelineno-0-925" name="__codelineno-0-925"></a><span class="w"> </span><span class="sd">"""Create a new UpdateSpec to update the partitioning of the table.</span> |
| <a id="__codelineno-0-926" name="__codelineno-0-926"></a> |
| <a id="__codelineno-0-927" name="__codelineno-0-927"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-928" name="__codelineno-0-928"></a><span class="sd"> A new UpdateSpec.</span> |
| <a id="__codelineno-0-929" name="__codelineno-0-929"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-930" name="__codelineno-0-930"></a> <span class="k">return</span> <span class="n">UpdateSpec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.update_statistics" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">update_statistics</span><span class="p">()</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.update_statistics" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Create a new UpdateStatistics to update the statistics of the table.</p> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpdateStatistics (pyiceberg.table.update.statistics.UpdateStatistics)" href="update/statistics/#pyiceberg.table.update.statistics.UpdateStatistics">UpdateStatistics</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>A new UpdateStatistics</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-468">468</a></span> |
| <span class="normal"><a href="#__codelineno-0-469">469</a></span> |
| <span class="normal"><a href="#__codelineno-0-470">470</a></span> |
| <span class="normal"><a href="#__codelineno-0-471">471</a></span> |
| <span class="normal"><a href="#__codelineno-0-472">472</a></span> |
| <span class="normal"><a href="#__codelineno-0-473">473</a></span> |
| <span class="normal"><a href="#__codelineno-0-474">474</a></span> |
| <span class="normal"><a href="#__codelineno-0-475">475</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-468" name="__codelineno-0-468"></a><span class="k">def</span><span class="w"> </span><span class="nf">update_statistics</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">UpdateStatistics</span><span class="p">:</span> |
| <a id="__codelineno-0-469" name="__codelineno-0-469"></a><span class="w"> </span><span class="sd">"""</span> |
| <a id="__codelineno-0-470" name="__codelineno-0-470"></a><span class="sd"> Create a new UpdateStatistics to update the statistics of the table.</span> |
| <a id="__codelineno-0-471" name="__codelineno-0-471"></a> |
| <a id="__codelineno-0-472" name="__codelineno-0-472"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-473" name="__codelineno-0-473"></a><span class="sd"> A new UpdateStatistics</span> |
| <a id="__codelineno-0-474" name="__codelineno-0-474"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-475" name="__codelineno-0-475"></a> <span class="k">return</span> <span class="n">UpdateStatistics</span><span class="p">(</span><span class="n">transaction</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.upgrade_table_version" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">upgrade_table_version</span><span class="p">(</span><span class="n">format_version</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.upgrade_table_version" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Set the table to a certain version.</p> |
| |
| |
| <p><span class="doc-section-title">Parameters:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Name</th> |
| <th>Type</th> |
| <th>Description</th> |
| <th>Default</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code>format_version</code> |
| </td> |
| <td> |
| <code><span title="pyiceberg.typedef.TableVersion">TableVersion</span></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The newly set version.</p> |
| </div> |
| </td> |
| <td> |
| <em>required</em> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="Transaction (pyiceberg.table.Transaction)" href="#pyiceberg.table.Transaction">Transaction</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>The alter table builder.</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-313">313</a></span> |
| <span class="normal"><a href="#__codelineno-0-314">314</a></span> |
| <span class="normal"><a href="#__codelineno-0-315">315</a></span> |
| <span class="normal"><a href="#__codelineno-0-316">316</a></span> |
| <span class="normal"><a href="#__codelineno-0-317">317</a></span> |
| <span class="normal"><a href="#__codelineno-0-318">318</a></span> |
| <span class="normal"><a href="#__codelineno-0-319">319</a></span> |
| <span class="normal"><a href="#__codelineno-0-320">320</a></span> |
| <span class="normal"><a href="#__codelineno-0-321">321</a></span> |
| <span class="normal"><a href="#__codelineno-0-322">322</a></span> |
| <span class="normal"><a href="#__codelineno-0-323">323</a></span> |
| <span class="normal"><a href="#__codelineno-0-324">324</a></span> |
| <span class="normal"><a href="#__codelineno-0-325">325</a></span> |
| <span class="normal"><a href="#__codelineno-0-326">326</a></span> |
| <span class="normal"><a href="#__codelineno-0-327">327</a></span> |
| <span class="normal"><a href="#__codelineno-0-328">328</a></span> |
| <span class="normal"><a href="#__codelineno-0-329">329</a></span> |
| <span class="normal"><a href="#__codelineno-0-330">330</a></span> |
| <span class="normal"><a href="#__codelineno-0-331">331</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-313" name="__codelineno-0-313"></a><span class="k">def</span><span class="w"> </span><span class="nf">upgrade_table_version</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">format_version</span><span class="p">:</span> <span class="n">TableVersion</span><span class="p">)</span> <span class="o">-></span> <span class="n">Transaction</span><span class="p">:</span> |
| <a id="__codelineno-0-314" name="__codelineno-0-314"></a><span class="w"> </span><span class="sd">"""Set the table to a certain version.</span> |
| <a id="__codelineno-0-315" name="__codelineno-0-315"></a> |
| <a id="__codelineno-0-316" name="__codelineno-0-316"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="sd"> format_version: The newly set version.</span> |
| <a id="__codelineno-0-318" name="__codelineno-0-318"></a> |
| <a id="__codelineno-0-319" name="__codelineno-0-319"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd"> The alter table builder.</span> |
| <a id="__codelineno-0-321" name="__codelineno-0-321"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-322" name="__codelineno-0-322"></a> <span class="k">if</span> <span class="n">format_version</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">}:</span> |
| <a id="__codelineno-0-323" name="__codelineno-0-323"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unsupported table format version: </span><span class="si">{</span><span class="n">format_version</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-324" name="__codelineno-0-324"></a> |
| <a id="__codelineno-0-325" name="__codelineno-0-325"></a> <span class="k">if</span> <span class="n">format_version</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">:</span> |
| <a id="__codelineno-0-326" name="__codelineno-0-326"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Cannot downgrade v</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="si">}</span><span class="s2"> table to v</span><span class="si">{</span><span class="n">format_version</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-327" name="__codelineno-0-327"></a> |
| <a id="__codelineno-0-328" name="__codelineno-0-328"></a> <span class="k">if</span> <span class="n">format_version</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">:</span> |
| <a id="__codelineno-0-329" name="__codelineno-0-329"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">((</span><span class="n">UpgradeFormatVersionUpdate</span><span class="p">(</span><span class="n">format_version</span><span class="o">=</span><span class="n">format_version</span><span class="p">),))</span> |
| <a id="__codelineno-0-330" name="__codelineno-0-330"></a> |
| <a id="__codelineno-0-331" name="__codelineno-0-331"></a> <span class="k">return</span> <span class="bp">self</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-function"> |
| |
| |
| <h3 id="pyiceberg.table.Transaction.upsert" class="doc doc-heading"> |
| <code class="highlight language-python"><span class="n">upsert</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">join_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">when_matched_update_all</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">when_not_matched_insert_all</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">MAIN_BRANCH</span><span class="p">)</span></code> |
| |
| <a href="#pyiceberg.table.Transaction.upsert" class="headerlink" title="Permanent link">¶</a></h3> |
| |
| |
| <div class="doc doc-contents "> |
| |
| <p>Shorthand API for performing an upsert to an iceberg table.</p> |
| <p>Args:</p> |
| <div class="highlight"><pre><span></span><code>df: The input dataframe to upsert with the table's data. |
| join_cols: Columns to join on, if not provided, it will use the identifier-field-ids. |
| when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing |
| when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any existing rows in the table |
| case_sensitive: Bool indicating if the match should be case-sensitive |
| branch: Branch Reference to run the upsert operation |
| |
| To learn more about the identifier-field-ids: https://iceberg.apache.org/spec/#identifier-field-ids |
| |
| Example Use Cases: |
| Case 1: Both Parameters = True (Full Upsert) |
| Existing row found → Update it |
| New row found → Insert it |
| |
| Case 2: when_matched_update_all = False, when_not_matched_insert_all = True |
| Existing row found → Do nothing (no updates) |
| New row found → Insert it |
| |
| Case 3: when_matched_update_all = True, when_not_matched_insert_all = False |
| Existing row found → Update it |
| New row found → Do nothing (no inserts) |
| |
| Case 4: Both Parameters = False (No Merge Effect) |
| Existing row found → Do nothing |
| New row found → Do nothing |
| (Function effectively does nothing) |
| </code></pre></div> |
| |
| |
| <p><span class="doc-section-title">Returns:</span></p> |
| <table> |
| <thead> |
| <tr> |
| <th>Type</th> |
| <th>Description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="doc-section-item"> |
| <td> |
| <code><a class="autorefs autorefs-internal" title="UpsertResult |
| |
| |
| |
| dataclass |
| (pyiceberg.table.UpsertResult)" href="#pyiceberg.table.UpsertResult">UpsertResult</a></code> |
| </td> |
| <td> |
| <div class="doc-md-description"> |
| <p>An UpsertResult class (contains details of rows updated and inserted)</p> |
| </div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-739">739</a></span> |
| <span class="normal"><a href="#__codelineno-0-740">740</a></span> |
| <span class="normal"><a href="#__codelineno-0-741">741</a></span> |
| <span class="normal"><a href="#__codelineno-0-742">742</a></span> |
| <span class="normal"><a href="#__codelineno-0-743">743</a></span> |
| <span class="normal"><a href="#__codelineno-0-744">744</a></span> |
| <span class="normal"><a href="#__codelineno-0-745">745</a></span> |
| <span class="normal"><a href="#__codelineno-0-746">746</a></span> |
| <span class="normal"><a href="#__codelineno-0-747">747</a></span> |
| <span class="normal"><a href="#__codelineno-0-748">748</a></span> |
| <span class="normal"><a href="#__codelineno-0-749">749</a></span> |
| <span class="normal"><a href="#__codelineno-0-750">750</a></span> |
| <span class="normal"><a href="#__codelineno-0-751">751</a></span> |
| <span class="normal"><a href="#__codelineno-0-752">752</a></span> |
| <span class="normal"><a href="#__codelineno-0-753">753</a></span> |
| <span class="normal"><a href="#__codelineno-0-754">754</a></span> |
| <span class="normal"><a href="#__codelineno-0-755">755</a></span> |
| <span class="normal"><a href="#__codelineno-0-756">756</a></span> |
| <span class="normal"><a href="#__codelineno-0-757">757</a></span> |
| <span class="normal"><a href="#__codelineno-0-758">758</a></span> |
| <span class="normal"><a href="#__codelineno-0-759">759</a></span> |
| <span class="normal"><a href="#__codelineno-0-760">760</a></span> |
| <span class="normal"><a href="#__codelineno-0-761">761</a></span> |
| <span class="normal"><a href="#__codelineno-0-762">762</a></span> |
| <span class="normal"><a href="#__codelineno-0-763">763</a></span> |
| <span class="normal"><a href="#__codelineno-0-764">764</a></span> |
| <span class="normal"><a href="#__codelineno-0-765">765</a></span> |
| <span class="normal"><a href="#__codelineno-0-766">766</a></span> |
| <span class="normal"><a href="#__codelineno-0-767">767</a></span> |
| <span class="normal"><a href="#__codelineno-0-768">768</a></span> |
| <span class="normal"><a href="#__codelineno-0-769">769</a></span> |
| <span class="normal"><a href="#__codelineno-0-770">770</a></span> |
| <span class="normal"><a href="#__codelineno-0-771">771</a></span> |
| <span class="normal"><a href="#__codelineno-0-772">772</a></span> |
| <span class="normal"><a href="#__codelineno-0-773">773</a></span> |
| <span class="normal"><a href="#__codelineno-0-774">774</a></span> |
| <span class="normal"><a href="#__codelineno-0-775">775</a></span> |
| <span class="normal"><a href="#__codelineno-0-776">776</a></span> |
| <span class="normal"><a href="#__codelineno-0-777">777</a></span> |
| <span class="normal"><a href="#__codelineno-0-778">778</a></span> |
| <span class="normal"><a href="#__codelineno-0-779">779</a></span> |
| <span class="normal"><a href="#__codelineno-0-780">780</a></span> |
| <span class="normal"><a href="#__codelineno-0-781">781</a></span> |
| <span class="normal"><a href="#__codelineno-0-782">782</a></span> |
| <span class="normal"><a href="#__codelineno-0-783">783</a></span> |
| <span class="normal"><a href="#__codelineno-0-784">784</a></span> |
| <span class="normal"><a href="#__codelineno-0-785">785</a></span> |
| <span class="normal"><a href="#__codelineno-0-786">786</a></span> |
| <span class="normal"><a href="#__codelineno-0-787">787</a></span> |
| <span class="normal"><a href="#__codelineno-0-788">788</a></span> |
| <span class="normal"><a href="#__codelineno-0-789">789</a></span> |
| <span class="normal"><a href="#__codelineno-0-790">790</a></span> |
| <span class="normal"><a href="#__codelineno-0-791">791</a></span> |
| <span class="normal"><a href="#__codelineno-0-792">792</a></span> |
| <span class="normal"><a href="#__codelineno-0-793">793</a></span> |
| <span class="normal"><a href="#__codelineno-0-794">794</a></span> |
| <span class="normal"><a href="#__codelineno-0-795">795</a></span> |
| <span class="normal"><a href="#__codelineno-0-796">796</a></span> |
| <span class="normal"><a href="#__codelineno-0-797">797</a></span> |
| <span class="normal"><a href="#__codelineno-0-798">798</a></span> |
| <span class="normal"><a href="#__codelineno-0-799">799</a></span> |
| <span class="normal"><a href="#__codelineno-0-800">800</a></span> |
| <span class="normal"><a href="#__codelineno-0-801">801</a></span> |
| <span class="normal"><a href="#__codelineno-0-802">802</a></span> |
| <span class="normal"><a href="#__codelineno-0-803">803</a></span> |
| <span class="normal"><a href="#__codelineno-0-804">804</a></span> |
| <span class="normal"><a href="#__codelineno-0-805">805</a></span> |
| <span class="normal"><a href="#__codelineno-0-806">806</a></span> |
| <span class="normal"><a href="#__codelineno-0-807">807</a></span> |
| <span class="normal"><a href="#__codelineno-0-808">808</a></span> |
| <span class="normal"><a href="#__codelineno-0-809">809</a></span> |
| <span class="normal"><a href="#__codelineno-0-810">810</a></span> |
| <span class="normal"><a href="#__codelineno-0-811">811</a></span> |
| <span class="normal"><a href="#__codelineno-0-812">812</a></span> |
| <span class="normal"><a href="#__codelineno-0-813">813</a></span> |
| <span class="normal"><a href="#__codelineno-0-814">814</a></span> |
| <span class="normal"><a href="#__codelineno-0-815">815</a></span> |
| <span class="normal"><a href="#__codelineno-0-816">816</a></span> |
| <span class="normal"><a href="#__codelineno-0-817">817</a></span> |
| <span class="normal"><a href="#__codelineno-0-818">818</a></span> |
| <span class="normal"><a href="#__codelineno-0-819">819</a></span> |
| <span class="normal"><a href="#__codelineno-0-820">820</a></span> |
| <span class="normal"><a href="#__codelineno-0-821">821</a></span> |
| <span class="normal"><a href="#__codelineno-0-822">822</a></span> |
| <span class="normal"><a href="#__codelineno-0-823">823</a></span> |
| <span class="normal"><a href="#__codelineno-0-824">824</a></span> |
| <span class="normal"><a href="#__codelineno-0-825">825</a></span> |
| <span class="normal"><a href="#__codelineno-0-826">826</a></span> |
| <span class="normal"><a href="#__codelineno-0-827">827</a></span> |
| <span class="normal"><a href="#__codelineno-0-828">828</a></span> |
| <span class="normal"><a href="#__codelineno-0-829">829</a></span> |
| <span class="normal"><a href="#__codelineno-0-830">830</a></span> |
| <span class="normal"><a href="#__codelineno-0-831">831</a></span> |
| <span class="normal"><a href="#__codelineno-0-832">832</a></span> |
| <span class="normal"><a href="#__codelineno-0-833">833</a></span> |
| <span class="normal"><a href="#__codelineno-0-834">834</a></span> |
| <span class="normal"><a href="#__codelineno-0-835">835</a></span> |
| <span class="normal"><a href="#__codelineno-0-836">836</a></span> |
| <span class="normal"><a href="#__codelineno-0-837">837</a></span> |
| <span class="normal"><a href="#__codelineno-0-838">838</a></span> |
| <span class="normal"><a href="#__codelineno-0-839">839</a></span> |
| <span class="normal"><a href="#__codelineno-0-840">840</a></span> |
| <span class="normal"><a href="#__codelineno-0-841">841</a></span> |
| <span class="normal"><a href="#__codelineno-0-842">842</a></span> |
| <span class="normal"><a href="#__codelineno-0-843">843</a></span> |
| <span class="normal"><a href="#__codelineno-0-844">844</a></span> |
| <span class="normal"><a href="#__codelineno-0-845">845</a></span> |
| <span class="normal"><a href="#__codelineno-0-846">846</a></span> |
| <span class="normal"><a href="#__codelineno-0-847">847</a></span> |
| <span class="normal"><a href="#__codelineno-0-848">848</a></span> |
| <span class="normal"><a href="#__codelineno-0-849">849</a></span> |
| <span class="normal"><a href="#__codelineno-0-850">850</a></span> |
| <span class="normal"><a href="#__codelineno-0-851">851</a></span> |
| <span class="normal"><a href="#__codelineno-0-852">852</a></span> |
| <span class="normal"><a href="#__codelineno-0-853">853</a></span> |
| <span class="normal"><a href="#__codelineno-0-854">854</a></span> |
| <span class="normal"><a href="#__codelineno-0-855">855</a></span> |
| <span class="normal"><a href="#__codelineno-0-856">856</a></span> |
| <span class="normal"><a href="#__codelineno-0-857">857</a></span> |
| <span class="normal"><a href="#__codelineno-0-858">858</a></span> |
| <span class="normal"><a href="#__codelineno-0-859">859</a></span> |
| <span class="normal"><a href="#__codelineno-0-860">860</a></span> |
| <span class="normal"><a href="#__codelineno-0-861">861</a></span> |
| <span class="normal"><a href="#__codelineno-0-862">862</a></span> |
| <span class="normal"><a href="#__codelineno-0-863">863</a></span> |
| <span class="normal"><a href="#__codelineno-0-864">864</a></span> |
| <span class="normal"><a href="#__codelineno-0-865">865</a></span> |
| <span class="normal"><a href="#__codelineno-0-866">866</a></span> |
| <span class="normal"><a href="#__codelineno-0-867">867</a></span> |
| <span class="normal"><a href="#__codelineno-0-868">868</a></span> |
| <span class="normal"><a href="#__codelineno-0-869">869</a></span> |
| <span class="normal"><a href="#__codelineno-0-870">870</a></span> |
| <span class="normal"><a href="#__codelineno-0-871">871</a></span> |
| <span class="normal"><a href="#__codelineno-0-872">872</a></span> |
| <span class="normal"><a href="#__codelineno-0-873">873</a></span> |
| <span class="normal"><a href="#__codelineno-0-874">874</a></span> |
| <span class="normal"><a href="#__codelineno-0-875">875</a></span> |
| <span class="normal"><a href="#__codelineno-0-876">876</a></span> |
| <span class="normal"><a href="#__codelineno-0-877">877</a></span> |
| <span class="normal"><a href="#__codelineno-0-878">878</a></span> |
| <span class="normal"><a href="#__codelineno-0-879">879</a></span> |
| <span class="normal"><a href="#__codelineno-0-880">880</a></span> |
| <span class="normal"><a href="#__codelineno-0-881">881</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-739" name="__codelineno-0-739"></a><span class="k">def</span><span class="w"> </span><span class="nf">upsert</span><span class="p">(</span> |
| <a id="__codelineno-0-740" name="__codelineno-0-740"></a> <span class="bp">self</span><span class="p">,</span> |
| <a id="__codelineno-0-741" name="__codelineno-0-741"></a> <span class="n">df</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">,</span> |
| <a id="__codelineno-0-742" name="__codelineno-0-742"></a> <span class="n">join_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <a id="__codelineno-0-743" name="__codelineno-0-743"></a> <span class="n">when_matched_update_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-744" name="__codelineno-0-744"></a> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-745" name="__codelineno-0-745"></a> <span class="n">case_sensitive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <a id="__codelineno-0-746" name="__codelineno-0-746"></a> <span class="n">branch</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">MAIN_BRANCH</span><span class="p">,</span> |
| <a id="__codelineno-0-747" name="__codelineno-0-747"></a><span class="p">)</span> <span class="o">-></span> <span class="n">UpsertResult</span><span class="p">:</span> |
| <a id="__codelineno-0-748" name="__codelineno-0-748"></a><span class="w"> </span><span class="sd">"""Shorthand API for performing an upsert to an iceberg table.</span> |
| <a id="__codelineno-0-749" name="__codelineno-0-749"></a> |
| <a id="__codelineno-0-750" name="__codelineno-0-750"></a><span class="sd"> Args:</span> |
| <a id="__codelineno-0-751" name="__codelineno-0-751"></a> |
| <a id="__codelineno-0-752" name="__codelineno-0-752"></a><span class="sd"> df: The input dataframe to upsert with the table's data.</span> |
| <a id="__codelineno-0-753" name="__codelineno-0-753"></a><span class="sd"> join_cols: Columns to join on, if not provided, it will use the identifier-field-ids.</span> |
| <a id="__codelineno-0-754" name="__codelineno-0-754"></a><span class="sd"> when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing</span> |
| <a id="__codelineno-0-755" name="__codelineno-0-755"></a><span class="sd"> when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any existing rows in the table</span> |
| <a id="__codelineno-0-756" name="__codelineno-0-756"></a><span class="sd"> case_sensitive: Bool indicating if the match should be case-sensitive</span> |
| <a id="__codelineno-0-757" name="__codelineno-0-757"></a><span class="sd"> branch: Branch Reference to run the upsert operation</span> |
| <a id="__codelineno-0-758" name="__codelineno-0-758"></a> |
| <a id="__codelineno-0-759" name="__codelineno-0-759"></a><span class="sd"> To learn more about the identifier-field-ids: https://iceberg.apache.org/spec/#identifier-field-ids</span> |
| <a id="__codelineno-0-760" name="__codelineno-0-760"></a> |
| <a id="__codelineno-0-761" name="__codelineno-0-761"></a><span class="sd"> Example Use Cases:</span> |
| <a id="__codelineno-0-762" name="__codelineno-0-762"></a><span class="sd"> Case 1: Both Parameters = True (Full Upsert)</span> |
| <a id="__codelineno-0-763" name="__codelineno-0-763"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-764" name="__codelineno-0-764"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-765" name="__codelineno-0-765"></a> |
| <a id="__codelineno-0-766" name="__codelineno-0-766"></a><span class="sd"> Case 2: when_matched_update_all = False, when_not_matched_insert_all = True</span> |
| <a id="__codelineno-0-767" name="__codelineno-0-767"></a><span class="sd"> Existing row found → Do nothing (no updates)</span> |
| <a id="__codelineno-0-768" name="__codelineno-0-768"></a><span class="sd"> New row found → Insert it</span> |
| <a id="__codelineno-0-769" name="__codelineno-0-769"></a> |
| <a id="__codelineno-0-770" name="__codelineno-0-770"></a><span class="sd"> Case 3: when_matched_update_all = True, when_not_matched_insert_all = False</span> |
| <a id="__codelineno-0-771" name="__codelineno-0-771"></a><span class="sd"> Existing row found → Update it</span> |
| <a id="__codelineno-0-772" name="__codelineno-0-772"></a><span class="sd"> New row found → Do nothing (no inserts)</span> |
| <a id="__codelineno-0-773" name="__codelineno-0-773"></a> |
| <a id="__codelineno-0-774" name="__codelineno-0-774"></a><span class="sd"> Case 4: Both Parameters = False (No Merge Effect)</span> |
| <a id="__codelineno-0-775" name="__codelineno-0-775"></a><span class="sd"> Existing row found → Do nothing</span> |
| <a id="__codelineno-0-776" name="__codelineno-0-776"></a><span class="sd"> New row found → Do nothing</span> |
| <a id="__codelineno-0-777" name="__codelineno-0-777"></a><span class="sd"> (Function effectively does nothing)</span> |
| <a id="__codelineno-0-778" name="__codelineno-0-778"></a> |
| <a id="__codelineno-0-779" name="__codelineno-0-779"></a> |
| <a id="__codelineno-0-780" name="__codelineno-0-780"></a><span class="sd"> Returns:</span> |
| <a id="__codelineno-0-781" name="__codelineno-0-781"></a><span class="sd"> An UpsertResult class (contains details of rows updated and inserted)</span> |
| <a id="__codelineno-0-782" name="__codelineno-0-782"></a><span class="sd"> """</span> |
| <a id="__codelineno-0-783" name="__codelineno-0-783"></a> <span class="k">try</span><span class="p">:</span> |
| <a id="__codelineno-0-784" name="__codelineno-0-784"></a> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> <span class="c1"># noqa: F401</span> |
| <a id="__codelineno-0-785" name="__codelineno-0-785"></a> <span class="k">except</span> <span class="ne">ModuleNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <a id="__codelineno-0-786" name="__codelineno-0-786"></a> <span class="k">raise</span> <span class="ne">ModuleNotFoundError</span><span class="p">(</span><span class="s2">"For writes PyArrow needs to be installed"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span> |
| <a id="__codelineno-0-787" name="__codelineno-0-787"></a> |
| <a id="__codelineno-0-788" name="__codelineno-0-788"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">expression_to_pyarrow</span> |
| <a id="__codelineno-0-789" name="__codelineno-0-789"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.table</span><span class="w"> </span><span class="kn">import</span> <span class="n">upsert_util</span> |
| <a id="__codelineno-0-790" name="__codelineno-0-790"></a> |
| <a id="__codelineno-0-791" name="__codelineno-0-791"></a> <span class="k">if</span> <span class="n">join_cols</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-792" name="__codelineno-0-792"></a> <span class="n">join_cols</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-793" name="__codelineno-0-793"></a> <span class="k">for</span> <span class="n">field_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">identifier_field_ids</span><span class="p">:</span> |
| <a id="__codelineno-0-794" name="__codelineno-0-794"></a> <span class="n">col</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">find_column_name</span><span class="p">(</span><span class="n">field_id</span><span class="p">)</span> |
| <a id="__codelineno-0-795" name="__codelineno-0-795"></a> <span class="k">if</span> <span class="n">col</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <a id="__codelineno-0-796" name="__codelineno-0-796"></a> <span class="n">join_cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col</span><span class="p">)</span> |
| <a id="__codelineno-0-797" name="__codelineno-0-797"></a> <span class="k">else</span><span class="p">:</span> |
| <a id="__codelineno-0-798" name="__codelineno-0-798"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Field-ID could not be found: </span><span class="si">{</span><span class="n">join_cols</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <a id="__codelineno-0-799" name="__codelineno-0-799"></a> |
| <a id="__codelineno-0-800" name="__codelineno-0-800"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">join_cols</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-801" name="__codelineno-0-801"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Join columns could not be found, please set identifier-field-ids or pass in explicitly."</span><span class="p">)</span> |
| <a id="__codelineno-0-802" name="__codelineno-0-802"></a> |
| <a id="__codelineno-0-803" name="__codelineno-0-803"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">when_matched_update_all</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> |
| <a id="__codelineno-0-804" name="__codelineno-0-804"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"no upsert options selected...exiting"</span><span class="p">)</span> |
| <a id="__codelineno-0-805" name="__codelineno-0-805"></a> |
| <a id="__codelineno-0-806" name="__codelineno-0-806"></a> <span class="k">if</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">has_duplicate_rows</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">):</span> |
| <a id="__codelineno-0-807" name="__codelineno-0-807"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Duplicate rows found in source dataset based on the key columns. No upsert executed"</span><span class="p">)</span> |
| <a id="__codelineno-0-808" name="__codelineno-0-808"></a> |
| <a id="__codelineno-0-809" name="__codelineno-0-809"></a> <span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.io.pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_check_pyarrow_schema_compatible</span> |
| <a id="__codelineno-0-810" name="__codelineno-0-810"></a> |
| <a id="__codelineno-0-811" name="__codelineno-0-811"></a> <span class="n">downcast_ns_timestamp_to_us</span> <span class="o">=</span> <span class="n">Config</span><span class="p">()</span><span class="o">.</span><span class="n">get_bool</span><span class="p">(</span><span class="n">DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">False</span> |
| <a id="__codelineno-0-812" name="__codelineno-0-812"></a> <span class="n">_check_pyarrow_schema_compatible</span><span class="p">(</span> |
| <a id="__codelineno-0-813" name="__codelineno-0-813"></a> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> |
| <a id="__codelineno-0-814" name="__codelineno-0-814"></a> <span class="n">provided_schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> |
| <a id="__codelineno-0-815" name="__codelineno-0-815"></a> <span class="n">downcast_ns_timestamp_to_us</span><span class="o">=</span><span class="n">downcast_ns_timestamp_to_us</span><span class="p">,</span> |
| <a id="__codelineno-0-816" name="__codelineno-0-816"></a> <span class="n">format_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">format_version</span><span class="p">,</span> |
| <a id="__codelineno-0-817" name="__codelineno-0-817"></a> <span class="p">)</span> |
| <a id="__codelineno-0-818" name="__codelineno-0-818"></a> |
| <a id="__codelineno-0-819" name="__codelineno-0-819"></a> <span class="c1"># get list of rows that exist so we don't have to load the entire target table</span> |
| <a id="__codelineno-0-820" name="__codelineno-0-820"></a> <span class="n">matched_predicate</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">create_match_filter</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-821" name="__codelineno-0-821"></a> |
| <a id="__codelineno-0-822" name="__codelineno-0-822"></a> <span class="c1"># We must use Transaction.table_metadata for the scan. This includes all uncommitted - but relevant - changes.</span> |
| <a id="__codelineno-0-823" name="__codelineno-0-823"></a> |
| <a id="__codelineno-0-824" name="__codelineno-0-824"></a> <span class="n">matched_iceberg_record_batches_scan</span> <span class="o">=</span> <span class="n">DataScan</span><span class="p">(</span> |
| <a id="__codelineno-0-825" name="__codelineno-0-825"></a> <span class="n">table_metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="p">,</span> |
| <a id="__codelineno-0-826" name="__codelineno-0-826"></a> <span class="n">io</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_table</span><span class="o">.</span><span class="n">io</span><span class="p">,</span> |
| <a id="__codelineno-0-827" name="__codelineno-0-827"></a> <span class="n">row_filter</span><span class="o">=</span><span class="n">matched_predicate</span><span class="p">,</span> |
| <a id="__codelineno-0-828" name="__codelineno-0-828"></a> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">,</span> |
| <a id="__codelineno-0-829" name="__codelineno-0-829"></a> <span class="p">)</span> |
| <a id="__codelineno-0-830" name="__codelineno-0-830"></a> |
| <a id="__codelineno-0-831" name="__codelineno-0-831"></a> <span class="k">if</span> <span class="n">branch</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">refs</span><span class="p">:</span> |
| <a id="__codelineno-0-832" name="__codelineno-0-832"></a> <span class="n">matched_iceberg_record_batches_scan</span> <span class="o">=</span> <span class="n">matched_iceberg_record_batches_scan</span><span class="o">.</span><span class="n">use_ref</span><span class="p">(</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-833" name="__codelineno-0-833"></a> |
| <a id="__codelineno-0-834" name="__codelineno-0-834"></a> <span class="n">matched_iceberg_record_batches</span> <span class="o">=</span> <span class="n">matched_iceberg_record_batches_scan</span><span class="o">.</span><span class="n">to_arrow_batch_reader</span><span class="p">()</span> |
| <a id="__codelineno-0-835" name="__codelineno-0-835"></a> |
| <a id="__codelineno-0-836" name="__codelineno-0-836"></a> <span class="n">batches_to_overwrite</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-837" name="__codelineno-0-837"></a> <span class="n">overwrite_predicates</span> <span class="o">=</span> <span class="p">[]</span> |
| <a id="__codelineno-0-838" name="__codelineno-0-838"></a> <span class="n">rows_to_insert</span> <span class="o">=</span> <span class="n">df</span> |
| <a id="__codelineno-0-839" name="__codelineno-0-839"></a> |
| <a id="__codelineno-0-840" name="__codelineno-0-840"></a> <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">matched_iceberg_record_batches</span><span class="p">:</span> |
| <a id="__codelineno-0-841" name="__codelineno-0-841"></a> <span class="n">rows</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="o">.</span><span class="n">from_batches</span><span class="p">([</span><span class="n">batch</span><span class="p">])</span> |
| <a id="__codelineno-0-842" name="__codelineno-0-842"></a> |
| <a id="__codelineno-0-843" name="__codelineno-0-843"></a> <span class="k">if</span> <span class="n">when_matched_update_all</span><span class="p">:</span> |
| <a id="__codelineno-0-844" name="__codelineno-0-844"></a> <span class="c1"># function get_rows_to_update is doing a check on non-key columns to see if any of the values have actually changed</span> |
| <a id="__codelineno-0-845" name="__codelineno-0-845"></a> <span class="c1"># we don't want to do just a blanket overwrite for matched rows if the actual non-key column data hasn't changed</span> |
| <a id="__codelineno-0-846" name="__codelineno-0-846"></a> <span class="c1"># this extra step avoids unnecessary IO and writes</span> |
| <a id="__codelineno-0-847" name="__codelineno-0-847"></a> <span class="n">rows_to_update</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">get_rows_to_update</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-848" name="__codelineno-0-848"></a> |
| <a id="__codelineno-0-849" name="__codelineno-0-849"></a> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> |
| <a id="__codelineno-0-850" name="__codelineno-0-850"></a> <span class="c1"># build the match predicate filter</span> |
| <a id="__codelineno-0-851" name="__codelineno-0-851"></a> <span class="n">overwrite_mask_predicate</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">create_match_filter</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-852" name="__codelineno-0-852"></a> |
| <a id="__codelineno-0-853" name="__codelineno-0-853"></a> <span class="n">batches_to_overwrite</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">)</span> |
| <a id="__codelineno-0-854" name="__codelineno-0-854"></a> <span class="n">overwrite_predicates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">overwrite_mask_predicate</span><span class="p">)</span> |
| <a id="__codelineno-0-855" name="__codelineno-0-855"></a> |
| <a id="__codelineno-0-856" name="__codelineno-0-856"></a> <span class="k">if</span> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> |
| <a id="__codelineno-0-857" name="__codelineno-0-857"></a> <span class="n">expr_match</span> <span class="o">=</span> <span class="n">upsert_util</span><span class="o">.</span><span class="n">create_match_filter</span><span class="p">(</span><span class="n">rows</span><span class="p">,</span> <span class="n">join_cols</span><span class="p">)</span> |
| <a id="__codelineno-0-858" name="__codelineno-0-858"></a> <span class="n">expr_match_bound</span> <span class="o">=</span> <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_metadata</span><span class="o">.</span><span class="n">schema</span><span class="p">(),</span> <span class="n">expr_match</span><span class="p">,</span> <span class="n">case_sensitive</span><span class="o">=</span><span class="n">case_sensitive</span><span class="p">)</span> |
| <a id="__codelineno-0-859" name="__codelineno-0-859"></a> <span class="n">expr_match_arrow</span> <span class="o">=</span> <span class="n">expression_to_pyarrow</span><span class="p">(</span><span class="n">expr_match_bound</span><span class="p">)</span> |
| <a id="__codelineno-0-860" name="__codelineno-0-860"></a> |
| <a id="__codelineno-0-861" name="__codelineno-0-861"></a> <span class="c1"># Filter rows per batch.</span> |
| <a id="__codelineno-0-862" name="__codelineno-0-862"></a> <span class="n">rows_to_insert</span> <span class="o">=</span> <span class="n">rows_to_insert</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">expr_match_arrow</span><span class="p">)</span> |
| <a id="__codelineno-0-863" name="__codelineno-0-863"></a> |
| <a id="__codelineno-0-864" name="__codelineno-0-864"></a> <span class="n">update_row_cnt</span> <span class="o">=</span> <span class="mi">0</span> |
| <a id="__codelineno-0-865" name="__codelineno-0-865"></a> <span class="n">insert_row_cnt</span> <span class="o">=</span> <span class="mi">0</span> |
| <a id="__codelineno-0-866" name="__codelineno-0-866"></a> |
| <a id="__codelineno-0-867" name="__codelineno-0-867"></a> <span class="k">if</span> <span class="n">batches_to_overwrite</span><span class="p">:</span> |
| <a id="__codelineno-0-868" name="__codelineno-0-868"></a> <span class="n">rows_to_update</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">concat_tables</span><span class="p">(</span><span class="n">batches_to_overwrite</span><span class="p">)</span> |
| <a id="__codelineno-0-869" name="__codelineno-0-869"></a> <span class="n">update_row_cnt</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">rows_to_update</span><span class="p">)</span> |
| <a id="__codelineno-0-870" name="__codelineno-0-870"></a> <span class="bp">self</span><span class="o">.</span><span class="n">overwrite</span><span class="p">(</span> |
| <a id="__codelineno-0-871" name="__codelineno-0-871"></a> <span class="n">rows_to_update</span><span class="p">,</span> |
| <a id="__codelineno-0-872" name="__codelineno-0-872"></a> <span class="n">overwrite_filter</span><span class="o">=</span><span class="n">Or</span><span class="p">(</span><span class="o">*</span><span class="n">overwrite_predicates</span><span class="p">)</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">overwrite_predicates</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span> <span class="k">else</span> <span class="n">overwrite_predicates</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> |
| <a id="__codelineno-0-873" name="__codelineno-0-873"></a> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">,</span> |
| <a id="__codelineno-0-874" name="__codelineno-0-874"></a> <span class="p">)</span> |
| <a id="__codelineno-0-875" name="__codelineno-0-875"></a> |
| <a id="__codelineno-0-876" name="__codelineno-0-876"></a> <span class="k">if</span> <span class="n">when_not_matched_insert_all</span><span class="p">:</span> |
| <a id="__codelineno-0-877" name="__codelineno-0-877"></a> <span class="n">insert_row_cnt</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">rows_to_insert</span><span class="p">)</span> |
| <a id="__codelineno-0-878" name="__codelineno-0-878"></a> <span class="k">if</span> <span class="n">rows_to_insert</span><span class="p">:</span> |
| <a id="__codelineno-0-879" name="__codelineno-0-879"></a> <span class="bp">self</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">rows_to_insert</span><span class="p">,</span> <span class="n">branch</span><span class="o">=</span><span class="n">branch</span><span class="p">)</span> |
| <a id="__codelineno-0-880" name="__codelineno-0-880"></a> |
| <a id="__codelineno-0-881" name="__codelineno-0-881"></a> <span class="k">return</span> <span class="n">UpsertResult</span><span class="p">(</span><span class="n">rows_updated</span><span class="o">=</span><span class="n">update_row_cnt</span><span class="p">,</span> <span class="n">rows_inserted</span><span class="o">=</span><span class="n">insert_row_cnt</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| </div> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.UpsertResult" class="doc doc-heading"> |
| <code>UpsertResult</code> |
| |
| |
| <span class="doc doc-labels"> |
| <small class="doc doc-label doc-label-dataclass"><code>dataclass</code></small> |
| </span> |
| |
| <a href="#pyiceberg.table.UpsertResult" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| |
| |
| <p>Summary the upsert operation.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-161">161</a></span> |
| <span class="normal"><a href="#__codelineno-0-162">162</a></span> |
| <span class="normal"><a href="#__codelineno-0-163">163</a></span> |
| <span class="normal"><a href="#__codelineno-0-164">164</a></span> |
| <span class="normal"><a href="#__codelineno-0-165">165</a></span> |
| <span class="normal"><a href="#__codelineno-0-166">166</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-161" name="__codelineno-0-161"></a><span class="nd">@dataclass</span><span class="p">()</span> |
| <a id="__codelineno-0-162" name="__codelineno-0-162"></a><span class="k">class</span><span class="w"> </span><span class="nc">UpsertResult</span><span class="p">:</span> |
| <a id="__codelineno-0-163" name="__codelineno-0-163"></a><span class="w"> </span><span class="sd">"""Summary the upsert operation."""</span> |
| <a id="__codelineno-0-164" name="__codelineno-0-164"></a> |
| <a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="n">rows_updated</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span> |
| <a id="__codelineno-0-166" name="__codelineno-0-166"></a> <span class="n">rows_inserted</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="doc doc-object doc-class"> |
| |
| |
| |
| <h2 id="pyiceberg.table.WriteTask" class="doc doc-heading"> |
| <code>WriteTask</code> |
| |
| |
| <span class="doc doc-labels"> |
| <small class="doc doc-label doc-label-dataclass"><code>dataclass</code></small> |
| </span> |
| |
| <a href="#pyiceberg.table.WriteTask" class="headerlink" title="Permanent link">¶</a></h2> |
| |
| |
| <div class="doc doc-contents "> |
| |
| |
| <p>Task with the parameters for writing a DataFile.</p> |
| |
| |
| |
| |
| |
| |
| |
| |
| <details class="quote"> |
| <summary>Source code in <code>pyiceberg/table/__init__.py</code></summary> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-2130">2130</a></span> |
| <span class="normal"><a href="#__codelineno-0-2131">2131</a></span> |
| <span class="normal"><a href="#__codelineno-0-2132">2132</a></span> |
| <span class="normal"><a href="#__codelineno-0-2133">2133</a></span> |
| <span class="normal"><a href="#__codelineno-0-2134">2134</a></span> |
| <span class="normal"><a href="#__codelineno-0-2135">2135</a></span> |
| <span class="normal"><a href="#__codelineno-0-2136">2136</a></span> |
| <span class="normal"><a href="#__codelineno-0-2137">2137</a></span> |
| <span class="normal"><a href="#__codelineno-0-2138">2138</a></span> |
| <span class="normal"><a href="#__codelineno-0-2139">2139</a></span> |
| <span class="normal"><a href="#__codelineno-0-2140">2140</a></span> |
| <span class="normal"><a href="#__codelineno-0-2141">2141</a></span> |
| <span class="normal"><a href="#__codelineno-0-2142">2142</a></span> |
| <span class="normal"><a href="#__codelineno-0-2143">2143</a></span> |
| <span class="normal"><a href="#__codelineno-0-2144">2144</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-2130" name="__codelineno-0-2130"></a><span class="nd">@dataclass</span><span class="p">(</span><span class="n">frozen</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| <a id="__codelineno-0-2131" name="__codelineno-0-2131"></a><span class="k">class</span><span class="w"> </span><span class="nc">WriteTask</span><span class="p">:</span> |
| <a id="__codelineno-0-2132" name="__codelineno-0-2132"></a><span class="w"> </span><span class="sd">"""Task with the parameters for writing a DataFile."""</span> |
| <a id="__codelineno-0-2133" name="__codelineno-0-2133"></a> |
| <a id="__codelineno-0-2134" name="__codelineno-0-2134"></a> <span class="n">write_uuid</span><span class="p">:</span> <span class="n">uuid</span><span class="o">.</span><span class="n">UUID</span> |
| <a id="__codelineno-0-2135" name="__codelineno-0-2135"></a> <span class="n">task_id</span><span class="p">:</span> <span class="nb">int</span> |
| <a id="__codelineno-0-2136" name="__codelineno-0-2136"></a> <span class="n">schema</span><span class="p">:</span> <span class="n">Schema</span> |
| <a id="__codelineno-0-2137" name="__codelineno-0-2137"></a> <span class="n">record_batches</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="p">]</span> |
| <a id="__codelineno-0-2138" name="__codelineno-0-2138"></a> <span class="n">sort_order_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span> |
| <a id="__codelineno-0-2139" name="__codelineno-0-2139"></a> <span class="n">partition_key</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">PartitionKey</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span> |
| <a id="__codelineno-0-2140" name="__codelineno-0-2140"></a> |
| <a id="__codelineno-0-2141" name="__codelineno-0-2141"></a> <span class="k">def</span><span class="w"> </span><span class="nf">generate_data_file_filename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">extension</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <a id="__codelineno-0-2142" name="__codelineno-0-2142"></a> <span class="c1"># Mimics the behavior in the Java API:</span> |
| <a id="__codelineno-0-2143" name="__codelineno-0-2143"></a> <span class="c1"># https://github.com/apache/iceberg/blob/a582968975dd30ff4917fbbe999f1be903efac02/core/src/main/java/org/apache/iceberg/io/OutputFileFactory.java#L92-L101</span> |
| <a id="__codelineno-0-2144" name="__codelineno-0-2144"></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"00000-</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">write_uuid</span><span class="si">}</span><span class="s2">.</span><span class="si">{</span><span class="n">extension</span><span class="si">}</span><span class="s2">"</span> |
| </code></pre></div></td></tr></table></div> |
| </details> |
| |
| |
| |
| <div class="doc doc-children"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| |
| |
| |
| </div> |
| |
| </div> |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </article> |
| </div> |
| |
| |
| <script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script> |
| </div> |
| |
| <button type="button" class="md-top md-icon" data-md-component="top" hidden> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg> |
| Back to top |
| </button> |
| |
| </main> |
| |
| <footer class="md-footer"> |
| |
| <div class="md-footer-meta md-typeset"> |
| <div class="md-footer-meta__inner md-grid"> |
| <div class="md-copyright"> |
| |
| |
| Made with |
| <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener"> |
| Material for MkDocs |
| </a> |
| |
| </div> |
| |
| </div> |
| </div> |
| </footer> |
| |
| </div> |
| <div class="md-dialog" data-md-component="dialog"> |
| <div class="md-dialog__inner md-typeset"></div> |
| </div> |
| |
| |
| |
| |
| <script id="__config" type="application/json">{"base": "../../..", "features": ["navigation.top", "navigation.tracking", "navigation.tabs", "navigation.tabs.sticky", "content.code.copy"], "search": "../../../assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script> |
| |
| |
| <script src="../../../assets/javascripts/bundle.f55a23d4.min.js"></script> |
| |
| |
| </body> |
| </html> |
