Google Git
Sign in
apache/iceberg-python/refs/heads/gh-pages/./index.html
blob: f489bccad51351cbc039c5cd8d67fa56d3751b70 [file]
<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="canonical" href="https://py.iceberg.apache.org/">
<link rel="next" href="configuration/">
<link rel="icon" href="assets/images/iceberg-logo-icon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.1">
<title>PyIceberg</title>
<link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Lato:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Lato";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="assets/_mkdocstrings.css">
<script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["setDoNotTrack", true]);
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '82']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo -->
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#getting-started-with-pyiceberg" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="." title="PyIceberg" class="md-header__button md-logo" aria-label="PyIceberg" data-md-component="logo">
<img src="assets/images/iceberg-logo-icon.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
PyIceberg
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Getting started
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/apache/iceberg-python" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
apache/iceberg-python
</div>
</a>
</div>
</nav>
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item md-tabs__item--active">
<a href="." class="md-tabs__link">
Getting started
</a>
</li>
<li class="md-tabs__item">
<a href="configuration/" class="md-tabs__link">
Configuration
</a>
</li>
<li class="md-tabs__item">
<a href="cli/" class="md-tabs__link">
CLI
</a>
</li>
<li class="md-tabs__item">
<a href="api/" class="md-tabs__link">
API
</a>
</li>
<li class="md-tabs__item">
<a href="contributing/" class="md-tabs__link">
Contributing
</a>
</li>
<li class="md-tabs__item">
<a href="community/" class="md-tabs__link">
Community
</a>
</li>
<li class="md-tabs__item">
<a href="verify-release/" class="md-tabs__link">
Releases
</a>
</li>
<li class="md-tabs__item">
<a href="reference/pyiceberg/" class="md-tabs__link">
Code Reference
</a>
</li>
</ul>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" hidden>
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="." title="PyIceberg" class="md-nav__button md-logo" aria-label="PyIceberg" data-md-component="logo">
<img src="assets/images/iceberg-logo-icon.png" alt="logo">
</a>
PyIceberg
</label>
<div class="md-nav__source">
<a href="https://github.com/apache/iceberg-python" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
apache/iceberg-python
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Getting started
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="." class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Getting started
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#installation" class="md-nav__link">
<span class="md-ellipsis">
Installation
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#connecting-to-a-catalog" class="md-nav__link">
<span class="md-ellipsis">
Connecting to a catalog
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#write-a-pyarrow-dataframe" class="md-nav__link">
<span class="md-ellipsis">
Write a PyArrow dataframe
</span>
</a>
<nav class="md-nav" aria-label="Write a PyArrow dataframe">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#explore-iceberg-data-and-metadata-files" class="md-nav__link">
<span class="md-ellipsis">
Explore Iceberg data and metadata files
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#try-it-yourself-with-jupyter-notebooks" class="md-nav__link">
<span class="md-ellipsis">
Try it yourself with Jupyter Notebooks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#more-details" class="md-nav__link">
<span class="md-ellipsis">
More details
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="configuration/" class="md-nav__link">
<span class="md-ellipsis">
Configuration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="cli/" class="md-nav__link">
<span class="md-ellipsis">
CLI
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
<div class="md-nav__link md-nav__container">
<a href="api/" class="md-nav__link ">
<span class="md-ellipsis">
API
</span>
</a>
<label class="md-nav__link " for="__nav_4" id="__nav_4_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
API
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="row-filter-syntax/" class="md-nav__link">
<span class="md-ellipsis">
Row Filter Syntax
</span>
</a>
</li>
<li class="md-nav__item">
<a href="expression-dsl/" class="md-nav__link">
<span class="md-ellipsis">
Expression DSL
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="contributing/" class="md-nav__link">
<span class="md-ellipsis">
Contributing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="community/" class="md-nav__link">
<span class="md-ellipsis">
Community
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" >
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
<span class="md-ellipsis">
Releases
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Releases
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="verify-release/" class="md-nav__link">
<span class="md-ellipsis">
Verify a release
</span>
</a>
</li>
<li class="md-nav__item">
<a href="how-to-release/" class="md-nav__link">
<span class="md-ellipsis">
How to release
</span>
</a>
</li>
<li class="md-nav__item">
<a href="https://github.com/apache/iceberg-python/releases" class="md-nav__link">
<span class="md-ellipsis">
Release Notes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="nightly-build/" class="md-nav__link">
<span class="md-ellipsis">
Nightly Build
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
<span class="md-ellipsis">
Code Reference
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8">
<span class="md-nav__icon md-icon"></span>
Code Reference
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/" class="md-nav__link ">
<span class="md-ellipsis">
pyiceberg
</span>
</a>
<label class="md-nav__link " for="__nav_8_1" id="__nav_8_1_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_8_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1">
<span class="md-nav__icon md-icon"></span>
pyiceberg
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_1" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/avro/" class="md-nav__link ">
<span class="md-ellipsis">
avro
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_1" id="__nav_8_1_1_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_1">
<span class="md-nav__icon md-icon"></span>
avro
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_1_1" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/avro/codecs/" class="md-nav__link ">
<span class="md-ellipsis">
codecs
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_1_1" id="__nav_8_1_1_1_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="4" aria-labelledby="__nav_8_1_1_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_1_1">
<span class="md-nav__icon md-icon"></span>
codecs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/codecs/bzip2/" class="md-nav__link">
<span class="md-ellipsis">
bzip2
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/codecs/codec/" class="md-nav__link">
<span class="md-ellipsis">
codec
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/codecs/deflate/" class="md-nav__link">
<span class="md-ellipsis">
deflate
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/codecs/snappy_codec/" class="md-nav__link">
<span class="md-ellipsis">
snappy_codec
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/codecs/zstandard_codec/" class="md-nav__link">
<span class="md-ellipsis">
zstandard_codec
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/decoder/" class="md-nav__link">
<span class="md-ellipsis">
decoder
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/encoder/" class="md-nav__link">
<span class="md-ellipsis">
encoder
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/file/" class="md-nav__link">
<span class="md-ellipsis">
file
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/reader/" class="md-nav__link">
<span class="md-ellipsis">
reader
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/resolver/" class="md-nav__link">
<span class="md-ellipsis">
resolver
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/avro/writer/" class="md-nav__link">
<span class="md-ellipsis">
writer
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_2" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/catalog/" class="md-nav__link ">
<span class="md-ellipsis">
catalog
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_2" id="__nav_8_1_2_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_2">
<span class="md-nav__icon md-icon"></span>
catalog
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/bigquery_metastore/" class="md-nav__link">
<span class="md-ellipsis">
bigquery_metastore
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/dynamodb/" class="md-nav__link">
<span class="md-ellipsis">
dynamodb
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/glue/" class="md-nav__link">
<span class="md-ellipsis">
glue
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/hive/" class="md-nav__link">
<span class="md-ellipsis">
hive
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/memory/" class="md-nav__link">
<span class="md-ellipsis">
memory
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/noop/" class="md-nav__link">
<span class="md-ellipsis">
noop
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_2_7" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/catalog/rest/" class="md-nav__link ">
<span class="md-ellipsis">
rest
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_2_7" id="__nav_8_1_2_7_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="4" aria-labelledby="__nav_8_1_2_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_2_7">
<span class="md-nav__icon md-icon"></span>
rest
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/rest/auth/" class="md-nav__link">
<span class="md-ellipsis">
auth
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/rest/response/" class="md-nav__link">
<span class="md-ellipsis">
response
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/rest/scan_planning/" class="md-nav__link">
<span class="md-ellipsis">
scan_planning
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/catalog/sql/" class="md-nav__link">
<span class="md-ellipsis">
sql
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_3" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/cli/" class="md-nav__link ">
<span class="md-ellipsis">
cli
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_3" id="__nav_8_1_3_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_3">
<span class="md-nav__icon md-icon"></span>
cli
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/cli/console/" class="md-nav__link">
<span class="md-ellipsis">
console
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/cli/output/" class="md-nav__link">
<span class="md-ellipsis">
output
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/conversions/" class="md-nav__link">
<span class="md-ellipsis">
conversions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/exceptions/" class="md-nav__link">
<span class="md-ellipsis">
exceptions
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_6" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/expressions/" class="md-nav__link ">
<span class="md-ellipsis">
expressions
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_6" id="__nav_8_1_6_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_6">
<span class="md-nav__icon md-icon"></span>
expressions
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/expressions/literals/" class="md-nav__link">
<span class="md-ellipsis">
literals
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/expressions/parser/" class="md-nav__link">
<span class="md-ellipsis">
parser
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/expressions/visitors/" class="md-nav__link">
<span class="md-ellipsis">
visitors
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_7" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/io/" class="md-nav__link ">
<span class="md-ellipsis">
io
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_7" id="__nav_8_1_7_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_7">
<span class="md-nav__icon md-icon"></span>
io
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/io/fsspec/" class="md-nav__link">
<span class="md-ellipsis">
fsspec
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/io/pyarrow/" class="md-nav__link">
<span class="md-ellipsis">
pyarrow
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/manifest/" class="md-nav__link">
<span class="md-ellipsis">
manifest
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/partitioning/" class="md-nav__link">
<span class="md-ellipsis">
partitioning
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/schema/" class="md-nav__link">
<span class="md-ellipsis">
schema
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/serializers/" class="md-nav__link">
<span class="md-ellipsis">
serializers
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_12" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/table/" class="md-nav__link ">
<span class="md-ellipsis">
table
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_12" id="__nav_8_1_12_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_12">
<span class="md-nav__icon md-icon"></span>
table
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/delete_file_index/" class="md-nav__link">
<span class="md-ellipsis">
delete_file_index
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/inspect/" class="md-nav__link">
<span class="md-ellipsis">
inspect
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/locations/" class="md-nav__link">
<span class="md-ellipsis">
locations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/maintenance/" class="md-nav__link">
<span class="md-ellipsis">
maintenance
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/metadata/" class="md-nav__link">
<span class="md-ellipsis">
metadata
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/name_mapping/" class="md-nav__link">
<span class="md-ellipsis">
name_mapping
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/puffin/" class="md-nav__link">
<span class="md-ellipsis">
puffin
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/refs/" class="md-nav__link">
<span class="md-ellipsis">
refs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/snapshots/" class="md-nav__link">
<span class="md-ellipsis">
snapshots
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/sorting/" class="md-nav__link">
<span class="md-ellipsis">
sorting
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/statistics/" class="md-nav__link">
<span class="md-ellipsis">
statistics
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_12_12" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/table/update/" class="md-nav__link ">
<span class="md-ellipsis">
update
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_12_12" id="__nav_8_1_12_12_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="4" aria-labelledby="__nav_8_1_12_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_12_12">
<span class="md-nav__icon md-icon"></span>
update
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/update/schema/" class="md-nav__link">
<span class="md-ellipsis">
schema
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/update/snapshot/" class="md-nav__link">
<span class="md-ellipsis">
snapshot
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/update/sorting/" class="md-nav__link">
<span class="md-ellipsis">
sorting
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/update/spec/" class="md-nav__link">
<span class="md-ellipsis">
spec
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/update/statistics/" class="md-nav__link">
<span class="md-ellipsis">
statistics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/update/validate/" class="md-nav__link">
<span class="md-ellipsis">
validate
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/table/upsert_util/" class="md-nav__link">
<span class="md-ellipsis">
upsert_util
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/transforms/" class="md-nav__link">
<span class="md-ellipsis">
transforms
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/typedef/" class="md-nav__link">
<span class="md-ellipsis">
typedef
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/types/" class="md-nav__link">
<span class="md-ellipsis">
types
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8_1_16" >
<div class="md-nav__link md-nav__container">
<a href="reference/pyiceberg/utils/" class="md-nav__link ">
<span class="md-ellipsis">
utils
</span>
</a>
<label class="md-nav__link " for="__nav_8_1_16" id="__nav_8_1_16_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_8_1_16_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8_1_16">
<span class="md-nav__icon md-icon"></span>
utils
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/bin_packing/" class="md-nav__link">
<span class="md-ellipsis">
bin_packing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/concurrent/" class="md-nav__link">
<span class="md-ellipsis">
concurrent
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/config/" class="md-nav__link">
<span class="md-ellipsis">
config
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/datetime/" class="md-nav__link">
<span class="md-ellipsis">
datetime
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/decimal/" class="md-nav__link">
<span class="md-ellipsis">
decimal
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/deprecated/" class="md-nav__link">
<span class="md-ellipsis">
deprecated
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/lazydict/" class="md-nav__link">
<span class="md-ellipsis">
lazydict
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/parsing/" class="md-nav__link">
<span class="md-ellipsis">
parsing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/properties/" class="md-nav__link">
<span class="md-ellipsis">
properties
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/schema_conversion/" class="md-nav__link">
<span class="md-ellipsis">
schema_conversion
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/singleton/" class="md-nav__link">
<span class="md-ellipsis">
singleton
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reference/pyiceberg/utils/truncate/" class="md-nav__link">
<span class="md-ellipsis">
truncate
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#installation" class="md-nav__link">
<span class="md-ellipsis">
Installation
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#connecting-to-a-catalog" class="md-nav__link">
<span class="md-ellipsis">
Connecting to a catalog
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#write-a-pyarrow-dataframe" class="md-nav__link">
<span class="md-ellipsis">
Write a PyArrow dataframe
</span>
</a>
<nav class="md-nav" aria-label="Write a PyArrow dataframe">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#explore-iceberg-data-and-metadata-files" class="md-nav__link">
<span class="md-ellipsis">
Explore Iceberg data and metadata files
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#try-it-yourself-with-jupyter-notebooks" class="md-nav__link">
<span class="md-ellipsis">
Try it yourself with Jupyter Notebooks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#more-details" class="md-nav__link">
<span class="md-ellipsis">
More details
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-->
<h1 id="getting-started-with-pyiceberg">Getting started with PyIceberg<a class="headerlink" href="#getting-started-with-pyiceberg" title="Permanent link">&para;</a></h1>
<p>PyIceberg is a Python implementation for accessing Iceberg tables, without the need of a JVM.</p>
<h2 id="installation">Installation<a class="headerlink" href="#installation" title="Permanent link">&para;</a></h2>
<p>Before installing PyIceberg, make sure that you're on an up-to-date version of <code>pip</code>:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>pip<span class="w"> </span>install<span class="w"> </span>--upgrade<span class="w"> </span>pip
</code></pre></div>
<p>You can install the latest release version from pypi:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>pip<span class="w"> </span>install<span class="w"> </span><span class="s2">&quot;pyiceberg[s3fs,hive]&quot;</span>
</code></pre></div>
<p>You can mix and match optional dependencies depending on your needs:</p>
<table>
<thead>
<tr>
<th>Key</th>
<th>Description:</th>
</tr>
</thead>
<tbody>
<tr>
<td>hive</td>
<td>Support for the Hive metastore</td>
</tr>
<tr>
<td>hive-kerberos</td>
<td>Support for Hive metastore in Kerberos environment</td>
</tr>
<tr>
<td>glue</td>
<td>Support for AWS Glue</td>
</tr>
<tr>
<td>dynamodb</td>
<td>Support for AWS DynamoDB</td>
</tr>
<tr>
<td>bigquery</td>
<td>Support for Google Cloud BigQuery</td>
</tr>
<tr>
<td>sql-postgres</td>
<td>Support for SQL Catalog backed by Postgresql</td>
</tr>
<tr>
<td>sql-sqlite</td>
<td>Support for SQL Catalog backed by SQLite</td>
</tr>
<tr>
<td>pyarrow</td>
<td>PyArrow as a FileIO implementation to interact with the object store</td>
</tr>
<tr>
<td>pandas</td>
<td>Installs both PyArrow and Pandas</td>
</tr>
<tr>
<td>duckdb</td>
<td>Installs both PyArrow and DuckDB</td>
</tr>
<tr>
<td>ray</td>
<td>Installs PyArrow, Pandas, and Ray</td>
</tr>
<tr>
<td>bodo</td>
<td>Installs Bodo</td>
</tr>
<tr>
<td>daft</td>
<td>Installs Daft</td>
</tr>
<tr>
<td>polars</td>
<td>Installs Polars</td>
</tr>
<tr>
<td>s3fs</td>
<td>S3FS as a FileIO implementation to interact with the object store</td>
</tr>
<tr>
<td>adlfs</td>
<td>ADLFS as a FileIO implementation to interact with the object store</td>
</tr>
<tr>
<td>snappy</td>
<td>Support for snappy Avro compression</td>
</tr>
<tr>
<td>gcsfs</td>
<td>GCSFS as a FileIO implementation to interact with the object store</td>
</tr>
<tr>
<td>rest-sigv4</td>
<td>Support for generating AWS SIGv4 authentication headers for REST Catalogs</td>
</tr>
<tr>
<td>pyiceberg-core</td>
<td>Installs iceberg-rust powered core</td>
</tr>
<tr>
<td>datafusion</td>
<td>Installs both PyArrow and Apache DataFusion</td>
</tr>
<tr>
<td>hf</td>
<td>Support for Hugging Face Hub</td>
</tr>
<tr>
<td>gcp-auth</td>
<td>Support for Google Cloud authentication</td>
</tr>
<tr>
<td>entra-auth</td>
<td>Support for Azure Entra authentication</td>
</tr>
</tbody>
</table>
<p>You either need to install <code>s3fs</code>, <code>adlfs</code>, <code>gcsfs</code>, or <code>pyarrow</code> to be able to fetch files from an object store.</p>
<h2 id="connecting-to-a-catalog">Connecting to a catalog<a class="headerlink" href="#connecting-to-a-catalog" title="Permanent link">&para;</a></h2>
<p>Iceberg leverages the <a href="https://iceberg.apache.org/terms/#catalog">catalog to have one centralized place to organize the tables</a>. This can be a traditional Hive catalog to store your Iceberg tables next to the rest, a vendor solution like the AWS Glue catalog, or an implementation of Icebergs' own <a href="https://github.com/apache/iceberg/tree/main/open-api">REST protocol</a>. Checkout the <a href="configuration/">configuration</a> page to find all the configuration details.</p>
<p>For the sake of demonstration, we'll configure the catalog to use the <code>SqlCatalog</code> implementation, which will store information in a local <code>sqlite</code> database. We'll also configure the catalog to store data files in the local filesystem instead of an object store. This should not be used in production due to the limited scalability.</p>
<p>Create a temporary location for Iceberg:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a>mkdir<span class="w"> </span>/tmp/warehouse
</code></pre></div>
<p>Open a Python 3 REPL to set up the catalog:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">pyiceberg.catalog</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_catalog</span>
<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>
<a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="n">warehouse_path</span> <span class="o">=</span> <span class="s2">&quot;/tmp/warehouse&quot;</span>
<a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a><span class="n">catalog</span> <span class="o">=</span> <span class="n">load_catalog</span><span class="p">(</span>
<a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a> <span class="s2">&quot;default&quot;</span><span class="p">,</span>
<a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a> <span class="o">**</span><span class="p">{</span>
<a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a> <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;sql&#39;</span><span class="p">,</span>
<a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a> <span class="s2">&quot;uri&quot;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;sqlite:///</span><span class="si">{</span><span class="n">warehouse_path</span><span class="si">}</span><span class="s2">/pyiceberg_catalog.db&quot;</span><span class="p">,</span>
<a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a> <span class="s2">&quot;warehouse&quot;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;file://</span><span class="si">{</span><span class="n">warehouse_path</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<a id="__codelineno-3-10" name="__codelineno-3-10" href="#__codelineno-3-10"></a> <span class="p">},</span>
<a id="__codelineno-3-11" name="__codelineno-3-11" href="#__codelineno-3-11"></a><span class="p">)</span>
</code></pre></div>
<p>The <code>sql</code> catalog works for testing locally without needing another service. If you want to try out another catalog, please <a href="https://py.iceberg.apache.org/configuration/#catalogs">check out the configuration</a>.</p>
<h2 id="write-a-pyarrow-dataframe">Write a PyArrow dataframe<a class="headerlink" href="#write-a-pyarrow-dataframe" title="Permanent link">&para;</a></h2>
<p>Let's take the Taxi dataset, and write this to an Iceberg table.</p>
<p>First download one month of data:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>curl<span class="w"> </span>https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-01.parquet<span class="w"> </span>-o<span class="w"> </span>/tmp/yellow_tripdata_2023-01.parquet
</code></pre></div>
<p>Load it into your PyArrow dataframe:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.parquet</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pq</span>
<a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a>
<a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a><span class="n">df</span> <span class="o">=</span> <span class="n">pq</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="s2">&quot;/tmp/yellow_tripdata_2023-01.parquet&quot;</span><span class="p">)</span>
</code></pre></div>
<p>Create a new Iceberg table:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="n">catalog</span><span class="o">.</span><span class="n">create_namespace</span><span class="p">(</span><span class="s2">&quot;default&quot;</span><span class="p">)</span>
<a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a>
<a id="__codelineno-6-3" name="__codelineno-6-3" href="#__codelineno-6-3"></a><span class="n">table</span> <span class="o">=</span> <span class="n">catalog</span><span class="o">.</span><span class="n">create_table</span><span class="p">(</span>
<a id="__codelineno-6-4" name="__codelineno-6-4" href="#__codelineno-6-4"></a> <span class="s2">&quot;default.taxi_dataset&quot;</span><span class="p">,</span>
<a id="__codelineno-6-5" name="__codelineno-6-5" href="#__codelineno-6-5"></a> <span class="n">schema</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
<a id="__codelineno-6-6" name="__codelineno-6-6" href="#__codelineno-6-6"></a><span class="p">)</span>
</code></pre></div>
<p>Append the dataframe to the table:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a><span class="n">table</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a><span class="nb">len</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">scan</span><span class="p">()</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span>
</code></pre></div>
<p>3066766 rows have been written to the table.</p>
<p>Now generate a tip-per-mile feature to train the model on:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.compute</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pc</span>
<a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a>
<a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a><span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">append_column</span><span class="p">(</span><span class="s2">&quot;tip_per_mile&quot;</span><span class="p">,</span> <span class="n">pc</span><span class="o">.</span><span class="n">divide</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s2">&quot;tip_amount&quot;</span><span class="p">],</span> <span class="n">df</span><span class="p">[</span><span class="s2">&quot;trip_distance&quot;</span><span class="p">]))</span>
</code></pre></div>
<p>Evolve the schema of the table with the new column:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="k">with</span> <span class="n">table</span><span class="o">.</span><span class="n">update_schema</span><span class="p">()</span> <span class="k">as</span> <span class="n">update_schema</span><span class="p">:</span>
<a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a> <span class="n">update_schema</span><span class="o">.</span><span class="n">union_by_name</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
</code></pre></div>
<p>And now we can write the new dataframe to the Iceberg table:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a><span class="n">table</span><span class="o">.</span><span class="n">overwrite</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a><span class="nb">print</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">scan</span><span class="p">()</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">())</span>
</code></pre></div>
<p>And the new column is there:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a><span class="n">taxi_dataset</span><span class="p">(</span>
<a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a> <span class="mi">1</span><span class="p">:</span> <span class="n">VendorID</span><span class="p">:</span> <span class="n">optional</span> <span class="n">long</span><span class="p">,</span>
<a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a> <span class="mi">2</span><span class="p">:</span> <span class="n">tpep_pickup_datetime</span><span class="p">:</span> <span class="n">optional</span> <span class="n">timestamp</span><span class="p">,</span>
<a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a> <span class="mi">3</span><span class="p">:</span> <span class="n">tpep_dropoff_datetime</span><span class="p">:</span> <span class="n">optional</span> <span class="n">timestamp</span><span class="p">,</span>
<a id="__codelineno-11-5" name="__codelineno-11-5" href="#__codelineno-11-5"></a> <span class="mi">4</span><span class="p">:</span> <span class="n">passenger_count</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-6" name="__codelineno-11-6" href="#__codelineno-11-6"></a> <span class="mi">5</span><span class="p">:</span> <span class="n">trip_distance</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-7" name="__codelineno-11-7" href="#__codelineno-11-7"></a> <span class="mi">6</span><span class="p">:</span> <span class="n">RatecodeID</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-8" name="__codelineno-11-8" href="#__codelineno-11-8"></a> <span class="mi">7</span><span class="p">:</span> <span class="n">store_and_fwd_flag</span><span class="p">:</span> <span class="n">optional</span> <span class="n">string</span><span class="p">,</span>
<a id="__codelineno-11-9" name="__codelineno-11-9" href="#__codelineno-11-9"></a> <span class="mi">8</span><span class="p">:</span> <span class="n">PULocationID</span><span class="p">:</span> <span class="n">optional</span> <span class="n">long</span><span class="p">,</span>
<a id="__codelineno-11-10" name="__codelineno-11-10" href="#__codelineno-11-10"></a> <span class="mi">9</span><span class="p">:</span> <span class="n">DOLocationID</span><span class="p">:</span> <span class="n">optional</span> <span class="n">long</span><span class="p">,</span>
<a id="__codelineno-11-11" name="__codelineno-11-11" href="#__codelineno-11-11"></a> <span class="mi">10</span><span class="p">:</span> <span class="n">payment_type</span><span class="p">:</span> <span class="n">optional</span> <span class="n">long</span><span class="p">,</span>
<a id="__codelineno-11-12" name="__codelineno-11-12" href="#__codelineno-11-12"></a> <span class="mi">11</span><span class="p">:</span> <span class="n">fare_amount</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-13" name="__codelineno-11-13" href="#__codelineno-11-13"></a> <span class="mi">12</span><span class="p">:</span> <span class="n">extra</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-14" name="__codelineno-11-14" href="#__codelineno-11-14"></a> <span class="mi">13</span><span class="p">:</span> <span class="n">mta_tax</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-15" name="__codelineno-11-15" href="#__codelineno-11-15"></a> <span class="mi">14</span><span class="p">:</span> <span class="n">tip_amount</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-16" name="__codelineno-11-16" href="#__codelineno-11-16"></a> <span class="mi">15</span><span class="p">:</span> <span class="n">tolls_amount</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-17" name="__codelineno-11-17" href="#__codelineno-11-17"></a> <span class="mi">16</span><span class="p">:</span> <span class="n">improvement_surcharge</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-18" name="__codelineno-11-18" href="#__codelineno-11-18"></a> <span class="mi">17</span><span class="p">:</span> <span class="n">total_amount</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-19" name="__codelineno-11-19" href="#__codelineno-11-19"></a> <span class="mi">18</span><span class="p">:</span> <span class="n">congestion_surcharge</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-20" name="__codelineno-11-20" href="#__codelineno-11-20"></a> <span class="mi">19</span><span class="p">:</span> <span class="n">airport_fee</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span><span class="p">,</span>
<a id="__codelineno-11-21" name="__codelineno-11-21" href="#__codelineno-11-21"></a> <span class="mi">20</span><span class="p">:</span> <span class="n">tip_per_mile</span><span class="p">:</span> <span class="n">optional</span> <span class="n">double</span>
<a id="__codelineno-11-22" name="__codelineno-11-22" href="#__codelineno-11-22"></a><span class="p">),</span>
</code></pre></div>
<p>And we can see that 2371784 rows have a tip-per-mile:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a><span class="n">df</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">scan</span><span class="p">(</span><span class="n">row_filter</span><span class="o">=</span><span class="s2">&quot;tip_per_mile &gt; 0&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">to_arrow</span><span class="p">()</span>
<a id="__codelineno-12-2" name="__codelineno-12-2" href="#__codelineno-12-2"></a><span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>
<h3 id="explore-iceberg-data-and-metadata-files">Explore Iceberg data and metadata files<a class="headerlink" href="#explore-iceberg-data-and-metadata-files" title="Permanent link">&para;</a></h3>
<p>Since the catalog was configured to use the local filesystem, we can explore how Iceberg saved data and metadata files from the above operations.</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a>find<span class="w"> </span>/tmp/warehouse/
</code></pre></div>
<h2 id="try-it-yourself-with-jupyter-notebooks">Try it yourself with Jupyter Notebooks<a class="headerlink" href="#try-it-yourself-with-jupyter-notebooks" title="Permanent link">&para;</a></h2>
<p>PyIceberg provides Jupyter notebooks for hands-on experimentation with the examples above and more. Check out the <a href="contributing/#notebooks-for-experimentation">Notebooks for Experimentation</a> guide.</p>
<h2 id="more-details">More details<a class="headerlink" href="#more-details" title="Permanent link">&para;</a></h2>
<p>For the details, please check the <a href="cli/">CLI</a> or <a href="api/">Python API</a> page.</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": ".", "features": ["navigation.top", "navigation.tracking", "navigation.tabs", "navigation.tabs.sticky", "content.code.copy"], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="assets/javascripts/bundle.79ae519e.min.js"></script>
</body>
</html>