| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <title>pyarrow — Apache Arrow v24.0.0.dev310</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || ""; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=7f76b32a3354e82990f2" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=7f76b32a3354e82990f2" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" /> |
| <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=8dcd28dc" /> |
| |
| <!-- So that users can add custom icons --> |
| <script defer src="../_static/scripts/fontawesome.js?digest=7f76b32a3354e82990f2"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=7f76b32a3354e82990f2" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=7f76b32a3354e82990f2" /> |
| |
| <script src="../_static/documentation_options.js?v=0c0cf538"></script> |
| <script src="../_static/doctools.js?v=fd6eb6e6"></script> |
| <script src="../_static/sphinx_highlight.js?v=6ffebe34"></script> |
| <script src="../_static/clipboard.min.js?v=a7894cd8"></script> |
| <script src="../_static/copybutton.js?v=3bb21c8c"></script> |
| <script src="../_static/design-tabs.js?v=f930bc37"></script> |
| <script type="module">import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@11.12.1/dist/mermaid.esm.min.mjs"; |
| |
| |
| |
| |
| |
| const initStyles = () => { |
| const defaultStyle = document.createElement('style'); |
| defaultStyle.textContent = `pre.mermaid { |
| /* Same as .mermaid-container > pre */ |
| display: block; |
| width: 100%; |
| } |
| |
| pre.mermaid > svg { |
| /* Same as .mermaid-container > pre > svg */ |
| height: 500px; |
| width: 100%; |
| max-width: 100% !important; |
| }`; |
| document.head.appendChild(defaultStyle); |
| |
| const fullscreenStyle = document.createElement('style'); |
| fullscreenStyle.textContent = `.mermaid-container { |
| display: flex; |
| flex-direction: row; |
| width: 100%; |
| } |
| |
| .mermaid-container > pre { |
| display: block; |
| width: 100%; |
| } |
| |
| .mermaid-container > pre > svg { |
| height: 500px; |
| width: 100%; |
| max-width: 100% !important; |
| } |
| |
| .mermaid-fullscreen-btn { |
| width: 28px; |
| height: 28px; |
| background: rgba(255, 255, 255, 0.95); |
| border: 1px solid rgba(0, 0, 0, 0.3); |
| border-radius: 4px; |
| cursor: pointer; |
| display: flex; |
| align-items: center; |
| justify-content: center; |
| transition: all 0.2s; |
| box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); |
| font-size: 14px; |
| line-height: 1; |
| padding: 0; |
| color: #333; |
| } |
| |
| .mermaid-fullscreen-btn:hover { |
| opacity: 100% !important; |
| background: rgba(255, 255, 255, 1); |
| box-shadow: 0 3px 10px rgba(0, 0, 0, 0.3); |
| transform: scale(1.1); |
| } |
| |
| .mermaid-fullscreen-btn.dark-theme { |
| background: rgba(50, 50, 50, 0.95); |
| border: 1px solid rgba(255, 255, 255, 0.3); |
| color: #e0e0e0; |
| } |
| |
| .mermaid-fullscreen-btn.dark-theme:hover { |
| background: rgba(60, 60, 60, 1); |
| box-shadow: 0 3px 10px rgba(255, 255, 255, 0.2); |
| } |
| |
| .mermaid-fullscreen-modal { |
| display: none; |
| position: fixed !important; |
| top: 0 !important; |
| left: 0 !important; |
| width: 95vw; |
| height: 100vh; |
| background: rgba(255, 255, 255, 0.98); |
| z-index: 9999; |
| padding: 20px; |
| overflow: auto; |
| } |
| |
| .mermaid-fullscreen-modal.dark-theme { |
| background: rgba(0, 0, 0, 0.98); |
| } |
| |
| .mermaid-fullscreen-modal.active { |
| display: flex; |
| align-items: center; |
| justify-content: center; |
| } |
| |
| .mermaid-container-fullscreen { |
| position: relative; |
| width: 95vw; |
| height: 90vh; |
| max-width: 95vw; |
| max-height: 90vh; |
| background: white; |
| border-radius: 8px; |
| padding: 20px; |
| box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3); |
| overflow: auto; |
| display: flex; |
| align-items: center; |
| justify-content: center; |
| } |
| |
| .mermaid-container-fullscreen.dark-theme { |
| background: #1a1a1a; |
| box-shadow: 0 10px 40px rgba(0, 0, 0, 0.8); |
| } |
| |
| .mermaid-container-fullscreen pre.mermaid { |
| width: 100%; |
| height: 100%; |
| display: flex; |
| align-items: center; |
| justify-content: center; |
| } |
| |
| .mermaid-container-fullscreen .mermaid svg { |
| height: 100% !important; |
| width: 100% !important; |
| cursor: grab; |
| } |
| |
| .mermaid-fullscreen-close { |
| position: fixed !important; |
| top: 20px !important; |
| right: 20px !important; |
| width: 40px; |
| height: 40px; |
| background: rgba(255, 255, 255, 0.95); |
| border: 1px solid rgba(0, 0, 0, 0.2); |
| border-radius: 50%; |
| cursor: pointer; |
| z-index: 10000; |
| display: flex; |
| align-items: center; |
| justify-content: center; |
| box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); |
| transition: all 0.2s; |
| font-size: 24px; |
| line-height: 1; |
| color: #333; |
| } |
| |
| .mermaid-fullscreen-close:hover { |
| background: white; |
| box-shadow: 0 6px 16px rgba(0, 0, 0, 0.4); |
| transform: scale(1.1); |
| } |
| |
| .mermaid-fullscreen-close.dark-theme { |
| background: rgba(50, 50, 50, 0.95); |
| border: 1px solid rgba(255, 255, 255, 0.2); |
| color: #e0e0e0; |
| } |
| |
| .mermaid-fullscreen-close.dark-theme:hover { |
| background: rgba(60, 60, 60, 1); |
| box-shadow: 0 6px 16px rgba(255, 255, 255, 0.2); |
| } |
| |
| .mermaid-fullscreen-modal .mermaid-fullscreen-btn { |
| display: none !important; |
| }`; |
| document.head.appendChild(fullscreenStyle); |
| } |
| |
| // Detect if page has dark background |
| const isDarkTheme = () => { |
| // We use a set of heuristics: |
| // 1. Check for common dark mode classes or attributes |
| // 2. Check computed background color brightness |
| if (document.documentElement.classList.contains('dark') || |
| document.documentElement.getAttribute('data-theme') === 'dark' || |
| document.body.classList.contains('dark') || |
| document.body.getAttribute('data-theme') === 'dark') { |
| // console.log("Dark theme detected via class/attribute"); |
| return true; |
| } |
| if (document.documentElement.classList.contains('light') || |
| document.documentElement.getAttribute('data-theme') === 'light' || |
| document.body.classList.contains('light') || |
| document.body.getAttribute('data-theme') === 'light') { |
| // console.log("Light theme detected via class/attribute"); |
| return false; |
| } |
| if (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { |
| // console.log("Dark theme detected via prefers-color-scheme"); |
| return true; |
| } |
| const bgColor = window.getComputedStyle(document.body).backgroundColor; |
| const match = bgColor.match(/rgb\((\d+),\s*(\d+),\s*(\d+)/); |
| if (match) { |
| const r = parseInt(match[1]); |
| const g = parseInt(match[2]); |
| const b = parseInt(match[3]); |
| const brightness = (r * 299 + g * 587 + b * 114) / 1000; |
| // console.log("Background color brightness:", brightness); |
| return brightness < 128; |
| } |
| // console.log("No dark or light theme detected, defaulting to light theme"); |
| return false; |
| }; |
| |
| let darkTheme = isDarkTheme(); |
| let modal = null; |
| let modalContent = null; |
| let previousScrollOffset = [window.scrollX, window.scrollY]; |
| |
| const runMermaid = async (rerun) => { |
| console.log("Running mermaid diagrams, rerun =", rerun); |
| // clear all existing mermaid charts |
| let all_mermaids = document.querySelectorAll(".mermaid"); |
| |
| if (rerun) { |
| all_mermaids.forEach((el) => { |
| if(!el.hasAttribute("data-original-code")) { |
| // store original code |
| // console.log(`Storing original code for first run: `, el.innerHTML); |
| el.setAttribute('data-original-code', el.innerHTML); |
| } |
| if(el.getAttribute("data-processed") === "true") { |
| // remove and restore original |
| el.removeAttribute("data-processed"); |
| // console.log(`Restoring original code for re-run: `, el.getAttribute('data-original-code')); |
| el.innerHTML = el.getAttribute('data-original-code'); |
| } else { |
| // store original code |
| // console.log(`Storing original code for re-run: `, el.innerHTML); |
| el.setAttribute('data-original-code', el.innerHTML); |
| } |
| }); |
| await mermaid.run(); |
| } |
| |
| all_mermaids = document.querySelectorAll(".mermaid"); |
| const mermaids_processed = document.querySelectorAll(".mermaid[data-processed='true']"); |
| |
| if ("False" === "True") { |
| const mermaids_to_add_zoom = -1 === -1 ? all_mermaids.length : -1; |
| if(mermaids_to_add_zoom > 0) { |
| var svgs = d3.selectAll(""); |
| if(all_mermaids.length !== mermaids_processed.length) { |
| setTimeout(() => runMermaid(false), 200); |
| return; |
| } else if(svgs.size() !== mermaids_to_add_zoom) { |
| setTimeout(() => runMermaid(false), 200); |
| return; |
| } else { |
| svgs.each(function() { |
| var svg = d3.select(this); |
| svg.html("<g class='wrapper'>" + svg.html() + "</g>"); |
| var inner = svg.select("g"); |
| var zoom = d3.zoom().on("zoom", function(event) { |
| inner.attr("transform", event.transform); |
| }); |
| svg.call(zoom); |
| }); |
| } |
| } |
| } else if(all_mermaids.length !== mermaids_processed.length) { |
| // Wait for mermaid to process all diagrams |
| setTimeout(() => runMermaid(false), 200); |
| return; |
| } |
| |
| // Stop here if not adding fullscreen capability |
| if ("True" !== "True") return; |
| |
| if (modal !== null ) { |
| // Destroy existing modal |
| modal.remove(); |
| modal = null; |
| modalContent = null; |
| } |
| |
| modal = document.createElement('div'); |
| modal.className = 'mermaid-fullscreen-modal' + (darkTheme ? ' dark-theme' : ''); |
| modal.setAttribute('role', 'dialog'); |
| modal.setAttribute('aria-modal', 'true'); |
| modal.setAttribute('aria-label', 'Fullscreen diagram viewer'); |
| modal.innerHTML = ` |
| <button class="mermaid-fullscreen-close${darkTheme ? ' dark-theme' : ''}" aria-label="Close fullscreen">✕</button> |
| <div class="mermaid-container-fullscreen${darkTheme ? ' dark-theme' : ''}"></div> |
| `; |
| document.body.appendChild(modal); |
| |
| modalContent = modal.querySelector('.mermaid-container-fullscreen'); |
| const closeBtn = modal.querySelector('.mermaid-fullscreen-close'); |
| |
| const closeModal = () => { |
| modal.classList.remove('active'); |
| modalContent.innerHTML = ''; |
| document.body.style.overflow = '' |
| window.scrollTo({left: previousScrollOffset[0], top: previousScrollOffset[1], behavior: 'instant'}); |
| }; |
| |
| closeBtn.addEventListener('click', closeModal); |
| modal.addEventListener('click', (e) => { |
| if (e.target === modal) closeModal(); |
| }); |
| document.addEventListener('keydown', (e) => { |
| if (e.key === 'Escape' && modal.classList.contains('active')) { |
| closeModal(); |
| } |
| }); |
| |
| document.querySelectorAll('.mermaid').forEach((mermaidDiv) => { |
| if (mermaidDiv.parentNode.classList.contains('mermaid-container') || |
| mermaidDiv.closest('.mermaid-fullscreen-modal')) { |
| // Already processed, adjust button class if needed |
| const existingBtn = mermaidDiv.parentNode.querySelector('.mermaid-fullscreen-btn'); |
| if (existingBtn) { |
| existingBtn.className = 'mermaid-fullscreen-btn' + (darkTheme ? ' dark-theme' : ''); |
| } |
| return; |
| } |
| |
| const container = document.createElement('div'); |
| container.className = 'mermaid-container'; |
| mermaidDiv.parentNode.insertBefore(container, mermaidDiv); |
| container.appendChild(mermaidDiv); |
| |
| const fullscreenBtn = document.createElement('button'); |
| fullscreenBtn.className = 'mermaid-fullscreen-btn' + (darkTheme ? ' dark-theme' : ''); |
| fullscreenBtn.setAttribute('aria-label', 'View diagram in fullscreen'); |
| fullscreenBtn.textContent = '⛶'; |
| fullscreenBtn.style.opacity = '50%'; |
| |
| // Calculate dynamic position based on diagram's margin and padding |
| const diagramStyle = window.getComputedStyle(mermaidDiv); |
| const marginTop = parseFloat(diagramStyle.marginTop) || 0; |
| const marginRight = parseFloat(diagramStyle.marginRight) || 0; |
| const paddingTop = parseFloat(diagramStyle.paddingTop) || 0; |
| const paddingRight = parseFloat(diagramStyle.paddingRight) || 0; |
| fullscreenBtn.style.top = `${marginTop + paddingTop + 4}px`; |
| fullscreenBtn.style.right = `${marginRight + paddingRight + 4}px`; |
| |
| fullscreenBtn.addEventListener('click', () => { |
| previousScrollOffset = [window.scroll, window.scrollY]; |
| const clone = mermaidDiv.cloneNode(true); |
| modalContent.innerHTML = ''; |
| modalContent.appendChild(clone); |
| |
| const svg = clone.querySelector('svg'); |
| if (svg) { |
| svg.removeAttribute('width'); |
| svg.removeAttribute('height'); |
| svg.style.width = '100%'; |
| svg.style.height = 'auto'; |
| svg.style.maxWidth = '100%'; |
| svg.style.sdisplay = 'block'; |
| |
| if ("False" === "True") { |
| setTimeout(() => { |
| const g = svg.querySelector('g'); |
| if (g) { |
| var svgD3 = d3.select(svg); |
| svgD3.html("<g class='wrapper'>" + svgD3.html() + "</g>"); |
| var inner = svgD3.select("g"); |
| var zoom = d3.zoom().on("zoom", function(event) { |
| inner.attr("transform", event.transform); |
| }); |
| svgD3.call(zoom); |
| } |
| }, 100); |
| } |
| } |
| |
| modal.classList.add('active'); |
| document.body.style.overflow = 'hidden'; |
| }); |
| container.appendChild(fullscreenBtn); |
| }); |
| }; |
| |
| const load = async () => { |
| initStyles(); |
| |
| await runMermaid(true); |
| |
| const reRunIfThemeChanges = async () => { |
| const newDarkTheme = isDarkTheme(); |
| if (newDarkTheme !== darkTheme) { |
| darkTheme = newDarkTheme; |
| console.log("Theme change detected, re-running mermaid with", darkTheme ? "dark" : "default", "theme"); |
| await mermaid.initialize( |
| {...JSON.parse( |
| `{"startOnLoad": false}` |
| ), |
| ...{ darkMode: darkTheme, theme: darkTheme ? 'dark' : 'default' }, |
| } |
| ); |
| await runMermaid(true); |
| } |
| }; |
| |
| // Update theme classes when theme changes |
| const themeObserver = new MutationObserver(reRunIfThemeChanges); |
| themeObserver.observe(document.documentElement, { |
| attributes: true, |
| attributeFilter: ['class', 'style', 'data-theme'] |
| }); |
| themeObserver.observe(document.body, { |
| attributes: true, |
| attributeFilter: ['class', 'style', 'data-theme'] |
| }); |
| }; |
| |
| |
| |
| |
| |
| console.log("Initializing mermaid with", darkTheme ? "dark" : "default", "theme"); |
| mermaid.initialize( |
| {...JSON.parse( |
| `{"startOnLoad": false}` |
| ), |
| ...{ darkMode: darkTheme, theme: darkTheme ? 'dark' : 'default' }, |
| } |
| ); |
| |
| window.addEventListener("load", load); |
| window.runMermaid = runMermaid;</script> |
| <script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyarrow';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.17.0'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = |
| true; |
| </script> |
| <script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/_modules/pyarrow.html" /> |
| <link rel="icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="24.0.0.dev310" /> |
| |
| |
| <script src="../_static/searchtools.js"></script> |
| <script src="../_static/language_data.js"></script> |
| <script src="../searchindex.js"></script> |
| |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| <body data-default-mode=""> |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class=" navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v24.0.0.dev310 - Home"/> |
| <img src="../_static/arrow-dark.png" class="logo__image only-dark pst-js-only" alt="Apache Arrow v24.0.0.dev310 - Home"/> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class=" navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar hide-on-wide"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Module code</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">pyarrow</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <h1>Source code for pyarrow</h1><div class="highlight"><pre> |
| <span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="c1"># or more contributor license agreements. See the NOTICE file</span> |
| <span class="c1"># distributed with this work for additional information</span> |
| <span class="c1"># regarding copyright ownership. The ASF licenses this file</span> |
| <span class="c1"># to you under the Apache License, Version 2.0 (the</span> |
| <span class="c1"># "License"); you may not use this file except in compliance</span> |
| <span class="c1"># with the License. You may obtain a copy of the License at</span> |
| <span class="c1">#</span> |
| <span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="c1">#</span> |
| <span class="c1"># Unless required by applicable law or agreed to in writing,</span> |
| <span class="c1"># software distributed under the License is distributed on an</span> |
| <span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> |
| <span class="c1"># KIND, either express or implied. See the License for the</span> |
| <span class="c1"># specific language governing permissions and limitations</span> |
| <span class="c1"># under the License.</span> |
| |
| <span class="c1"># flake8: noqa</span> |
| |
| <span class="sd">"""</span> |
| <span class="sd">PyArrow is the python implementation of Apache Arrow.</span> |
| |
| <span class="sd">Apache Arrow is a cross-language development platform for in-memory data.</span> |
| <span class="sd">It specifies a standardized language-independent columnar memory format for</span> |
| <span class="sd">flat and hierarchical data, organized for efficient analytic operations on</span> |
| <span class="sd">modern hardware. It also provides computational libraries and zero-copy</span> |
| <span class="sd">streaming messaging and interprocess communication.</span> |
| |
| <span class="sd">For more information see the official page at https://arrow.apache.org</span> |
| <span class="sd">"""</span> |
| |
| <span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">_importlib</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">os</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">_os</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">platform</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">_platform</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">sys</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">_sys</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">._generated_version</span><span class="w"> </span><span class="kn">import</span> <span class="n">version</span> <span class="k">as</span> <span class="n">__version__</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="c1"># Package is not installed, parse git tag at runtime</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">setuptools_scm</span> |
| <span class="c1"># Code duplicated from setup.py to avoid a dependency on each other</span> |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">parse_git</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Parse function for setuptools_scm that ignores tags for non-C++</span> |
| <span class="sd"> subprojects, e.g. apache-arrow-js-XXX tags.</span> |
| <span class="sd"> """</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">setuptools_scm.git</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span> |
| <span class="n">kwargs</span><span class="p">[</span><span class="s1">'describe_command'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"git describe --dirty --tags --long --match 'apache-arrow-[0-9]*.*'"</span> |
| <span class="k">return</span> <span class="n">parse</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> |
| <span class="n">__version__</span> <span class="o">=</span> <span class="n">setuptools_scm</span><span class="o">.</span><span class="n">get_version</span><span class="p">(</span><span class="s1">'../'</span><span class="p">,</span> |
| <span class="n">parse</span><span class="o">=</span><span class="n">parse_git</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="n">__version__</span> <span class="o">=</span> <span class="kc">None</span> |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">BuildInfo</span><span class="p">,</span> <span class="n">CppBuildInfo</span><span class="p">,</span> <span class="n">RuntimeInfo</span><span class="p">,</span> <span class="n">set_timezone_db_path</span><span class="p">,</span> |
| <span class="n">MonthDayNano</span><span class="p">,</span> <span class="n">VersionInfo</span><span class="p">,</span> <span class="n">build_info</span><span class="p">,</span> <span class="n">cpp_build_info</span><span class="p">,</span> |
| <span class="n">cpp_version</span><span class="p">,</span> <span class="n">cpp_version_info</span><span class="p">,</span> <span class="n">runtime_info</span><span class="p">,</span> |
| <span class="n">cpu_count</span><span class="p">,</span> <span class="n">set_cpu_count</span><span class="p">,</span> <span class="n">enable_signal_handlers</span><span class="p">,</span> |
| <span class="n">io_thread_count</span><span class="p">,</span> <span class="n">is_opentelemetry_enabled</span><span class="p">,</span> <span class="n">set_io_thread_count</span><span class="p">)</span> |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">show_versions</span><span class="p">():</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Print various version information, to help with error reporting.</span> |
| <span class="sd"> """</span> |
| <span class="k">def</span><span class="w"> </span><span class="nf">print_entry</span><span class="p">(</span><span class="n">label</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span> |
| <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">label</span><span class="si">:</span><span class="s2"> <26</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">value</span><span class="si">:</span><span class="s2"> <8</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"pyarrow version info</span><span class="se">\n</span><span class="s2">--------------------"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Package kind"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">package_kind</span> |
| <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">package_kind</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span> |
| <span class="k">else</span> <span class="s2">"not indicated"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Arrow C++ library version"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">version</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Arrow C++ compiler"</span><span class="p">,</span> |
| <span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">compiler_id</span><span class="si">}</span><span class="s2"> "</span> |
| <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">compiler_version</span><span class="si">}</span><span class="s2">"</span><span class="p">))</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Arrow C++ compiler flags"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">compiler_flags</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Arrow C++ git revision"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">git_id</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Arrow C++ git description"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">git_description</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Arrow C++ build type"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">cpp_build_info</span><span class="o">.</span><span class="n">build_type</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"PyArrow build type"</span><span class="p">,</span> <span class="n">build_info</span><span class="o">.</span><span class="n">build_type</span><span class="p">)</span> |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">_module_is_available</span><span class="p">(</span><span class="n">module</span><span class="p">):</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">_importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="sa">f</span><span class="s1">'pyarrow.</span><span class="si">{</span><span class="n">module</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">False</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">True</span> |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">_filesystem_is_available</span><span class="p">(</span><span class="n">fs</span><span class="p">):</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.fs</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">False</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="nb">getattr</span><span class="p">(</span><span class="n">pyarrow</span><span class="o">.</span><span class="n">fs</span><span class="p">,</span> <span class="n">fs</span><span class="p">)</span> |
| <span class="k">except</span> <span class="p">(</span><span class="ne">ImportError</span><span class="p">,</span> <span class="ne">AttributeError</span><span class="p">):</span> |
| <span class="k">return</span> <span class="kc">False</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">True</span> |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">show_info</span><span class="p">():</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Print detailed version and platform information, for error reporting</span> |
| <span class="sd"> """</span> |
| <span class="n">show_versions</span><span class="p">()</span> |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">print_entry</span><span class="p">(</span><span class="n">label</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span> |
| <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">label</span><span class="si">:</span><span class="s2"> <20</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">value</span><span class="si">:</span><span class="s2"> <8</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">Platform:"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"OS / Arch"</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">_platform</span><span class="o">.</span><span class="n">system</span><span class="p">()</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">_platform</span><span class="o">.</span><span class="n">machine</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"SIMD Level"</span><span class="p">,</span> <span class="n">runtime_info</span><span class="p">()</span><span class="o">.</span><span class="n">simd_level</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Detected SIMD Level"</span><span class="p">,</span> <span class="n">runtime_info</span><span class="p">()</span><span class="o">.</span><span class="n">detected_simd_level</span><span class="p">)</span> |
| |
| <span class="n">pool</span> <span class="o">=</span> <span class="n">default_memory_pool</span><span class="p">()</span> |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">Memory:"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Default backend"</span><span class="p">,</span> <span class="n">pool</span><span class="o">.</span><span class="n">backend_name</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Bytes allocated"</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">pool</span><span class="o">.</span><span class="n">bytes_allocated</span><span class="p">()</span><span class="si">}</span><span class="s2"> bytes"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Max memory"</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">pool</span><span class="o">.</span><span class="n">max_memory</span><span class="p">()</span><span class="si">}</span><span class="s2"> bytes"</span><span class="p">)</span> |
| <span class="n">print_entry</span><span class="p">(</span><span class="s2">"Supported Backends"</span><span class="p">,</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">supported_memory_backends</span><span class="p">()))</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">Optional modules:"</span><span class="p">)</span> |
| <span class="n">modules</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"csv"</span><span class="p">,</span> <span class="s2">"cuda"</span><span class="p">,</span> <span class="s2">"dataset"</span><span class="p">,</span> <span class="s2">"feather"</span><span class="p">,</span> <span class="s2">"flight"</span><span class="p">,</span> <span class="s2">"fs"</span><span class="p">,</span> <span class="s2">"gandiva"</span><span class="p">,</span> <span class="s2">"json"</span><span class="p">,</span> |
| <span class="s2">"orc"</span><span class="p">,</span> <span class="s2">"parquet"</span><span class="p">]</span> |
| <span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">modules</span><span class="p">:</span> |
| <span class="n">status</span> <span class="o">=</span> <span class="s2">"Enabled"</span> <span class="k">if</span> <span class="n">_module_is_available</span><span class="p">(</span><span class="n">module</span><span class="p">)</span> <span class="k">else</span> <span class="s2">"-"</span> |
| <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">module</span><span class="si">:</span><span class="s2"> <20</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">status</span><span class="si">:</span><span class="s2"> <8</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="s1">'opentelemetry'</span><span class="si">:</span><span class="s2"> <20</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="s1">'Enabled'</span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="n">is_opentelemetry_enabled</span><span class="p">()</span><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="s1">'-'</span><span class="si">:</span><span class="s2"> <8</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">Filesystems:"</span><span class="p">)</span> |
| <span class="n">filesystems</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"AzureFileSystem"</span><span class="p">,</span> <span class="s2">"GcsFileSystem"</span><span class="p">,</span> |
| <span class="s2">"HadoopFileSystem"</span><span class="p">,</span> <span class="s2">"S3FileSystem"</span><span class="p">]</span> |
| <span class="k">for</span> <span class="n">fs</span> <span class="ow">in</span> <span class="n">filesystems</span><span class="p">:</span> |
| <span class="n">status</span> <span class="o">=</span> <span class="s2">"Enabled"</span> <span class="k">if</span> <span class="n">_filesystem_is_available</span><span class="p">(</span><span class="n">fs</span><span class="p">)</span> <span class="k">else</span> <span class="s2">"-"</span> |
| <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">fs</span><span class="si">:</span><span class="s2"> <20</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">status</span><span class="si">:</span><span class="s2"> <8</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">Compression Codecs:"</span><span class="p">)</span> |
| <span class="n">codecs</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"brotli"</span><span class="p">,</span> <span class="s2">"bz2"</span><span class="p">,</span> <span class="s2">"gzip"</span><span class="p">,</span> <span class="s2">"lz4_frame"</span><span class="p">,</span> <span class="s2">"lz4"</span><span class="p">,</span> <span class="s2">"snappy"</span><span class="p">,</span> <span class="s2">"zstd"</span><span class="p">]</span> |
| <span class="k">for</span> <span class="n">codec</span> <span class="ow">in</span> <span class="n">codecs</span><span class="p">:</span> |
| <span class="n">status</span> <span class="o">=</span> <span class="s2">"Enabled"</span> <span class="k">if</span> <span class="n">Codec</span><span class="o">.</span><span class="n">is_available</span><span class="p">(</span><span class="n">codec</span><span class="p">)</span> <span class="k">else</span> <span class="s2">"-"</span> |
| <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">codec</span><span class="si">:</span><span class="s2"> <20</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">status</span><span class="si">:</span><span class="s2"> <8</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">null</span><span class="p">,</span> <span class="n">bool_</span><span class="p">,</span> |
| <span class="n">int8</span><span class="p">,</span> <span class="n">int16</span><span class="p">,</span> <span class="n">int32</span><span class="p">,</span> <span class="n">int64</span><span class="p">,</span> |
| <span class="n">uint8</span><span class="p">,</span> <span class="n">uint16</span><span class="p">,</span> <span class="n">uint32</span><span class="p">,</span> <span class="n">uint64</span><span class="p">,</span> |
| <span class="n">time32</span><span class="p">,</span> <span class="n">time64</span><span class="p">,</span> <span class="n">timestamp</span><span class="p">,</span> <span class="n">date32</span><span class="p">,</span> <span class="n">date64</span><span class="p">,</span> <span class="n">duration</span><span class="p">,</span> |
| <span class="n">month_day_nano_interval</span><span class="p">,</span> |
| <span class="n">float16</span><span class="p">,</span> <span class="n">float32</span><span class="p">,</span> <span class="n">float64</span><span class="p">,</span> |
| <span class="n">binary</span><span class="p">,</span> <span class="n">string</span><span class="p">,</span> <span class="n">utf8</span><span class="p">,</span> <span class="n">binary_view</span><span class="p">,</span> <span class="n">string_view</span><span class="p">,</span> |
| <span class="n">large_binary</span><span class="p">,</span> <span class="n">large_string</span><span class="p">,</span> <span class="n">large_utf8</span><span class="p">,</span> |
| <span class="n">decimal32</span><span class="p">,</span> <span class="n">decimal64</span><span class="p">,</span> <span class="n">decimal128</span><span class="p">,</span> <span class="n">decimal256</span><span class="p">,</span> |
| <span class="n">list_</span><span class="p">,</span> <span class="n">large_list</span><span class="p">,</span> <span class="n">list_view</span><span class="p">,</span> <span class="n">large_list_view</span><span class="p">,</span> |
| <span class="n">map_</span><span class="p">,</span> <span class="n">struct</span><span class="p">,</span> |
| <span class="n">union</span><span class="p">,</span> <span class="n">sparse_union</span><span class="p">,</span> <span class="n">dense_union</span><span class="p">,</span> |
| <span class="n">dictionary</span><span class="p">,</span> |
| <span class="n">run_end_encoded</span><span class="p">,</span> |
| <span class="n">bool8</span><span class="p">,</span> <span class="n">fixed_shape_tensor</span><span class="p">,</span> <span class="n">json_</span><span class="p">,</span> <span class="n">opaque</span><span class="p">,</span> <span class="n">uuid</span><span class="p">,</span> |
| <span class="n">field</span><span class="p">,</span> |
| <span class="n">type_for_alias</span><span class="p">,</span> |
| <span class="n">DataType</span><span class="p">,</span> <span class="n">DictionaryType</span><span class="p">,</span> <span class="n">StructType</span><span class="p">,</span> |
| <span class="n">ListType</span><span class="p">,</span> <span class="n">LargeListType</span><span class="p">,</span> <span class="n">FixedSizeListType</span><span class="p">,</span> |
| <span class="n">ListViewType</span><span class="p">,</span> <span class="n">LargeListViewType</span><span class="p">,</span> |
| <span class="n">MapType</span><span class="p">,</span> <span class="n">UnionType</span><span class="p">,</span> <span class="n">SparseUnionType</span><span class="p">,</span> <span class="n">DenseUnionType</span><span class="p">,</span> |
| <span class="n">TimestampType</span><span class="p">,</span> <span class="n">Time32Type</span><span class="p">,</span> <span class="n">Time64Type</span><span class="p">,</span> <span class="n">DurationType</span><span class="p">,</span> |
| <span class="n">FixedSizeBinaryType</span><span class="p">,</span> |
| <span class="n">Decimal32Type</span><span class="p">,</span> <span class="n">Decimal64Type</span><span class="p">,</span> <span class="n">Decimal128Type</span><span class="p">,</span> <span class="n">Decimal256Type</span><span class="p">,</span> |
| <span class="n">BaseExtensionType</span><span class="p">,</span> <span class="n">ExtensionType</span><span class="p">,</span> |
| <span class="n">RunEndEncodedType</span><span class="p">,</span> <span class="n">Bool8Type</span><span class="p">,</span> <span class="n">FixedShapeTensorType</span><span class="p">,</span> |
| <span class="n">JsonType</span><span class="p">,</span> <span class="n">OpaqueType</span><span class="p">,</span> <span class="n">UuidType</span><span class="p">,</span> |
| <span class="n">UnknownExtensionType</span><span class="p">,</span> |
| <span class="n">register_extension_type</span><span class="p">,</span> <span class="n">unregister_extension_type</span><span class="p">,</span> |
| <span class="n">DictionaryMemo</span><span class="p">,</span> |
| <span class="n">KeyValueMetadata</span><span class="p">,</span> |
| <span class="n">Field</span><span class="p">,</span> |
| <span class="n">Schema</span><span class="p">,</span> |
| <span class="n">schema</span><span class="p">,</span> |
| <span class="n">unify_schemas</span><span class="p">,</span> |
| <span class="n">Array</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">,</span> |
| <span class="n">array</span><span class="p">,</span> <span class="n">chunked_array</span><span class="p">,</span> <span class="n">record_batch</span><span class="p">,</span> <span class="n">nulls</span><span class="p">,</span> <span class="n">repeat</span><span class="p">,</span> |
| <span class="n">SparseCOOTensor</span><span class="p">,</span> <span class="n">SparseCSRMatrix</span><span class="p">,</span> <span class="n">SparseCSCMatrix</span><span class="p">,</span> |
| <span class="n">SparseCSFTensor</span><span class="p">,</span> |
| <span class="n">infer_type</span><span class="p">,</span> <span class="n">from_numpy_dtype</span><span class="p">,</span> |
| <span class="n">arange</span><span class="p">,</span> |
| <span class="n">NullArray</span><span class="p">,</span> |
| <span class="n">NumericArray</span><span class="p">,</span> <span class="n">IntegerArray</span><span class="p">,</span> <span class="n">FloatingPointArray</span><span class="p">,</span> |
| <span class="n">BooleanArray</span><span class="p">,</span> |
| <span class="n">Int8Array</span><span class="p">,</span> <span class="n">UInt8Array</span><span class="p">,</span> |
| <span class="n">Int16Array</span><span class="p">,</span> <span class="n">UInt16Array</span><span class="p">,</span> |
| <span class="n">Int32Array</span><span class="p">,</span> <span class="n">UInt32Array</span><span class="p">,</span> |
| <span class="n">Int64Array</span><span class="p">,</span> <span class="n">UInt64Array</span><span class="p">,</span> |
| <span class="n">HalfFloatArray</span><span class="p">,</span> <span class="n">FloatArray</span><span class="p">,</span> <span class="n">DoubleArray</span><span class="p">,</span> |
| <span class="n">ListArray</span><span class="p">,</span> <span class="n">LargeListArray</span><span class="p">,</span> <span class="n">FixedSizeListArray</span><span class="p">,</span> |
| <span class="n">ListViewArray</span><span class="p">,</span> <span class="n">LargeListViewArray</span><span class="p">,</span> |
| <span class="n">MapArray</span><span class="p">,</span> <span class="n">UnionArray</span><span class="p">,</span> |
| <span class="n">BinaryArray</span><span class="p">,</span> <span class="n">StringArray</span><span class="p">,</span> |
| <span class="n">LargeBinaryArray</span><span class="p">,</span> <span class="n">LargeStringArray</span><span class="p">,</span> |
| <span class="n">BinaryViewArray</span><span class="p">,</span> <span class="n">StringViewArray</span><span class="p">,</span> |
| <span class="n">FixedSizeBinaryArray</span><span class="p">,</span> |
| <span class="n">DictionaryArray</span><span class="p">,</span> |
| <span class="n">Date32Array</span><span class="p">,</span> <span class="n">Date64Array</span><span class="p">,</span> <span class="n">TimestampArray</span><span class="p">,</span> |
| <span class="n">Time32Array</span><span class="p">,</span> <span class="n">Time64Array</span><span class="p">,</span> <span class="n">DurationArray</span><span class="p">,</span> |
| <span class="n">MonthDayNanoIntervalArray</span><span class="p">,</span> |
| <span class="n">Decimal32Array</span><span class="p">,</span> <span class="n">Decimal64Array</span><span class="p">,</span> <span class="n">Decimal128Array</span><span class="p">,</span> <span class="n">Decimal256Array</span><span class="p">,</span> |
| <span class="n">StructArray</span><span class="p">,</span> <span class="n">ExtensionArray</span><span class="p">,</span> |
| <span class="n">RunEndEncodedArray</span><span class="p">,</span> <span class="n">Bool8Array</span><span class="p">,</span> <span class="n">FixedShapeTensorArray</span><span class="p">,</span> |
| <span class="n">JsonArray</span><span class="p">,</span> <span class="n">OpaqueArray</span><span class="p">,</span> <span class="n">UuidArray</span><span class="p">,</span> |
| <span class="n">scalar</span><span class="p">,</span> <span class="n">NA</span><span class="p">,</span> <span class="n">_NULL</span> <span class="k">as</span> <span class="n">NULL</span><span class="p">,</span> <span class="n">Scalar</span><span class="p">,</span> |
| <span class="n">NullScalar</span><span class="p">,</span> <span class="n">BooleanScalar</span><span class="p">,</span> |
| <span class="n">Int8Scalar</span><span class="p">,</span> <span class="n">Int16Scalar</span><span class="p">,</span> <span class="n">Int32Scalar</span><span class="p">,</span> <span class="n">Int64Scalar</span><span class="p">,</span> |
| <span class="n">UInt8Scalar</span><span class="p">,</span> <span class="n">UInt16Scalar</span><span class="p">,</span> <span class="n">UInt32Scalar</span><span class="p">,</span> <span class="n">UInt64Scalar</span><span class="p">,</span> |
| <span class="n">HalfFloatScalar</span><span class="p">,</span> <span class="n">FloatScalar</span><span class="p">,</span> <span class="n">DoubleScalar</span><span class="p">,</span> |
| <span class="n">Decimal32Scalar</span><span class="p">,</span> <span class="n">Decimal64Scalar</span><span class="p">,</span> <span class="n">Decimal128Scalar</span><span class="p">,</span> <span class="n">Decimal256Scalar</span><span class="p">,</span> |
| <span class="n">ListScalar</span><span class="p">,</span> <span class="n">LargeListScalar</span><span class="p">,</span> <span class="n">FixedSizeListScalar</span><span class="p">,</span> |
| <span class="n">ListViewScalar</span><span class="p">,</span> <span class="n">LargeListViewScalar</span><span class="p">,</span> |
| <span class="n">Date32Scalar</span><span class="p">,</span> <span class="n">Date64Scalar</span><span class="p">,</span> |
| <span class="n">Time32Scalar</span><span class="p">,</span> <span class="n">Time64Scalar</span><span class="p">,</span> |
| <span class="n">TimestampScalar</span><span class="p">,</span> <span class="n">DurationScalar</span><span class="p">,</span> |
| <span class="n">MonthDayNanoIntervalScalar</span><span class="p">,</span> |
| <span class="n">BinaryScalar</span><span class="p">,</span> <span class="n">LargeBinaryScalar</span><span class="p">,</span> <span class="n">BinaryViewScalar</span><span class="p">,</span> |
| <span class="n">StringScalar</span><span class="p">,</span> <span class="n">LargeStringScalar</span><span class="p">,</span> <span class="n">StringViewScalar</span><span class="p">,</span> |
| <span class="n">FixedSizeBinaryScalar</span><span class="p">,</span> <span class="n">DictionaryScalar</span><span class="p">,</span> |
| <span class="n">MapScalar</span><span class="p">,</span> <span class="n">StructScalar</span><span class="p">,</span> <span class="n">UnionScalar</span><span class="p">,</span> |
| <span class="n">RunEndEncodedScalar</span><span class="p">,</span> <span class="n">Bool8Scalar</span><span class="p">,</span> <span class="n">ExtensionScalar</span><span class="p">,</span> |
| <span class="n">FixedShapeTensorScalar</span><span class="p">,</span> <span class="n">JsonScalar</span><span class="p">,</span> <span class="n">OpaqueScalar</span><span class="p">,</span> <span class="n">UuidScalar</span><span class="p">)</span> |
| |
| <span class="c1"># Buffers, allocation</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">DeviceAllocationType</span><span class="p">,</span> <span class="n">Device</span><span class="p">,</span> <span class="n">MemoryManager</span><span class="p">,</span> |
| <span class="n">default_cpu_memory_manager</span><span class="p">)</span> |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">Buffer</span><span class="p">,</span> <span class="n">ResizableBuffer</span><span class="p">,</span> <span class="n">foreign_buffer</span><span class="p">,</span> <span class="n">py_buffer</span><span class="p">,</span> |
| <span class="n">Codec</span><span class="p">,</span> <span class="n">compress</span><span class="p">,</span> <span class="n">decompress</span><span class="p">,</span> <span class="n">allocate_buffer</span><span class="p">)</span> |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">MemoryPool</span><span class="p">,</span> <span class="n">LoggingMemoryPool</span><span class="p">,</span> <span class="n">ProxyMemoryPool</span><span class="p">,</span> |
| <span class="n">total_allocated_bytes</span><span class="p">,</span> <span class="n">set_memory_pool</span><span class="p">,</span> |
| <span class="n">default_memory_pool</span><span class="p">,</span> <span class="n">system_memory_pool</span><span class="p">,</span> |
| <span class="n">jemalloc_memory_pool</span><span class="p">,</span> <span class="n">mimalloc_memory_pool</span><span class="p">,</span> |
| <span class="n">logging_memory_pool</span><span class="p">,</span> <span class="n">proxy_memory_pool</span><span class="p">,</span> |
| <span class="n">log_memory_allocations</span><span class="p">,</span> <span class="n">jemalloc_set_decay_ms</span><span class="p">,</span> |
| <span class="n">supported_memory_backends</span><span class="p">)</span> |
| |
| <span class="c1"># I/O</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">NativeFile</span><span class="p">,</span> <span class="n">PythonFile</span><span class="p">,</span> |
| <span class="n">BufferedInputStream</span><span class="p">,</span> <span class="n">BufferedOutputStream</span><span class="p">,</span> <span class="n">CacheOptions</span><span class="p">,</span> |
| <span class="n">CompressedInputStream</span><span class="p">,</span> <span class="n">CompressedOutputStream</span><span class="p">,</span> |
| <span class="n">TransformInputStream</span><span class="p">,</span> <span class="n">transcoding_input_stream</span><span class="p">,</span> |
| <span class="n">FixedSizeBufferWriter</span><span class="p">,</span> |
| <span class="n">BufferReader</span><span class="p">,</span> <span class="n">BufferOutputStream</span><span class="p">,</span> |
| <span class="n">OSFile</span><span class="p">,</span> <span class="n">MemoryMappedFile</span><span class="p">,</span> <span class="n">memory_map</span><span class="p">,</span> |
| <span class="n">create_memory_map</span><span class="p">,</span> <span class="n">MockOutputStream</span><span class="p">,</span> |
| <span class="n">input_stream</span><span class="p">,</span> <span class="n">output_stream</span><span class="p">,</span> |
| <span class="n">have_libhdfs</span><span class="p">)</span> |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">ChunkedArray</span><span class="p">,</span> <span class="n">RecordBatch</span><span class="p">,</span> <span class="n">Table</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> |
| <span class="n">concat_arrays</span><span class="p">,</span> <span class="n">concat_tables</span><span class="p">,</span> <span class="n">TableGroupBy</span><span class="p">,</span> |
| <span class="n">RecordBatchReader</span><span class="p">,</span> <span class="n">concat_batches</span><span class="p">)</span> |
| |
| <span class="c1"># Exceptions</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">ArrowCancelled</span><span class="p">,</span> |
| <span class="n">ArrowCapacityError</span><span class="p">,</span> |
| <span class="n">ArrowException</span><span class="p">,</span> |
| <span class="n">ArrowKeyError</span><span class="p">,</span> |
| <span class="n">ArrowIndexError</span><span class="p">,</span> |
| <span class="n">ArrowInvalid</span><span class="p">,</span> |
| <span class="n">ArrowIOError</span><span class="p">,</span> |
| <span class="n">ArrowMemoryError</span><span class="p">,</span> |
| <span class="n">ArrowNotImplementedError</span><span class="p">,</span> |
| <span class="n">ArrowTypeError</span><span class="p">,</span> |
| <span class="n">ArrowSerializationError</span><span class="p">)</span> |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.ipc</span><span class="w"> </span><span class="kn">import</span> <span class="n">serialize_pandas</span><span class="p">,</span> <span class="n">deserialize_pandas</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.ipc</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">ipc</span> |
| |
| <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.types</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">types</span> |
| |
| |
| <span class="c1"># ----------------------------------------------------------------------</span> |
| <span class="c1"># Deprecations</span> |
| |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.util</span><span class="w"> </span><span class="kn">import</span> <span class="n">_deprecate_api</span><span class="p">,</span> <span class="n">_deprecate_class</span> |
| |
| |
| <span class="c1"># TODO: Deprecate these somehow in the pyarrow namespace</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.ipc</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">Message</span><span class="p">,</span> <span class="n">MessageReader</span><span class="p">,</span> <span class="n">MetadataVersion</span><span class="p">,</span> |
| <span class="n">RecordBatchFileReader</span><span class="p">,</span> <span class="n">RecordBatchFileWriter</span><span class="p">,</span> |
| <span class="n">RecordBatchStreamReader</span><span class="p">,</span> <span class="n">RecordBatchStreamWriter</span><span class="p">)</span> |
| |
| <span class="c1"># ----------------------------------------------------------------------</span> |
| <span class="c1"># Returning absolute path to the pyarrow include directory (if bundled, e.g. in</span> |
| <span class="c1"># wheels)</span> |
| |
| |
| <div class="viewcode-block" id="get_include"> |
| <a class="viewcode-back" href="../python/generated/pyarrow.get_include.html#pyarrow.get_include">[docs]</a> |
| <span class="k">def</span><span class="w"> </span><span class="nf">get_include</span><span class="p">():</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Return absolute path to directory containing Arrow C++ include</span> |
| <span class="sd"> headers. Similar to numpy.get_include</span> |
| <span class="sd"> """</span> |
| <span class="c1"># Use pyarrow.lib location instead of just __file__. That works</span> |
| <span class="c1"># for both editable and non-editable builds as it points</span> |
| <span class="c1"># to the actual location of the compiled C++ extension.</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">lib</span> <span class="k">as</span> <span class="n">_lib</span> |
| <span class="k">return</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">_lib</span><span class="o">.</span><span class="vm">__file__</span><span class="p">),</span> <span class="s1">'include'</span><span class="p">)</span></div> |
| |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">_get_pkg_config_executable</span><span class="p">():</span> |
| <span class="k">return</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PKG_CONFIG'</span><span class="p">,</span> <span class="s1">'pkg-config'</span><span class="p">)</span> |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">_has_pkg_config</span><span class="p">(</span><span class="n">pkgname</span><span class="p">):</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">subprocess</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">call</span><span class="p">([</span><span class="n">_get_pkg_config_executable</span><span class="p">(),</span> |
| <span class="s1">'--exists'</span><span class="p">,</span> <span class="n">pkgname</span><span class="p">])</span> <span class="o">==</span> <span class="mi">0</span> |
| <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">False</span> |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">_read_pkg_config_variable</span><span class="p">(</span><span class="n">pkgname</span><span class="p">,</span> <span class="n">cli_args</span><span class="p">):</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">subprocess</span> |
| <span class="n">cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">_get_pkg_config_executable</span><span class="p">(),</span> <span class="n">pkgname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cli_args</span> |
| <span class="n">proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> |
| <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span> |
| <span class="n">out</span><span class="p">,</span> <span class="n">err</span> <span class="o">=</span> <span class="n">proc</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span> |
| <span class="k">if</span> <span class="n">proc</span><span class="o">.</span><span class="n">returncode</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"pkg-config failed: "</span> <span class="o">+</span> <span class="n">err</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf8'</span><span class="p">))</span> |
| <span class="k">return</span> <span class="n">out</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf8'</span><span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="get_libraries"> |
| <a class="viewcode-back" href="../python/generated/pyarrow.get_libraries.html#pyarrow.get_libraries">[docs]</a> |
| <span class="k">def</span><span class="w"> </span><span class="nf">get_libraries</span><span class="p">():</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Return list of library names to include in the `libraries` argument for C</span> |
| <span class="sd"> or Cython extensions using pyarrow</span> |
| <span class="sd"> """</span> |
| <span class="k">return</span> <span class="p">[</span><span class="s1">'arrow_python'</span><span class="p">,</span> <span class="s1">'arrow'</span><span class="p">]</span></div> |
| |
| |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">create_library_symlinks</span><span class="p">():</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> With Linux and macOS wheels, the bundled shared libraries have an embedded</span> |
| <span class="sd"> ABI version like libarrow.so.17 or libarrow.17.dylib and so linking to them</span> |
| <span class="sd"> with -larrow won't work unless we create symlinks at locations like</span> |
| <span class="sd"> site-packages/pyarrow/libarrow.so. This unfortunate workaround addresses</span> |
| <span class="sd"> prior problems we had with shipping two copies of the shared libraries to</span> |
| <span class="sd"> permit third party projects like turbodbc to build their C++ extensions</span> |
| <span class="sd"> against the pyarrow wheels.</span> |
| |
| <span class="sd"> This function must only be invoked once and only when the shared libraries</span> |
| <span class="sd"> are bundled with the Python package, which should only apply to wheel-based</span> |
| <span class="sd"> installs. It requires write access to the site-packages/pyarrow directory</span> |
| <span class="sd"> and so depending on your system may need to be run with root.</span> |
| <span class="sd"> """</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">glob</span> |
| <span class="k">if</span> <span class="n">_sys</span><span class="o">.</span><span class="n">platform</span> <span class="o">==</span> <span class="s1">'win32'</span><span class="p">:</span> |
| <span class="k">return</span> |
| <span class="n">package_cwd</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">_sys</span><span class="o">.</span><span class="n">platform</span> <span class="o">==</span> <span class="s1">'linux'</span><span class="p">:</span> |
| <span class="n">bundled_libs</span> <span class="o">=</span> <span class="n">glob</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">package_cwd</span><span class="p">,</span> <span class="s1">'*.so.*'</span><span class="p">))</span> |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">get_symlink_path</span><span class="p">(</span><span class="n">hard_path</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">hard_path</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">bundled_libs</span> <span class="o">=</span> <span class="n">glob</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">package_cwd</span><span class="p">,</span> <span class="s1">'*.*.dylib'</span><span class="p">))</span> |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">get_symlink_path</span><span class="p">(</span><span class="n">hard_path</span><span class="p">):</span> |
| <span class="k">return</span> <span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">hard_path</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">2</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span> <span class="s1">'dylib'</span><span class="p">))</span> |
| |
| <span class="k">for</span> <span class="n">lib_hard_path</span> <span class="ow">in</span> <span class="n">bundled_libs</span><span class="p">:</span> |
| <span class="n">symlink_path</span> <span class="o">=</span> <span class="n">get_symlink_path</span><span class="p">(</span><span class="n">lib_hard_path</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">symlink_path</span><span class="p">):</span> |
| <span class="k">continue</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">_os</span><span class="o">.</span><span class="n">symlink</span><span class="p">(</span><span class="n">lib_hard_path</span><span class="p">,</span> <span class="n">symlink_path</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">PermissionError</span><span class="p">:</span> |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"Tried creating symlink </span><span class="si">{}</span><span class="s2">. If you need to link to "</span> |
| <span class="s2">"bundled shared libraries, run "</span> |
| <span class="s2">"pyarrow.create_library_symlinks() as root"</span><span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="get_library_dirs"> |
| <a class="viewcode-back" href="../python/generated/pyarrow.get_library_dirs.html#pyarrow.get_library_dirs">[docs]</a> |
| <span class="k">def</span><span class="w"> </span><span class="nf">get_library_dirs</span><span class="p">():</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Return lists of directories likely to contain Arrow C++ libraries for</span> |
| <span class="sd"> linking C or Cython extensions using pyarrow</span> |
| <span class="sd"> """</span> |
| <span class="c1"># Use pyarrow.lib location instead of just __file__. That works</span> |
| <span class="c1"># for both editable and non-editable builds as it points</span> |
| <span class="c1"># to the actual location of the compiled C++ extension.</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">lib</span> <span class="k">as</span> <span class="n">_lib</span> |
| <span class="n">package_cwd</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">_lib</span><span class="o">.</span><span class="vm">__file__</span><span class="p">)</span> |
| <span class="n">library_dirs</span> <span class="o">=</span> <span class="p">[</span><span class="n">package_cwd</span><span class="p">]</span> |
| |
| <span class="k">def</span><span class="w"> </span><span class="nf">append_library_dir</span><span class="p">(</span><span class="n">library_dir</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">library_dir</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">library_dirs</span><span class="p">:</span> |
| <span class="n">library_dirs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">library_dir</span><span class="p">)</span> |
| |
| <span class="c1"># Search library paths via pkg-config. This is necessary if the user</span> |
| <span class="c1"># installed libarrow and the other shared libraries manually and they</span> |
| <span class="c1"># are not shipped inside the pyarrow package (see also ARROW-2976).</span> |
| <span class="n">pkg_config_executable</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PKG_CONFIG'</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">'pkg-config'</span> |
| <span class="k">for</span> <span class="n">pkgname</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"arrow"</span><span class="p">,</span> <span class="s2">"arrow_python"</span><span class="p">]:</span> |
| <span class="k">if</span> <span class="n">_has_pkg_config</span><span class="p">(</span><span class="n">pkgname</span><span class="p">):</span> |
| <span class="n">library_dir</span> <span class="o">=</span> <span class="n">_read_pkg_config_variable</span><span class="p">(</span><span class="n">pkgname</span><span class="p">,</span> |
| <span class="p">[</span><span class="s2">"--libs-only-L"</span><span class="p">])</span> |
| <span class="c1"># pkg-config output could be empty if Arrow is installed</span> |
| <span class="c1"># as a system package.</span> |
| <span class="k">if</span> <span class="n">library_dir</span><span class="p">:</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">library_dir</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"-L"</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <span class="s2">"pkg-config --libs-only-L returned unexpected "</span> |
| <span class="sa">f</span><span class="s2">"value </span><span class="si">{</span><span class="n">library_dir</span><span class="si">!r}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="n">append_library_dir</span><span class="p">(</span><span class="n">library_dir</span><span class="p">[</span><span class="mi">2</span><span class="p">:])</span> |
| |
| <span class="k">if</span> <span class="n">_sys</span><span class="o">.</span><span class="n">platform</span> <span class="o">==</span> <span class="s1">'win32'</span><span class="p">:</span> |
| <span class="c1"># TODO(wesm): Is this necessary, or does setuptools within a conda</span> |
| <span class="c1"># installation add Library\lib to the linker path for MSVC?</span> |
| <span class="n">python_base_install</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">_sys</span><span class="o">.</span><span class="n">executable</span><span class="p">)</span> |
| <span class="n">library_dir</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">python_base_install</span><span class="p">,</span> <span class="s1">'Library'</span><span class="p">,</span> <span class="s1">'lib'</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">library_dir</span><span class="p">,</span> <span class="s1">'arrow.lib'</span><span class="p">)):</span> |
| <span class="n">append_library_dir</span><span class="p">(</span><span class="n">library_dir</span><span class="p">)</span> |
| |
| <span class="c1"># GH-45530: Add pyarrow.libs dir containing delvewheel-mangled</span> |
| <span class="c1"># msvcp140.dll</span> |
| <span class="n">pyarrow_libs_dir</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span> |
| <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">),</span> <span class="n">_os</span><span class="o">.</span><span class="n">pardir</span><span class="p">,</span> <span class="s2">"pyarrow.libs"</span><span class="p">)</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">pyarrow_libs_dir</span><span class="p">):</span> |
| <span class="n">append_library_dir</span><span class="p">(</span><span class="n">pyarrow_libs_dir</span><span class="p">)</span> |
| |
| <span class="c1"># ARROW-4074: Allow for ARROW_HOME to be set to some other directory</span> |
| <span class="k">if</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ARROW_HOME'</span><span class="p">):</span> |
| <span class="n">append_library_dir</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'ARROW_HOME'</span><span class="p">],</span> <span class="s1">'lib'</span><span class="p">))</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="c1"># Python wheels bundle the Arrow libraries in the pyarrow directory.</span> |
| <span class="n">append_library_dir</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)))</span> |
| |
| <span class="k">return</span> <span class="n">library_dirs</span></div> |
| |
| </pre></div> |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../_static/scripts/bootstrap.js?digest=7f76b32a3354e82990f2"></script> |
| <script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=7f76b32a3354e82990f2"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2026 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.1.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized --> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.17.0. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
