| |
| <!doctype html> |
| <html lang="en" class="no-js"> |
| <head> |
| |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| |
| |
| |
| |
| |
| <link rel="next" href="examples/basic/"> |
| |
| |
| <link rel="icon" href="assets/images/favicon.png"> |
| <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.22"> |
| |
| |
| |
| <title>Apache OpenDAL™</title> |
| |
| |
| |
| <link rel="stylesheet" href="assets/stylesheets/main.84d31ad4.min.css"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback"> |
| <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style> |
| |
| |
| |
| <link rel="stylesheet" href="assets/_mkdocstrings.css"> |
| |
| <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script> |
| |
| |
| |
| |
| |
| |
| </head> |
| |
| |
| <body dir="ltr"> |
| |
| |
| <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off"> |
| <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off"> |
| <label class="md-overlay" for="__drawer"></label> |
| <div data-md-component="skip"> |
| |
| |
| <a href="#apache-opendaltm-python-binding" class="md-skip"> |
| Skip to content |
| </a> |
| |
| </div> |
| <div data-md-component="announce"> |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| <header class="md-header md-header--shadow" data-md-component="header"> |
| <nav class="md-header__inner md-grid" aria-label="Header"> |
| <a href="." title="Apache OpenDAL™" class="md-header__button md-logo" aria-label="Apache OpenDAL™" data-md-component="logo"> |
| |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg> |
| |
| </a> |
| <label class="md-header__button md-icon" for="__drawer"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg> |
| </label> |
| <div class="md-header__title" data-md-component="header-title"> |
| <div class="md-header__ellipsis"> |
| <div class="md-header__topic"> |
| <span class="md-ellipsis"> |
| Apache OpenDAL™ |
| </span> |
| </div> |
| <div class="md-header__topic" data-md-component="header-topic"> |
| <span class="md-ellipsis"> |
| |
| Overview |
| |
| </span> |
| </div> |
| </div> |
| </div> |
| |
| |
| <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script> |
| |
| |
| |
| |
| |
| <label class="md-header__button md-icon" for="__search"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg> |
| </label> |
| <div class="md-search" data-md-component="search" role="dialog"> |
| <label class="md-search__overlay" for="__search"></label> |
| <div class="md-search__inner" role="search"> |
| <form class="md-search__form" name="search"> |
| <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required> |
| <label class="md-search__icon md-icon" for="__search"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg> |
| </label> |
| <nav class="md-search__options" aria-label="Search"> |
| |
| <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg> |
| </button> |
| </nav> |
| |
| </form> |
| <div class="md-search__output"> |
| <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix> |
| <div class="md-search-result" data-md-component="search-result"> |
| <div class="md-search-result__meta"> |
| Initializing search |
| </div> |
| <ol class="md-search-result__list" role="presentation"></ol> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div class="md-header__source"> |
| <a href="https://github.com/apache/opendal" title="Go to repository" class="md-source" data-md-component="source"> |
| <div class="md-source__icon md-icon"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg> |
| </div> |
| <div class="md-source__repository"> |
| GitHub |
| </div> |
| </a> |
| </div> |
| |
| </nav> |
| |
| </header> |
| |
| <div class="md-container" data-md-component="container"> |
| |
| |
| |
| |
| |
| |
| <main class="md-main" data-md-component="main"> |
| <div class="md-main__inner md-grid"> |
| |
| |
| |
| <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" > |
| <div class="md-sidebar__scrollwrap"> |
| <div class="md-sidebar__inner"> |
| |
| |
| |
| |
| <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0"> |
| <label class="md-nav__title" for="__drawer"> |
| <a href="." title="Apache OpenDAL™" class="md-nav__button md-logo" aria-label="Apache OpenDAL™" data-md-component="logo"> |
| |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg> |
| |
| </a> |
| Apache OpenDAL™ |
| </label> |
| |
| <div class="md-nav__source"> |
| <a href="https://github.com/apache/opendal" title="Go to repository" class="md-source" data-md-component="source"> |
| <div class="md-source__icon md-icon"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg> |
| </div> |
| <div class="md-source__repository"> |
| GitHub |
| </div> |
| </a> |
| </div> |
| |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--active"> |
| |
| <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc"> |
| |
| |
| |
| |
| |
| <label class="md-nav__link md-nav__link--active" for="__toc"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Overview |
| |
| </span> |
| |
| |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| <a href="." class="md-nav__link md-nav__link--active"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Overview |
| |
| </span> |
| |
| |
| </a> |
| |
| |
| |
| <nav class="md-nav md-nav--secondary" aria-label="Table of contents"> |
| |
| |
| |
| |
| |
| |
| <label class="md-nav__title" for="__toc"> |
| <span class="md-nav__icon md-icon"></span> |
| Table of contents |
| </label> |
| <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix> |
| |
| <li class="md-nav__item"> |
| <a href="#installation" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Installation |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#local-usage" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Local Usage |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="Local Usage"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#reading" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Reading |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#writing" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Writing |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#async" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Async |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" > |
| |
| |
| <label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Examples |
| |
| </span> |
| |
| |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_2"> |
| <span class="md-nav__icon md-icon"></span> |
| Examples |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="examples/basic/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Basic |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="examples/pandas/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Pandas |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="examples/polars/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Polars |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item md-nav__item--nested"> |
| |
| |
| |
| <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" > |
| |
| |
| <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| API Reference |
| |
| </span> |
| |
| |
| <span class="md-nav__icon md-icon"></span> |
| </label> |
| |
| <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false"> |
| <label class="md-nav__title" for="__nav_3"> |
| <span class="md-nav__icon md-icon"></span> |
| API Reference |
| </label> |
| <ul class="md-nav__list" data-md-scrollfix> |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/async_file/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| AsyncFile |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/async_operator/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| AsyncOperator |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/capability/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Capability |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/exceptions/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Exceptions |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/file/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| File |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/layers/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Layers |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/operator/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Operator |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="md-nav__item"> |
| <a href="api/types/" class="md-nav__link"> |
| |
| |
| |
| <span class="md-ellipsis"> |
| Types |
| |
| </span> |
| |
| |
| </a> |
| </li> |
| |
| |
| |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| |
| |
| </ul> |
| </nav> |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" > |
| <div class="md-sidebar__scrollwrap"> |
| <div class="md-sidebar__inner"> |
| |
| |
| <nav class="md-nav md-nav--secondary" aria-label="Table of contents"> |
| |
| |
| |
| |
| |
| |
| <label class="md-nav__title" for="__toc"> |
| <span class="md-nav__icon md-icon"></span> |
| Table of contents |
| </label> |
| <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix> |
| |
| <li class="md-nav__item"> |
| <a href="#installation" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Installation |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#local-usage" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Local Usage |
| </span> |
| </a> |
| |
| <nav class="md-nav" aria-label="Local Usage"> |
| <ul class="md-nav__list"> |
| |
| <li class="md-nav__item"> |
| <a href="#reading" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Reading |
| </span> |
| </a> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#writing" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Writing |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| </nav> |
| |
| </li> |
| |
| <li class="md-nav__item"> |
| <a href="#async" class="md-nav__link"> |
| <span class="md-ellipsis"> |
| Async |
| </span> |
| </a> |
| |
| </li> |
| |
| </ul> |
| |
| </nav> |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div class="md-content" data-md-component="content"> |
| <article class="md-content__inner md-typeset"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <h1 id="apache-opendaltm-python-binding">Apache OpenDAL™ Python binding<a class="headerlink" href="#apache-opendaltm-python-binding" title="Permanent link">#</a></h1> |
| <h2 id="installation">Installation<a class="headerlink" href="#installation" title="Permanent link">#</a></h2> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>pip<span class="w"> </span>install<span class="w"> </span>opendal |
| </code></pre></div></td></tr></table></div> |
| <h2 id="local-usage">Local Usage<a class="headerlink" href="#local-usage" title="Permanent link">#</a></h2> |
| <p>Developer must set two required arguments to work with files locally: |
| - <code>scheme</code>: which should be specified as <code>fs</code> |
| - <code>root</code>: where OpenDAl considers the root of the directory for operations will be.</p> |
| <p>For example in the following operator: |
| <code>opendal.Operator("fs", root="/foo")</code></p> |
| <p>OpenDAL considers <code>/foo</code> to be the root of all paths, and that means that we can access paths inside of <code>/foo</code> without specifying anything else. |
| If <code>/foo</code> happens to contain the file <code>baz.txt</code>, we can simply call <code>.list("/")</code></p> |
| <p>We can see this in the following example:</p> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 1</span> |
| <span class="normal"> 2</span> |
| <span class="normal"> 3</span> |
| <span class="normal"> 4</span> |
| <span class="normal"> 5</span> |
| <span class="normal"> 6</span> |
| <span class="normal"> 7</span> |
| <span class="normal"> 8</span> |
| <span class="normal"> 9</span> |
| <span class="normal">10</span> |
| <span class="normal">11</span> |
| <span class="normal">12</span> |
| <span class="normal">13</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">from</span><span class="w"> </span><span class="nn">opendal</span><span class="w"> </span><span class="kn">import</span> <span class="n">Operator</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">os</span> |
| |
| |
| <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| |
| <span class="n">file</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="s2">"bar.txt"</span><span class="p">)</span> |
| <span class="n">file</span><span class="o">.</span><span class="n">touch</span><span class="p">()</span> |
| |
| <span class="n">op</span> <span class="o">=</span> <span class="n">Operator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">)</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> |
| <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">op</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="s2">"/"</span><span class="p">):</span> |
| <span class="nb">print</span><span class="p">(</span><span class="n">file</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| <p>When running we get the following output:</p> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span> |
| <span class="normal">2</span></pre></div></td><td class="code"><div><pre><span></span><code>/ |
| bar.txt |
| </code></pre></div></td></tr></table></div> |
| <p>If we want full access to our file system, we can specify a root of <code>"/"</code>, but note that for any operations you will always need to specify the full path to a file or directory.</p> |
| <h3 id="reading">Reading<a class="headerlink" href="#reading" title="Permanent link">#</a></h3> |
| <p>There are two ways to read data using OpenDAL. One way is to use the <code>read</code> method on an operator: |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 1</span> |
| <span class="normal"> 2</span> |
| <span class="normal"> 3</span> |
| <span class="normal"> 4</span> |
| <span class="normal"> 5</span> |
| <span class="normal"> 6</span> |
| <span class="normal"> 7</span> |
| <span class="normal"> 8</span> |
| <span class="normal"> 9</span> |
| <span class="normal">10</span> |
| <span class="normal">11</span> |
| <span class="normal">12</span> |
| <span class="normal">13</span> |
| <span class="normal">14</span> |
| <span class="normal">15</span> |
| <span class="normal">16</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">from</span><span class="w"> </span><span class="nn">opendal</span><span class="w"> </span><span class="kn">import</span> <span class="n">Operator</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">os</span> |
| |
| |
| <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| |
| <span class="n">file</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="s2">"bar.txt"</span><span class="p">)</span> |
| <span class="n">file</span><span class="o">.</span><span class="n">touch</span><span class="p">()</span> |
| <span class="n">file</span><span class="o">.</span><span class="n">write_text</span><span class="p">(</span><span class="s2">"baz"</span><span class="p">)</span> |
| |
| |
| <span class="n">op</span> <span class="o">=</span> <span class="n">Operator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">)</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> |
| |
| <span class="n">data</span> <span class="o">=</span> <span class="n">op</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">file</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> |
| <span class="nb">print</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div></p> |
| <p>Yields the following: |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>b'baz' |
| </code></pre></div></td></tr></table></div></p> |
| <p>Note that the output is bytes, but we can get it as a string by simply calling <code>.decode()</code> on the data we read. All reads with OpenDAL return bytes.</p> |
| <p>Now lets use the <code>open</code> method on an operator: |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 1</span> |
| <span class="normal"> 2</span> |
| <span class="normal"> 3</span> |
| <span class="normal"> 4</span> |
| <span class="normal"> 5</span> |
| <span class="normal"> 6</span> |
| <span class="normal"> 7</span> |
| <span class="normal"> 8</span> |
| <span class="normal"> 9</span> |
| <span class="normal">10</span> |
| <span class="normal">11</span> |
| <span class="normal">12</span> |
| <span class="normal">13</span> |
| <span class="normal">14</span> |
| <span class="normal">15</span> |
| <span class="normal">16</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">from</span><span class="w"> </span><span class="nn">opendal</span><span class="w"> </span><span class="kn">import</span> <span class="n">Operator</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span> |
| <span class="kn">import</span><span class="w"> </span><span class="nn">os</span> |
| |
| |
| <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| |
| <span class="n">file</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="s2">"bar.txt"</span><span class="p">)</span> |
| <span class="n">file</span><span class="o">.</span><span class="n">touch</span><span class="p">()</span> |
| <span class="n">file</span><span class="o">.</span><span class="n">write_text</span><span class="p">(</span><span class="s2">"baz"</span><span class="p">)</span> |
| |
| |
| <span class="n">op</span> <span class="o">=</span> <span class="n">Operator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">)</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> |
| |
| <span class="k">with</span> <span class="n">op</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">file</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> |
| <span class="nb">print</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span> |
| </code></pre></div></td></tr></table></div></p> |
| <p>This again yields |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>b'baz' |
| </code></pre></div></td></tr></table></div></p> |
| <p>Again, note that all reads with OpenDAL return bytes, so specifying a mode of <code>"r"</code> will raise an exception.</p> |
| <h3 id="writing">Writing<a class="headerlink" href="#writing" title="Permanent link">#</a></h3> |
| <p>Now that we know how to read data, let's replace the <code>Pathlib</code> code above with OpenDAL using the <code>write</code> method: |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span> |
| <span class="normal">2</span> |
| <span class="normal">3</span> |
| <span class="normal">4</span> |
| <span class="normal">5</span> |
| <span class="normal">6</span> |
| <span class="normal">7</span> |
| <span class="normal">8</span> |
| <span class="normal">9</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">from</span><span class="w"> </span><span class="nn">opendal</span><span class="w"> </span><span class="kn">import</span> <span class="n">Operator</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span> |
| |
| |
| <span class="n">op</span> <span class="o">=</span> <span class="n">Operator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">)</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> |
| |
| <span class="n">op</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"baz.txt"</span><span class="p">,</span> <span class="s2">"my amazing data"</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="n">op</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="s2">"baz.txt"</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span> |
| </code></pre></div></td></tr></table></div></p> |
| <p>This yields the following: |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>my amazing data |
| </code></pre></div></td></tr></table></div></p> |
| <p>And again, but using the <code>open</code> method: |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 1</span> |
| <span class="normal"> 2</span> |
| <span class="normal"> 3</span> |
| <span class="normal"> 4</span> |
| <span class="normal"> 5</span> |
| <span class="normal"> 6</span> |
| <span class="normal"> 7</span> |
| <span class="normal"> 8</span> |
| <span class="normal"> 9</span> |
| <span class="normal">10</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">from</span><span class="w"> </span><span class="nn">opendal</span><span class="w"> </span><span class="kn">import</span> <span class="n">Operator</span> |
| <span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span> |
| |
| |
| <span class="n">op</span> <span class="o">=</span> <span class="n">Operator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">Path</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">)</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> |
| |
| <span class="k">with</span> <span class="n">op</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">"baz.txt"</span><span class="p">,</span> <span class="s2">"wb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> |
| <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"my amazing data"</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span> |
| |
| <span class="nb">print</span><span class="p">(</span><span class="n">op</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="s2">"baz.txt"</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span> |
| </code></pre></div></td></tr></table></div></p> |
| <p>This again yields:</p> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>my amazing data |
| </code></pre></div></td></tr></table></div> |
| <p>Again, note that all writing happens in bytes and a mode of <code>"w"</code> will raise an exception.</p> |
| <h2 id="async">Async<a class="headerlink" href="#async" title="Permanent link">#</a></h2> |
| <p>OpenDAL supports async operation on all operator methods. One can simply replace the <code>Operator</code> with an <code>AsyncOperator</code> and await on method calls. The below example illustrates this behavior:</p> |
| <p>Standard API:</p> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span> |
| <span class="normal">2</span> |
| <span class="normal">3</span> |
| <span class="normal">4</span> |
| <span class="normal">5</span> |
| <span class="normal">6</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">import</span><span class="w"> </span><span class="nn">opendal</span> |
| |
| <span class="n">op</span> <span class="o">=</span> <span class="n">opendal</span><span class="o">.</span><span class="n">Operator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="s2">"/tmp"</span><span class="p">)</span> |
| <span class="n">op</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"test.txt"</span><span class="p">,</span> <span class="sa">b</span><span class="s2">"Hello World"</span><span class="p">)</span> |
| <span class="nb">print</span><span class="p">(</span><span class="n">op</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="s2">"test.txt"</span><span class="p">))</span> |
| <span class="nb">print</span><span class="p">(</span><span class="n">op</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="s2">"test.txt"</span><span class="p">)</span><span class="o">.</span><span class="n">content_length</span><span class="p">)</span> |
| </code></pre></div></td></tr></table></div> |
| <p>Async API equivalent:</p> |
| <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span> |
| <span class="normal">2</span> |
| <span class="normal">3</span> |
| <span class="normal">4</span> |
| <span class="normal">5</span> |
| <span class="normal">6</span> |
| <span class="normal">7</span> |
| <span class="normal">8</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="kn">import</span><span class="w"> </span><span class="nn">asyncio</span> |
| |
| <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">main</span><span class="p">():</span> |
| <span class="n">op</span> <span class="o">=</span> <span class="n">opendal</span><span class="o">.</span><span class="n">AsyncOperator</span><span class="p">(</span><span class="s2">"fs"</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="s2">"/tmp"</span><span class="p">)</span> |
| <span class="k">await</span> <span class="n">op</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">"test.txt"</span><span class="p">,</span> <span class="sa">b</span><span class="s2">"Hello World"</span><span class="p">)</span> |
| <span class="nb">print</span><span class="p">(</span><span class="k">await</span> <span class="n">op</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="s2">"test.txt"</span><span class="p">))</span> |
| |
| <span class="n">asyncio</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">main</span><span class="p">())</span> |
| </code></pre></div></td></tr></table></div> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </article> |
| </div> |
| |
| |
| <script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script> |
| </div> |
| |
| </main> |
| |
| <footer class="md-footer"> |
| |
| <div class="md-footer-meta md-typeset"> |
| <div class="md-footer-meta__inner md-grid"> |
| <div class="md-copyright"> |
| |
| <div class="md-copyright__highlight"> |
| Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0" target="_blank">Apache License, Version 2.0</a>.<br>Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. |
| </div> |
| |
| |
| Made with |
| <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener"> |
| Material for MkDocs |
| </a> |
| |
| </div> |
| |
| </div> |
| </div> |
| </footer> |
| |
| </div> |
| <div class="md-dialog" data-md-component="dialog"> |
| <div class="md-dialog__inner md-typeset"></div> |
| </div> |
| |
| |
| |
| |
| <script id="__config" type="application/json">{"base": ".", "features": ["content.code.copy"], "search": "assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script> |
| |
| |
| <script src="assets/javascripts/bundle.f55a23d4.min.js"></script> |
| |
| |
| </body> |
| </html> |
