| <!doctype html><html lang=en-US dir=ltr class="docs-wrapper plugin-docs plugin-id-default docs-version-1.3.0 docs-doc-page docs-doc-id-specification/row_format_spec" data-has-hydrated=false><head><meta charset=UTF-8><meta name=generator content="Docusaurus v3.10.1"><title data-rh=true>Row Format | Apache Fory™</title><meta data-rh=true name=viewport content="width=device-width, initial-scale=1.0"/><meta data-rh=true property=og:url content=https://fory.apache.org/docs/specification/row_format_spec /><meta data-rh=true property=og:locale content=en_US /><meta data-rh=true property=og:locale:alternate content=zh_CN /><meta data-rh=true name=docusaurus_locale content=en-US /><meta data-rh=true name=docsearch:language content=en-US /><meta data-rh=true http-equiv=Content-Security-Policy content="frame-src 'self' https://ghbtns.com/;"/><meta data-rh=true property=og:image content=https://fory.apache.org/img/logo.png /><meta data-rh=true property=og:image:width content=1500 /><meta data-rh=true property=og:image:height content=1500 /><meta data-rh=true name=twitter:card content=summary_large_image /><meta data-rh=true name=twitter:image content=https://fory.apache.org/img/logo.png /><meta data-rh=true name=docusaurus_version content=1.3.0 /><meta data-rh=true name=docusaurus_tag content=docs-default-1.3.0 /><meta data-rh=true name=docsearch:version content=1.3.0 /><meta data-rh=true name=docsearch:docusaurus_tag content=docs-default-1.3.0 /><meta data-rh=true property=og:title content="Row Format | Apache Fory™"/><meta data-rh=true name=description content=Overview /><meta data-rh=true property=og:description content=Overview /><link data-rh=true rel=icon href=/img/favicon.ico /><link data-rh=true rel=canonical href=https://fory.apache.org/docs/specification/row_format_spec /><link data-rh=true rel=alternate href=https://fory.apache.org/docs/specification/row_format_spec hreflang=en-US /><link data-rh=true rel=alternate href=https://fory.apache.org/zh-CN/docs/specification/row_format_spec hreflang=zh-CN /><link data-rh=true rel=alternate href=https://fory.apache.org/docs/specification/row_format_spec hreflang=x-default /><script data-rh=true type=application/ld+json>{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","item":"https://fory.apache.org/docs/specification/row_format_spec","name":"Row Format","position":1}]}</script><link rel=alternate type=application/rss+xml href=/blog/rss.xml title="Apache Fory™ RSS Feed"><link rel=alternate type=application/atom+xml href=/blog/atom.xml title="Apache Fory™ Atom Feed"><script type=text/javascript>"fury.apache.org"===window.location.host&&(window.location.href="https://fory.apache.org"),function(){for(var o=window.location.pathname,a=["guide","introduction","start"],n=0;n<a.length;n++){var t=a[n],i=RegExp("^((?:/[a-z]{2}-[A-Z]{2})?/docs(?:/(?:next|[0-9]+.[0-9]+))?)/docs/"+t+"(/.*)?$"),c=o.match(i);if(c){var r=c[1]+"/"+t+(c[2]||"");window.location.replace(r+window.location.search+window.location.hash);return}}}()</script><link rel=stylesheet href=/assets/css/styles.3a1edaa1.css /><script src=/assets/js/runtime~main.605b11b1.js defer></script><script src=/assets/js/main.283fea4b.js defer></script></head><body><svg style="display: none;"><defs> |
| <symbol id=theme-svg-external-link viewBox="0 0 24 24"><path fill=currentColor d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"/></symbol> |
| </defs></svg> |
| <script>!function(){var t=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return window.localStorage.getItem("theme")}catch(t){}}();document.documentElement.setAttribute("data-theme",t||"light"),document.documentElement.setAttribute("data-theme-choice",t||"light")}(),function(){try{for(var[t,e]of new URLSearchParams(window.location.search).entries())if(t.startsWith("docusaurus-data-")){var a=t.replace("docusaurus-data-","data-");document.documentElement.setAttribute(a,e)}}catch(t){}}()</script><div id=__docusaurus><div role=region aria-label="Skip to main content"><a class=skipToContent_fXgn href=#__docusaurus_skipToContent_fallback>Skip to main content</a></div><nav aria-label=Main class="theme-layout-navbar navbar navbar--fixed-top"><div class=navbar__inner><div class="theme-layout-navbar-left navbar__items"><button aria-label="Toggle navigation bar" aria-expanded=false class="navbar__toggle clean-btn" type=button><svg width=30 height=30 viewBox="0 0 30 30" aria-hidden=true><path stroke=currentColor stroke-linecap=round stroke-miterlimit=10 stroke-width=2 d="M4 7h22M4 15h22M4 23h22"/></svg></button><a class=navbar__brand href=/><div class=navbar__logo><img src=/img/fory-logo-light.png alt="Apache Fory™ Logo" class="themedComponent_mlkZ themedComponent--light_NVdE"/><img src=/img/fory-logo-dark.png alt="Apache Fory™ Logo" class="themedComponent_mlkZ themedComponent--dark_xIcU"/></div><b class="navbar__title text--truncate"></b></a><a class="navbar__item navbar__link" href=/docs/introduction/overview>Docs</a><a aria-current=page class="navbar__item navbar__link navbar__link--active" href=/docs/specification/xlang_serialization_spec>Specification</a><a class="navbar__item navbar__link" href=/docs/community/>Community</a><a class="navbar__item navbar__link" href=/user>Users</a><a class="navbar__item navbar__link" href=/download>Download</a><a class="navbar__item navbar__link" href=/blog>Blog</a></div><div class="theme-layout-navbar-right navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href=# aria-haspopup=true aria-expanded=false role=button class=navbar__link>ASF</a><ul class=dropdown__menu><li><a href=https://www.apache.org/ target=_blank rel="noopener noreferrer" class=dropdown__link>Foundation</a><li><a href=https://www.apache.org/licenses/ target=_blank rel="noopener noreferrer" class=dropdown__link>License</a><li><a href=https://www.apache.org/events/current-event.html target=_blank rel="noopener noreferrer" class=dropdown__link>Events</a><li><a href=https://privacy.apache.org/policies/privacy-policy-public.html target=_blank rel="noopener noreferrer" class=dropdown__link>Privacy</a><li><a href=https://www.apache.org/security/ target=_blank rel="noopener noreferrer" class=dropdown__link>Security</a><li><a href=https://www.apache.org/foundation/sponsorship.html target=_blank rel="noopener noreferrer" class=dropdown__link>Sponsorship</a><li><a href=https://www.apache.org/foundation/thanks.html target=_blank rel="noopener noreferrer" class=dropdown__link>Thanks</a><li><a href=https://www.apache.org/foundation/policies/conduct.html target=_blank rel="noopener noreferrer" class=dropdown__link>Code of Conduct</a></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class=navbar__link aria-haspopup=true aria-expanded=false role=button href=/docs/specification/row_format_spec>1.3.0</a><ul class=dropdown__menu><li><a class=dropdown__link href=/docs/next/specification/row_format_spec>dev</a><li><a aria-current=page class="dropdown__link dropdown__link--active" href=/docs/specification/row_format_spec>1.3.0</a><li><a class=dropdown__link href=/docs/1.2.0/specification/row_format_spec>1.2.0</a><li><a class=dropdown__link href=/docs/1.1.0/specification/row_format_spec>1.1.0</a><li><a class=dropdown__link href=/docs/1.0.0/specification/row_format_spec>1.0.0</a><li><a class=dropdown__link href=/docs/0.17/specification/row_format_spec>0.17</a><li><a class=dropdown__link href=/docs/0.16/specification/row_format_spec>0.16</a><li><a class=dropdown__link href=/docs/0.15/specification/row_format_spec>0.15</a><li><a class=dropdown__link href=/docs/0.14/specification/row_format_spec>0.14</a><li><a class=dropdown__link href=/docs/0.13/specification/row_format_spec>0.13</a><li><a class=dropdown__link href=/docs/0.12/specification/row_format_spec>0.12</a><li><a class=dropdown__link href=/docs/0.11/specification/row_format_spec>0.11</a><li><a class=dropdown__link href=/docs/0.10/specification/row_format_spec>0.10</a></ul></div><a href=https://github.com/apache/fory target=_blank rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href=# aria-haspopup=true aria-expanded=false role=button class=navbar__link><svg viewBox="0 0 24 24" width=20 height=20 aria-hidden=true class=iconLanguage_nlXk><path fill=currentColor d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"/></svg>English</a><ul class=dropdown__menu><li><a href=/docs/specification/row_format_spec target=_self rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang=en-US>English</a><li><a href=/zh-CN/docs/specification/row_format_spec target=_self rel="noopener noreferrer" class=dropdown__link lang=zh-CN>简体中文</a></ul></div><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type=button disabled title="system mode" aria-label="Switch between dark and light mode (currently system mode)"><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP lightToggleIcon_pyhR"><path fill=currentColor d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP darkToggleIcon_wfgR"><path fill=currentColor d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP systemToggleIcon_QzmC"><path fill=currentColor d="m12 21c4.971 0 9-4.029 9-9s-4.029-9-9-9-9 4.029-9 9 4.029 9 9 9zm4.95-13.95c1.313 1.313 2.05 3.093 2.05 4.95s-0.738 3.637-2.05 4.95c-1.313 1.313-3.093 2.05-4.95 2.05v-14c1.857 0 3.637 0.737 4.95 2.05z"/></svg></button></div><div class=navbarSearchContainer_Bca1><div class=navbar__search><span aria-label="expand searchbar" role=button class=search-icon tabindex=0></span><input id=search_input_react type=search placeholder=Loading... aria-label=Search class="navbar__search-input search-bar" disabled/></div></div></div></div><div role=presentation class=navbar-sidebar__backdrop></div></nav><div id=__docusaurus_skipToContent_fallback class="theme-layout-main main-wrapper mainWrapper_z2l0"><div class=docsWrapper_hBAB><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type=button></button><div class=docRoot_UBD9><aside class="theme-doc-sidebar-container docSidebarContainer_YfHR"><div class=sidebarViewport_aRkj><div class=sidebar_njMd><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/specification/xlang_serialization_spec><span title="Xlang Serialization Format" class=linkLabel_WmDU>Xlang Serialization Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/specification/java_serialization_spec><span title="Java Serialization Format" class=linkLabel_WmDU>Java Serialization Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current=page href=/docs/specification/row_format_spec><span title="Row Format" class=linkLabel_WmDU>Row Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/specification/xlang_type_mapping><span title="Xlang Type Mapping" class=linkLabel_WmDU>Xlang Type Mapping</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/specification/xlang_implementation_guide><span title="Xlang Implementation Guide" class=linkLabel_WmDU>Xlang Implementation Guide</span></a></ul></nav></div></div></aside><main class=docMainContainer_TBSr><div class="container padding-top--md padding-bottom--lg"><div class=row><div class="col docItemCol_VOVn"><div class=docItemContainer_Djhp><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label=Breadcrumbs><ul class=breadcrumbs><li class=breadcrumbs__item><a aria-label="Home page" class=breadcrumbs__link href=/><svg viewBox="0 0 24 24" class=breadcrumbHomeIcon_YNFT><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill=currentColor /></svg></a><li class="breadcrumbs__item breadcrumbs__item--active"><span class=breadcrumbs__link>Row Format</span></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.3.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type=button class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Row Format</h1></header><h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=overview>Overview<a href=#overview class=hash-link aria-label="Direct link to Overview" title="Direct link to Overview" translate=no></a></h2> |
| <p>Apache Fory Row Format is a cache-friendly, random-access binary format designed for high-performance data processing. Unlike traditional serialization formats that require full deserialization, the row format enables:</p> |
| <ul> |
| <li class=""><strong>Random Field Access</strong>: Read individual fields without deserializing the entire row</li> |
| <li class=""><strong>Zero-Copy Operations</strong>: Direct memory access without data transformation</li> |
| <li class=""><strong>Cache-Friendly Layout</strong>: Optimized memory layout for CPU cache efficiency</li> |
| <li class=""><strong>Cross-Language Support</strong>: Consistent binary format across Java, C++, and Python</li> |
| </ul> |
| <p>Fory provides two row format variants:</p> |
| <table><thead><tr><th>Format<th>Languages<th>Use Case<tbody><tr><td>Standard Format<td>Java, C++, Python<td>Cross-language compatibility<tr><td>Compact Format<td>Java only<td>Space efficiency, smaller rows</table> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=format-comparison>Format Comparison<a href=#format-comparison class=hash-link aria-label="Direct link to Format Comparison" title="Direct link to Format Comparison" translate=no></a></h2> |
| <table><thead><tr><th>Feature<th>Standard Format<th>Compact Format<tbody><tr><td>Field Slot Size<td>Fixed 8 bytes<td>Natural width (1, 2, 4, or 8 bytes)<tr><td>Null Bitmap Size<td>8-byte aligned<td>Byte-aligned, can borrow padding<tr><td>Null Bitmap Position<td>Before field slots<td>After field slots (at end)<tr><td>Fixed-Size Structs<td>Variable region (offset+size)<td>Inline in fixed region<tr><td>Field Ordering<td>Schema-defined order<td>Sorted by alignment<tr><td>All Non-Nullable<td>Bitmap still present<td>Bitmap skipped entirely<tr><td>Alignment<td>Strict 8-byte<td>Relaxed (2, 4, or 8-byte)</table> |
| <hr/> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=standard-row-format>Standard Row Format<a href=#standard-row-format class=hash-link aria-label="Direct link to Standard Row Format" title="Direct link to Standard Row Format" translate=no></a></h2> |
| <p>The standard format prioritizes cross-language compatibility and simplicity with uniform 8-byte field slots.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=design-principles>Design Principles<a href=#design-principles class=hash-link aria-label="Direct link to Design Principles" title="Direct link to Design Principles" translate=no></a></h3> |
| <ol> |
| <li class=""><strong>8-Byte Alignment</strong>: All major structures are aligned to 8-byte boundaries for optimal memory access</li> |
| <li class=""><strong>Fixed-Width Field Slots</strong>: Every field uses an 8-byte slot for uniform offset calculation</li> |
| <li class=""><strong>Null Bitmap</strong>: Compact null tracking using bit vectors</li> |
| <li class=""><strong>Relative Offsets</strong>: Variable-length data uses relative offsets for sub-buffer navigation</li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=row-binary-layout>Row Binary Layout<a href=#row-binary-layout class=hash-link aria-label="Direct link to Row Binary Layout" title="Direct link to Row Binary Layout" translate=no></a></h3> |
| <p>A row stores structured data with the following layout:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">+----------------+------------------+------------------+-----+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Null Bitmap | Field 0 Slot | Field 1 Slot | ... | Field N-1 Slot | Variable Data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+------------------+------------------+-----+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| B bytes | 8 bytes | 8 bytes | | 8 bytes | Variable size |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=null-bitmap>Null Bitmap<a href=#null-bitmap class=hash-link aria-label="Direct link to Null Bitmap" title="Direct link to Null Bitmap" translate=no></a></h4> |
| <p>The null bitmap tracks which fields contain null values:</p> |
| <ul> |
| <li class=""><strong>Size</strong>: <code>((num_fields + 63) / 64) * 8</code> bytes (rounded up to nearest 8-byte word)</li> |
| <li class=""><strong>Encoding</strong>: Each bit corresponds to a field index<!-- --> |
| <ul> |
| <li class="">Bit value <code>1</code> = field is null</li> |
| <li class="">Bit value <code>0</code> = field is not null</li> |
| </ul> |
| </li> |
| <li class=""><strong>Bit Order</strong>: Bit 0 of the first byte corresponds to field 0</li> |
| </ul> |
| <p><strong>Example</strong>: For 10 fields, bitmap size = <code>((10 + 63) / 64) * 8 = 8</code> bytes</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-slots>Field Slots<a href=#field-slots class=hash-link aria-label="Direct link to Field Slots" title="Direct link to Field Slots" translate=no></a></h4> |
| <p>Each field occupies a fixed 8-byte slot regardless of its actual data type:</p> |
| <ul> |
| <li class=""><strong>Slot Offset</strong>: <code>bitmap_size + field_index * 8</code></li> |
| <li class=""><strong>Total Fixed Region</strong>: <code>bitmap_size + num_fields * 8</code> bytes</li> |
| </ul> |
| <p><strong>Field Slot Contents by Type</strong>:</p> |
| <table><thead><tr><th>Type Category<th>Slot Contents<tbody><tr><td>Fixed-width<td>Value stored directly (zero-padded)<tr><td>Variable-width<td>Offset + Size encoding (see below)</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=variable-width-data-encoding>Variable-Width Data Encoding<a href=#variable-width-data-encoding class=hash-link aria-label="Direct link to Variable-Width Data Encoding" title="Direct link to Variable-Width Data Encoding" translate=no></a></h4> |
| <p>Variable-length fields (strings, arrays, maps, nested structs) store an offset-size pair in their slot:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">+---------------------------+---------------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Relative Offset | Size |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| (32 bits) | (32 bits) |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+---------------------------+---------------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">|<-------------- 64-bit field slot value -------------->|</span><br/></div></code></pre></div></div> |
| <ul> |
| <li class=""><strong>Relative Offset</strong> (upper 32 bits): Offset from the row's base address</li> |
| <li class=""><strong>Size</strong> (lower 32 bits): Size of the variable-width data in bytes</li> |
| </ul> |
| <p><strong>Encoding</strong>:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">offset_and_size = (relative_offset << 32) | size</span><br/></div></code></pre></div></div> |
| <p><strong>Decoding</strong>:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">relative_offset = (offset_and_size >> 32) & 0xFFFFFFFF</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">size = offset_and_size & 0xFFFFFFFF</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=variable-data-region>Variable Data Region<a href=#variable-data-region class=hash-link aria-label="Direct link to Variable Data Region" title="Direct link to Variable Data Region" translate=no></a></h4> |
| <p>Variable-length data is stored after the fixed region:</p> |
| <ul> |
| <li class="">Data is written sequentially as fields are set</li> |
| <li class="">Each variable-length value is padded to 8-byte alignment</li> |
| <li class="">Padding bytes are zeroed for deterministic output</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=array-binary-layout>Array Binary Layout<a href=#array-binary-layout class=hash-link aria-label="Direct link to Array Binary Layout" title="Direct link to Array Binary Layout" translate=no></a></h3> |
| <p>Arrays store homogeneous sequences of elements:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Element Count | Null Bitmap | Element Data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| 8 bytes | B bytes | Variable size |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=array-header>Array Header<a href=#array-header class=hash-link aria-label="Direct link to Array Header" title="Direct link to Array Header" translate=no></a></h4> |
| <table><thead><tr><th>Field<th>Size<th>Description<tbody><tr><td>Element Count<td>8 bytes<td>Number of elements (uint64)<tr><td>Null Bitmap<td><code>((count + 63) / 64) * 8</code> bytes<td>Per-element null flags</table> |
| <p><strong>Header Size</strong>: <code>8 + ((num_elements + 63) / 64) * 8</code> bytes</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=array-element-data>Array Element Data<a href=#array-element-data class=hash-link aria-label="Direct link to Array Element Data" title="Direct link to Array Element Data" translate=no></a></h4> |
| <p>Elements are stored contiguously after the header:</p> |
| <ul> |
| <li class=""><strong>Fixed-width elements</strong>: Stored with their natural width (1, 2, 4, or 8 bytes)</li> |
| <li class=""><strong>Variable-width elements</strong>: Stored as 8-byte offset+size pairs</li> |
| </ul> |
| <p><strong>Element Offset</strong>: <code>header_size + element_index * element_size</code></p> |
| <p><strong>Data Region Size</strong>: Rounded up to nearest 8-byte boundary</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=array-element-sizes>Array Element Sizes<a href=#array-element-sizes class=hash-link aria-label="Direct link to Array Element Sizes" title="Direct link to Array Element Sizes" translate=no></a></h4> |
| <table><thead><tr><th>Element Type<th>Element Size<tbody><tr><td>bool<td>1 byte<tr><td>int8<td>1 byte<tr><td>int16<td>2 bytes<tr><td>int32<td>4 bytes<tr><td>int64<td>8 bytes<tr><td>float32<td>4 bytes<tr><td>float64<td>8 bytes<tr><td>string/binary<td>8 bytes (offset+size)<tr><td>array/map/struct<td>8 bytes (offset+size)</table> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=map-binary-layout>Map Binary Layout<a href=#map-binary-layout class=hash-link aria-label="Direct link to Map Binary Layout" title="Direct link to Map Binary Layout" translate=no></a></h3> |
| <p>Maps store key-value pairs as two separate arrays:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Keys Array Size | Keys Array | Values Array |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| 8 bytes | Variable size | Variable size |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=map-structure>Map Structure<a href=#map-structure class=hash-link aria-label="Direct link to Map Structure" title="Direct link to Map Structure" translate=no></a></h4> |
| <table><thead><tr><th>Field<th>Size<th>Description<tbody><tr><td>Keys Array Size<td>8 bytes<td>Total size of keys array in bytes<tr><td>Keys Array<td>Variable<td>Full array structure for keys<tr><td>Values Array<td>Variable<td>Full array structure for values</table> |
| <p><strong>Keys Array Offset</strong>: <code>base_offset + 8</code> |
| <strong>Values Array Offset</strong>: <code>base_offset + 8 + keys_array_size</code></p> |
| <p>Both keys and values arrays follow the standard array binary layout.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=nested-struct-layout>Nested Struct Layout<a href=#nested-struct-layout class=hash-link aria-label="Direct link to Nested Struct Layout" title="Direct link to Nested Struct Layout" translate=no></a></h3> |
| <p>Nested structs are stored as complete row structures within the variable data region:</p> |
| <ol> |
| <li class="">Parent field slot contains offset+size pointing to nested row</li> |
| <li class="">Nested row has its own null bitmap and field slots</li> |
| <li class="">Supports arbitrary nesting depth</li> |
| </ol> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Parent Row:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Null Bitmap | ... Slots ... | Nested Row Data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> | ^</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> | offset+size |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> +------------------->+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">Nested Row:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Null Bitmap | Field Slots | Variable Data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+------------------+------------------+</span><br/></div></code></pre></div></div> |
| <hr/> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-row-format-java-only>Compact Row Format (Java Only)<a href=#compact-row-format-java-only class=hash-link aria-label="Direct link to Compact Row Format (Java Only)" title="Direct link to Compact Row Format (Java Only)" translate=no></a></h2> |
| <p>The compact format provides space-efficient encoding with additional optimizations. It is currently implemented in Java only.</p> |
| <blockquote> |
| <p><strong>Note</strong>: The compact format is still under development and may not be stable yet.</p> |
| </blockquote> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=design-principles-1>Design Principles<a href=#design-principles-1 class=hash-link aria-label="Direct link to Design Principles" title="Direct link to Design Principles" translate=no></a></h3> |
| <ol> |
| <li class=""><strong>Natural Width Storage</strong>: Fixed-size fields use their natural byte width instead of 8 bytes</li> |
| <li class=""><strong>Alignment-Based Field Sorting</strong>: Fields are sorted by alignment requirements to minimize padding</li> |
| <li class=""><strong>Conditional Null Bitmap</strong>: Null bitmap is omitted when all fields are non-nullable</li> |
| <li class=""><strong>Inline Fixed-Size Structs</strong>: Nested structs with all fixed-size fields are stored inline</li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-row-binary-layout>Compact Row Binary Layout<a href=#compact-row-binary-layout class=hash-link aria-label="Direct link to Compact Row Binary Layout" title="Direct link to Compact Row Binary Layout" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+-----+------------------+----------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Field 0 Value | Field 1 Value | ... | Field N-1 Value | Null Bitmap | Variable Data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+-----+------------------+----------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| W0 bytes | W1 bytes | | WN-1 bytes | B bytes (opt) | Variable size |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=key-differences-from-standard-format>Key Differences from Standard Format<a href=#key-differences-from-standard-format class=hash-link aria-label="Direct link to Key Differences from Standard Format" title="Direct link to Key Differences from Standard Format" translate=no></a></h4> |
| <ol> |
| <li class=""><strong>Field Slot Sizes</strong>: Each field uses its natural width (Wi = type width or 8 for variable)</li> |
| <li class=""><strong>Null Bitmap Position</strong>: Placed after field slots, can borrow alignment padding</li> |
| <li class=""><strong>Field Order</strong>: Fields are sorted by alignment (8-byte → 4-byte → 2-byte → 1-byte → variable)</li> |
| <li class=""><strong>Conditional Bitmap</strong>: Skipped entirely if all fields are non-nullable</li> |
| </ol> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=null-bitmap-compact>Null Bitmap (Compact)<a href=#null-bitmap-compact class=hash-link aria-label="Direct link to Null Bitmap (Compact)" title="Direct link to Null Bitmap (Compact)" translate=no></a></h4> |
| <ul> |
| <li class=""><strong>Size</strong>: <code>(num_nullable_fields + 7) / 8</code> bytes (byte-aligned, not 8-byte aligned)</li> |
| <li class=""><strong>Skipped</strong>: When all fields are primitive/non-nullable</li> |
| <li class=""><strong>Position</strong>: After all fixed-size field slots, can use alignment padding space</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-sorting-algorithm>Field Sorting Algorithm<a href=#field-sorting-algorithm class=hash-link aria-label="Direct link to Field Sorting Algorithm" title="Direct link to Field Sorting Algorithm" translate=no></a></h4> |
| <p>Fields are sorted to minimize padding and optimize alignment:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Priority order (highest to lowest):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">1. Fields with 8-byte alignment (int64, float64, variable-width)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">2. Fields with 4-byte alignment (int32, float32)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">3. Fields with 2-byte alignment (int16)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">4. Fields with 1-byte alignment (int8, bool)</span><br/></div></code></pre></div></div> |
| <p>Within each alignment group, larger fields come first.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=fixed-size-struct-inlining>Fixed-Size Struct Inlining<a href=#fixed-size-struct-inlining class=hash-link aria-label="Direct link to Fixed-Size Struct Inlining" title="Direct link to Fixed-Size Struct Inlining" translate=no></a></h4> |
| <p>Nested structs with all fixed-size fields are stored inline in the parent row:</p> |
| <p><strong>Standard Format</strong> (nested struct with 2 int32 fields):</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Parent slot: [offset (4 bytes) | size (4 bytes)] → Points to nested row (8+ bytes elsewhere)</span><br/></div></code></pre></div></div> |
| <p><strong>Compact Format</strong> (same nested struct):</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Parent slot: [int32 field 0 | int32 field 1] → 8 bytes total, inline</span><br/></div></code></pre></div></div> |
| <p>This eliminates the offset+size indirection for fixed-size nested structures.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=fixed-width-calculation>Fixed-Width Calculation<a href=#fixed-width-calculation class=hash-link aria-label="Direct link to Fixed-Width Calculation" title="Direct link to Fixed-Width Calculation" translate=no></a></h4> |
| <p>A field's fixed width is determined recursively:</p> |
| <ul> |
| <li class=""><strong>Primitive types</strong>: Natural byte width (1, 2, 4, or 8)</li> |
| <li class=""><strong>Struct types</strong>: Sum of all child fixed widths (if all children are fixed-width)</li> |
| <li class=""><strong>Variable types</strong> (string, array, map): Returns -1 (uses 8-byte offset+size slot)</li> |
| </ul> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">fixed_width(field) =</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if primitive: type_width</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if struct and all_children_fixed: header_bytes + sum(fixed_width(child) for each child)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else: -1 (variable, uses 8-byte slot)</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-array-binary-layout>Compact Array Binary Layout<a href=#compact-array-binary-layout class=hash-link aria-label="Direct link to Compact Array Binary Layout" title="Direct link to Compact Array Binary Layout" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| Element Count | Null Bitmap | Element Data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+------------------+------------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| 4 bytes | B bytes (opt) | Variable size |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-array-header>Compact Array Header<a href=#compact-array-header class=hash-link aria-label="Direct link to Compact Array Header" title="Direct link to Compact Array Header" translate=no></a></h4> |
| <table><thead><tr><th>Field<th>Size<th>Description<tbody><tr><td>Element Count<td>4 bytes<td>Number of elements (int32)<tr><td>Null Bitmap<td><code>(count + 7) / 8</code> bytes (opt)<td>Per-element null flags</table> |
| <p><strong>Header Size Calculation</strong>:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">header_size = 4 + (element_nullable ? (num_elements + 7) / 8 : 0)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">// Round to 8-byte boundary only if element width is 8-byte aligned</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">if (fixed_width % 8 == 0):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> header_size = round_to_8_bytes(header_size)</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=key-differences-from-standard-array>Key Differences from Standard Array<a href=#key-differences-from-standard-array class=hash-link aria-label="Direct link to Key Differences from Standard Array" title="Direct link to Key Differences from Standard Array" translate=no></a></h4> |
| <ol> |
| <li class=""><strong>Element Count</strong>: 4 bytes instead of 8 bytes</li> |
| <li class=""><strong>Null Bitmap</strong>: Byte-aligned, skipped if elements are non-nullable</li> |
| <li class=""><strong>Fixed-Size Structs</strong>: Inline storage for fixed-width struct elements</li> |
| </ol> |
| <hr/> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=common-specifications>Common Specifications<a href=#common-specifications class=hash-link aria-label="Direct link to Common Specifications" title="Direct link to Common Specifications" translate=no></a></h2> |
| <p>The following specifications apply to both standard and compact formats.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-encoding>Type Encoding<a href=#type-encoding class=hash-link aria-label="Direct link to Type Encoding" title="Direct link to Type Encoding" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=primitive-types>Primitive Types<a href=#primitive-types class=hash-link aria-label="Direct link to Primitive Types" title="Direct link to Primitive Types" translate=no></a></h4> |
| <table><thead><tr><th>Type<th>Width<th>Encoding<tbody><tr><td>bool<td>1 byte<td><code>0x00</code> (false) or <code>0x01</code> (true)<tr><td>int8<td>1 byte<td>Two's complement<tr><td>int16<td>2 bytes<td>Two's complement, little-endian<tr><td>int32<td>4 bytes<td>Two's complement, little-endian<tr><td>int64<td>8 bytes<td>Two's complement, little-endian<tr><td>float32<td>4 bytes<td>IEEE 754 single precision<tr><td>float64<td>8 bytes<td>IEEE 754 double precision</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=temporal-types>Temporal Types<a href=#temporal-types class=hash-link aria-label="Direct link to Temporal Types" title="Direct link to Temporal Types" translate=no></a></h4> |
| <table><thead><tr><th>Type<th>Width<th>Encoding<tbody><tr><td>timestamp<td>8 bytes<td>Microseconds since Unix epoch (int64)<tr><td>date32<td>4 bytes<td>Days since Unix epoch (int32)<tr><td>duration<td>8 bytes<td>Duration in microseconds (int64)</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=string-and-binary>String and Binary<a href=#string-and-binary class=hash-link aria-label="Direct link to String and Binary" title="Direct link to String and Binary" translate=no></a></h4> |
| <ul> |
| <li class=""><strong>Encoding</strong>: UTF-8 for strings, raw bytes for binary</li> |
| <li class=""><strong>Storage</strong>: Offset+size pair in field slot, data in variable region</li> |
| <li class=""><strong>Padding</strong>: Data padded to 8-byte alignment (standard) or natural alignment (compact)</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=null-handling>Null Handling<a href=#null-handling class=hash-link aria-label="Direct link to Null Handling" title="Direct link to Null Handling" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=row-null-handling>Row Null Handling<a href=#row-null-handling class=hash-link aria-label="Direct link to Row Null Handling" title="Direct link to Row Null Handling" translate=no></a></h4> |
| <ul> |
| <li class="">Null fields have their corresponding bit set to 1 in the null bitmap</li> |
| <li class="">Field slot contents are undefined for null fields (standard) or zeroed (compact)</li> |
| <li class="">Reading a null field returns a null/empty value indicator</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=array-null-handling>Array Null Handling<a href=#array-null-handling class=hash-link aria-label="Direct link to Array Null Handling" title="Direct link to Array Null Handling" translate=no></a></h4> |
| <ul> |
| <li class="">Null elements have their corresponding bit set to 1 in the array's null bitmap</li> |
| <li class="">Element data is undefined for null elements</li> |
| <li class="">Compact format: Bitmap skipped if elements are non-nullable</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=variable-width-null-semantics>Variable-Width Null Semantics<a href=#variable-width-null-semantics class=hash-link aria-label="Direct link to Variable-Width Null Semantics" title="Direct link to Variable-Width Null Semantics" translate=no></a></h4> |
| <p>When reading variable-width data from a null field:</p> |
| <ul> |
| <li class="">Returns size of -1 or equivalent null indicator</li> |
| <li class="">No data access is performed</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=alignment-and-padding>Alignment and Padding<a href=#alignment-and-padding class=hash-link aria-label="Direct link to Alignment and Padding" title="Direct link to Alignment and Padding" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=standard-format-alignment>Standard Format Alignment<a href=#standard-format-alignment class=hash-link aria-label="Direct link to Standard Format Alignment" title="Direct link to Standard Format Alignment" translate=no></a></h4> |
| <ol> |
| <li class=""><strong>Null Bitmap</strong>: Size rounded to 8-byte boundary</li> |
| <li class=""><strong>Field Slots</strong>: Always 8 bytes each</li> |
| <li class=""><strong>Variable Data</strong>: Each value padded to 8-byte boundary</li> |
| <li class=""><strong>Array Data</strong>: Total data region padded to 8-byte boundary</li> |
| </ol> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-format-alignment>Compact Format Alignment<a href=#compact-format-alignment class=hash-link aria-label="Direct link to Compact Format Alignment" title="Direct link to Compact Format Alignment" translate=no></a></h4> |
| <ol> |
| <li class=""><strong>Field Slots</strong>: Natural width (1, 2, 4, or 8 bytes)</li> |
| <li class=""><strong>Null Bitmap</strong>: Byte-aligned, placed after fields</li> |
| <li class=""><strong>Variable Data</strong>: Padded to 8-byte boundary only when needed</li> |
| <li class=""><strong>Header</strong>: May use relaxed alignment for smaller overhead</li> |
| </ol> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=padding-bytes>Padding Bytes<a href=#padding-bytes class=hash-link aria-label="Direct link to Padding Bytes" title="Direct link to Padding Bytes" translate=no></a></h4> |
| <ul> |
| <li class="">All padding bytes must be set to zero</li> |
| <li class="">Ensures deterministic serialization output</li> |
| <li class="">Prevents information leakage from uninitialized memory</li> |
| </ul> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=size-calculations>Size Calculations<a href=#size-calculations class=hash-link aria-label="Direct link to Size Calculations" title="Direct link to Size Calculations" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=standard-row-size>Standard Row Size<a href=#standard-row-size class=hash-link aria-label="Direct link to Standard Row Size" title="Direct link to Standard Row Size" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">row_size = bitmap_size + num_fields * 8 + variable_data_size</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">where:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bitmap_size = ((num_fields + 63) / 64) * 8</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> variable_data_size = sum of (padded_size for each variable field)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> padded_size = ((size + 7) / 8) * 8</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-row-size>Compact Row Size<a href=#compact-row-size class=hash-link aria-label="Direct link to Compact Row Size" title="Direct link to Compact Row Size" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">row_size = fixed_region_size + bitmap_size + variable_data_size</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">where:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fixed_region_size = sum of (fixed_width(field) or 8 for each field)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bitmap_size = all_non_nullable ? 0 : (num_nullable_fields + 7) / 8</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // May be rounded to 8-byte boundary if has variable fields</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=standard-array-size>Standard Array Size<a href=#standard-array-size class=hash-link aria-label="Direct link to Standard Array Size" title="Direct link to Standard Array Size" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">array_size = header_size + data_size</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">where:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> header_size = 8 + ((num_elements + 63) / 64) * 8</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> data_size = ((num_elements * element_size + 7) / 8) * 8</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=compact-array-size>Compact Array Size<a href=#compact-array-size class=hash-link aria-label="Direct link to Compact Array Size" title="Direct link to Compact Array Size" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">array_size = header_size + data_size</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">where:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> header_size = 4 + (element_nullable ? (num_elements + 7) / 8 : 0)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // header_size rounded to 8 if element_width % 8 == 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> data_size = num_elements * element_width</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=map-size>Map Size<a href=#map-size class=hash-link aria-label="Direct link to Map Size" title="Direct link to Map Size" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">map_size = 8 + keys_array_size + values_array_size</span><br/></div></code></pre></div></div> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=summary-tables>Summary Tables<a href=#summary-tables class=hash-link aria-label="Direct link to Summary Tables" title="Direct link to Summary Tables" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=layout-summary>Layout Summary<a href=#layout-summary class=hash-link aria-label="Direct link to Layout Summary" title="Direct link to Layout Summary" translate=no></a></h3> |
| <table><thead><tr><th>Component<th>Standard Format<th>Compact Format<tbody><tr><td>Row Header<td><code>((N + 63) / 64) * 8</code> bytes<td>0 or <code>(N + 7) / 8</code> bytes (at end)<tr><td>Row Field Slots<td><code>N * 8</code> bytes<td><code>sum(field_widths)</code> bytes<tr><td>Array Header<td><code>8 + ((E + 63) / 64) * 8</code> bytes<td><code>4 + (E + 7) / 8</code> bytes (if nullable)<tr><td>Array Elements<td><code>E * element_size</code> (8-aligned)<td><code>E * element_width</code><tr><td>Map Header<td>8 bytes<td>8 bytes<tr><td>Offset+Size Pair<td>8 bytes (32-bit offset + size)<td>8 bytes (same)</table> |
| <p>Where N = number of fields, E = number of elements</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-width-summary>Type Width Summary<a href=#type-width-summary class=hash-link aria-label="Direct link to Type Width Summary" title="Direct link to Type Width Summary" translate=no></a></h3> |
| <table><thead><tr><th>Category<th>Storage Width<th>Standard Slot<th>Compact Slot<tbody><tr><td>bool<td>1 byte<td>8 bytes<td>1 byte<tr><td>int8<td>1 byte<td>8 bytes<td>1 byte<tr><td>int16<td>2 bytes<td>8 bytes<td>2 bytes<tr><td>int32<td>4 bytes<td>8 bytes<td>4 bytes<tr><td>int64<td>8 bytes<td>8 bytes<td>8 bytes<tr><td>float32<td>4 bytes<td>8 bytes<td>4 bytes<tr><td>float64<td>8 bytes<td>8 bytes<td>8 bytes<tr><td>string/binary<td>Variable<td>8 bytes<td>8 bytes<tr><td>array<td>Variable<td>8 bytes<td>8 bytes<tr><td>map<td>Variable<td>8 bytes<td>8 bytes<tr><td>struct<td>Variable<td>8 bytes<td>inline or 8</table> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=implementation-notes>Implementation Notes<a href=#implementation-notes class=hash-link aria-label="Direct link to Implementation Notes" title="Direct link to Implementation Notes" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=endianness>Endianness<a href=#endianness class=hash-link aria-label="Direct link to Endianness" title="Direct link to Endianness" translate=no></a></h3> |
| <ul> |
| <li class="">All multi-byte integers are stored in <strong>little-endian</strong> format</li> |
| <li class="">Floating-point values use native IEEE 754 representation</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=memory-safety>Memory Safety<a href=#memory-safety class=hash-link aria-label="Direct link to Memory Safety" title="Direct link to Memory Safety" translate=no></a></h3> |
| <ul> |
| <li class="">Writers must zero padding bytes to prevent information leakage</li> |
| <li class="">Readers must validate offsets and sizes before accessing data</li> |
| <li class="">Buffer bounds checking is required for untrusted input</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=performance-considerations>Performance Considerations<a href=#performance-considerations class=hash-link aria-label="Direct link to Performance Considerations" title="Direct link to Performance Considerations" translate=no></a></h3> |
| <p><strong>Standard Format</strong>:</p> |
| <ul> |
| <li class="">Fixed 8-byte slots enable O(1) field access with simple arithmetic</li> |
| <li class="">8-byte alignment optimizes CPU cache line usage</li> |
| <li class="">Best for cross-language interoperability</li> |
| </ul> |
| <p><strong>Compact Format</strong>:</p> |
| <ul> |
| <li class="">Smaller row sizes reduce memory bandwidth</li> |
| <li class="">Field sorting minimizes padding waste</li> |
| <li class="">Inline structs eliminate pointer chasing</li> |
| <li class="">Relaxed alignment may have slight CPU overhead on some architectures</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=when-to-use-each-format>When to Use Each Format<a href=#when-to-use-each-format class=hash-link aria-label="Direct link to When to Use Each Format" title="Direct link to When to Use Each Format" translate=no></a></h3> |
| <table><thead><tr><th>Scenario<th>Recommended Format<tbody><tr><td>Cross-language data exchange<td>Standard<tr><td>Java-only, memory-constrained<td>Compact<tr><td>Many small primitive fields<td>Compact<tr><td>Many nested fixed-size structs<td>Compact<tr><td>Maximum read performance<td>Standard<tr><td>Interoperability with C++/Python<td>Standard</table></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="row margin-top--sm theme-doc-footer-edit-meta-row"><div class="col noPrint_WFHX"><a href=https://github.com/apache/fory-site/tree/main/docs/specification/row_format_spec.md target=_blank rel="noopener noreferrer" class=theme-edit-this-page><svg fill=currentColor height=20 width=20 viewBox="0 0 40 40" class=iconEdit_Z9Sw aria-hidden=true><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"/></g></svg>Edit this page</a></div><div class="col lastUpdated_JAkA"></div></div></footer></article><nav class="docusaurus-mt-lg pagination-nav" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href=/docs/specification/java_serialization_spec><div class=pagination-nav__sublabel>Previous</div><div class=pagination-nav__label>Java Serialization Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href=/docs/specification/xlang_type_mapping><div class=pagination-nav__sublabel>Next</div><div class=pagination-nav__label>Xlang Type Mapping</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href=#overview class="table-of-contents__link toc-highlight">Overview</a><li><a href=#format-comparison class="table-of-contents__link toc-highlight">Format Comparison</a><li><a href=#standard-row-format class="table-of-contents__link toc-highlight">Standard Row Format</a><ul><li><a href=#design-principles class="table-of-contents__link toc-highlight">Design Principles</a><li><a href=#row-binary-layout class="table-of-contents__link toc-highlight">Row Binary Layout</a><li><a href=#array-binary-layout class="table-of-contents__link toc-highlight">Array Binary Layout</a><li><a href=#map-binary-layout class="table-of-contents__link toc-highlight">Map Binary Layout</a><li><a href=#nested-struct-layout class="table-of-contents__link toc-highlight">Nested Struct Layout</a></ul><li><a href=#compact-row-format-java-only class="table-of-contents__link toc-highlight">Compact Row Format (Java Only)</a><ul><li><a href=#design-principles-1 class="table-of-contents__link toc-highlight">Design Principles</a><li><a href=#compact-row-binary-layout class="table-of-contents__link toc-highlight">Compact Row Binary Layout</a><li><a href=#compact-array-binary-layout class="table-of-contents__link toc-highlight">Compact Array Binary Layout</a></ul><li><a href=#common-specifications class="table-of-contents__link toc-highlight">Common Specifications</a><ul><li><a href=#type-encoding class="table-of-contents__link toc-highlight">Type Encoding</a><li><a href=#null-handling class="table-of-contents__link toc-highlight">Null Handling</a><li><a href=#alignment-and-padding class="table-of-contents__link toc-highlight">Alignment and Padding</a></ul><li><a href=#size-calculations class="table-of-contents__link toc-highlight">Size Calculations</a><ul><li><a href=#standard-row-size class="table-of-contents__link toc-highlight">Standard Row Size</a><li><a href=#compact-row-size class="table-of-contents__link toc-highlight">Compact Row Size</a><li><a href=#standard-array-size class="table-of-contents__link toc-highlight">Standard Array Size</a><li><a href=#compact-array-size class="table-of-contents__link toc-highlight">Compact Array Size</a><li><a href=#map-size class="table-of-contents__link toc-highlight">Map Size</a></ul><li><a href=#summary-tables class="table-of-contents__link toc-highlight">Summary Tables</a><ul><li><a href=#layout-summary class="table-of-contents__link toc-highlight">Layout Summary</a><li><a href=#type-width-summary class="table-of-contents__link toc-highlight">Type Width Summary</a></ul><li><a href=#implementation-notes class="table-of-contents__link toc-highlight">Implementation Notes</a><ul><li><a href=#endianness class="table-of-contents__link toc-highlight">Endianness</a><li><a href=#memory-safety class="table-of-contents__link toc-highlight">Memory Safety</a><li><a href=#performance-considerations class="table-of-contents__link toc-highlight">Performance Considerations</a><li><a href=#when-to-use-each-format class="table-of-contents__link toc-highlight">When to Use Each Format</a></ul></ul></div></div></div></div></main></div></div></div><footer class="theme-layout-footer footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Community</div><ul class="footer__items clean-list"><li class=footer__item><a href=https://lists.apache.org/list.html?dev@fory.apache.org target=_blank rel="noopener noreferrer" class=footer__link-item>Mailing list<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://join.slack.com/t/fory-project/shared_invite/zt-1u8soj4qc-ieYEu7ciHOqA2mo47llS8A target=_blank rel="noopener noreferrer" class=footer__link-item>Slack<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://twitter.com/ApacheFory target=_blank rel="noopener noreferrer" class=footer__link-item>Twitter<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a></ul></div><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Docs</div><ul class="footer__items clean-list"><li class=footer__item><a class=footer__link-item href=/docs/start/install>Install</a><li class=footer__item><a class=footer__link-item href=/docs/start/usage>Usage</a><li class=footer__item><a class=footer__link-item href=/docs/introduction/benchmark>Benchmark</a></ul></div><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Repositories</div><ul class="footer__items clean-list"><li class=footer__item><a href=https://github.com/apache/fory target=_blank rel="noopener noreferrer" class=footer__link-item>Apache Fory™<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://github.com/apache/fory-site target=_blank rel="noopener noreferrer" class=footer__link-item>Website<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a></ul></div></div><div class="footer__bottom text--center"><div class=margin-bottom--sm><a href=https://apache.org/ rel="noopener noreferrer" class=footerLogoLink_BH7S><img src=/img/asf_logo.svg alt="ASF Logo" class="footer__logo themedComponent_mlkZ themedComponent--light_NVdE" width=200 /><img src=/img/asf_logo.svg alt="ASF Logo" class="footer__logo themedComponent_mlkZ themedComponent--dark_xIcU" width=200 /></a></div><div class=footer__copyright><div> |
| <p> |
| Copyright © 2026 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br/> |
| Apache Fory, Fory, Apache, the Apache Logo and the Apache Fory logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. |
| </p> |
| </div></div></div></div></footer></div></body> |
