| <!doctype html><html lang=en-US dir=ltr class="docs-wrapper plugin-docs plugin-id-default docs-version-0.14 docs-doc-page docs-doc-id-specification/xlang_serialization_spec" data-has-hydrated=false><head><meta charset=UTF-8><meta name=generator content="Docusaurus v3.10.1"><title data-rh=true>Xlang Serialization Format | Apache Fory™</title><meta data-rh=true name=viewport content="width=device-width, initial-scale=1.0"/><meta data-rh=true property=og:url content=https://fory.apache.org/docs/0.14/specification/xlang_serialization_spec /><meta data-rh=true property=og:locale content=en_US /><meta data-rh=true property=og:locale:alternate content=zh_CN /><meta data-rh=true name=docusaurus_locale content=en-US /><meta data-rh=true name=docsearch:language content=en-US /><meta data-rh=true http-equiv=Content-Security-Policy content="frame-src 'self' https://ghbtns.com/;"/><meta data-rh=true property=og:image content=https://fory.apache.org/img/logo.png /><meta data-rh=true property=og:image:width content=1500 /><meta data-rh=true property=og:image:height content=1500 /><meta data-rh=true name=twitter:card content=summary_large_image /><meta data-rh=true name=twitter:image content=https://fory.apache.org/img/logo.png /><meta data-rh=true name=docusaurus_version content=0.14 /><meta data-rh=true name=docusaurus_tag content=docs-default-0.14 /><meta data-rh=true name=docsearch:version content=0.14 /><meta data-rh=true name=docsearch:docusaurus_tag content=docs-default-0.14 /><meta data-rh=true property=og:title content="Xlang Serialization Format | Apache Fory™"/><meta data-rh=true name=description content="Cross-language Serialization Specification"/><meta data-rh=true property=og:description content="Cross-language Serialization Specification"/><link data-rh=true rel=icon href=/img/favicon.ico /><link data-rh=true rel=canonical href=https://fory.apache.org/docs/0.14/specification/xlang_serialization_spec /><link data-rh=true rel=alternate href=https://fory.apache.org/docs/0.14/specification/xlang_serialization_spec hreflang=en-US /><link data-rh=true rel=alternate href=https://fory.apache.org/zh-CN/docs/0.14/specification/xlang_serialization_spec hreflang=zh-CN /><link data-rh=true rel=alternate href=https://fory.apache.org/docs/0.14/specification/xlang_serialization_spec hreflang=x-default /><script data-rh=true type=application/ld+json>{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","item":"https://fory.apache.org/docs/0.14/specification/xlang_serialization_spec","name":"Xlang Serialization Format","position":1}]}</script><link rel=alternate type=application/rss+xml href=/blog/rss.xml title="Apache Fory™ RSS Feed"><link rel=alternate type=application/atom+xml href=/blog/atom.xml title="Apache Fory™ Atom Feed"><script type=text/javascript>"fury.apache.org"===window.location.host&&(window.location.href="https://fory.apache.org"),function(){for(var o=window.location.pathname,a=["guide","introduction","start"],n=0;n<a.length;n++){var t=a[n],i=RegExp("^((?:/[a-z]{2}-[A-Z]{2})?/docs(?:/(?:next|[0-9]+.[0-9]+))?)/docs/"+t+"(/.*)?$"),c=o.match(i);if(c){var r=c[1]+"/"+t+(c[2]||"");window.location.replace(r+window.location.search+window.location.hash);return}}}()</script><link rel=stylesheet href=/assets/css/styles.3a1edaa1.css /><script src=/assets/js/runtime~main.605b11b1.js defer></script><script src=/assets/js/main.283fea4b.js defer></script></head><body><svg style="display: none;"><defs> |
| <symbol id=theme-svg-external-link viewBox="0 0 24 24"><path fill=currentColor d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"/></symbol> |
| </defs></svg> |
| <script>!function(){var t=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return window.localStorage.getItem("theme")}catch(t){}}();document.documentElement.setAttribute("data-theme",t||"light"),document.documentElement.setAttribute("data-theme-choice",t||"light")}(),function(){try{for(var[t,e]of new URLSearchParams(window.location.search).entries())if(t.startsWith("docusaurus-data-")){var a=t.replace("docusaurus-data-","data-");document.documentElement.setAttribute(a,e)}}catch(t){}}()</script><div id=__docusaurus><div role=region aria-label="Skip to main content"><a class=skipToContent_fXgn href=#__docusaurus_skipToContent_fallback>Skip to main content</a></div><nav aria-label=Main class="theme-layout-navbar navbar navbar--fixed-top"><div class=navbar__inner><div class="theme-layout-navbar-left navbar__items"><button aria-label="Toggle navigation bar" aria-expanded=false class="navbar__toggle clean-btn" type=button><svg width=30 height=30 viewBox="0 0 30 30" aria-hidden=true><path stroke=currentColor stroke-linecap=round stroke-miterlimit=10 stroke-width=2 d="M4 7h22M4 15h22M4 23h22"/></svg></button><a class=navbar__brand href=/><div class=navbar__logo><img src=/img/fory-logo-light.png alt="Apache Fory™ Logo" class="themedComponent_mlkZ themedComponent--light_NVdE"/><img src=/img/fory-logo-dark.png alt="Apache Fory™ Logo" class="themedComponent_mlkZ themedComponent--dark_xIcU"/></div><b class="navbar__title text--truncate"></b></a><a class="navbar__item navbar__link" href=/docs/0.14/introduction/overview>Docs</a><a aria-current=page class="navbar__item navbar__link navbar__link--active" href=/docs/0.14/specification/xlang_serialization_spec>Specification</a><a class="navbar__item navbar__link" href=/docs/0.14/community/>Community</a><a class="navbar__item navbar__link" href=/user>Users</a><a class="navbar__item navbar__link" href=/download>Download</a><a class="navbar__item navbar__link" href=/blog>Blog</a></div><div class="theme-layout-navbar-right navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href=# aria-haspopup=true aria-expanded=false role=button class=navbar__link>ASF</a><ul class=dropdown__menu><li><a href=https://www.apache.org/ target=_blank rel="noopener noreferrer" class=dropdown__link>Foundation</a><li><a href=https://www.apache.org/licenses/ target=_blank rel="noopener noreferrer" class=dropdown__link>License</a><li><a href=https://www.apache.org/events/current-event.html target=_blank rel="noopener noreferrer" class=dropdown__link>Events</a><li><a href=https://privacy.apache.org/policies/privacy-policy-public.html target=_blank rel="noopener noreferrer" class=dropdown__link>Privacy</a><li><a href=https://www.apache.org/security/ target=_blank rel="noopener noreferrer" class=dropdown__link>Security</a><li><a href=https://www.apache.org/foundation/sponsorship.html target=_blank rel="noopener noreferrer" class=dropdown__link>Sponsorship</a><li><a href=https://www.apache.org/foundation/thanks.html target=_blank rel="noopener noreferrer" class=dropdown__link>Thanks</a><li><a href=https://www.apache.org/foundation/policies/conduct.html target=_blank rel="noopener noreferrer" class=dropdown__link>Code of Conduct</a></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class=navbar__link aria-haspopup=true aria-expanded=false role=button href=/docs/0.14/specification/xlang_serialization_spec>0.14</a><ul class=dropdown__menu><li><a class=dropdown__link href=/docs/next/specification/xlang_serialization_spec>dev</a><li><a class=dropdown__link href=/docs/specification/xlang_serialization_spec>1.3.0</a><li><a class=dropdown__link href=/docs/1.2.0/specification/xlang_serialization_spec>1.2.0</a><li><a class=dropdown__link href=/docs/1.1.0/specification/xlang_serialization_spec>1.1.0</a><li><a class=dropdown__link href=/docs/1.0.0/specification/xlang_serialization_spec>1.0.0</a><li><a class=dropdown__link href=/docs/0.17/specification/xlang_serialization_spec>0.17</a><li><a class=dropdown__link href=/docs/0.16/specification/xlang_serialization_spec>0.16</a><li><a class=dropdown__link href=/docs/0.15/specification/xlang_serialization_spec>0.15</a><li><a aria-current=page class="dropdown__link dropdown__link--active" href=/docs/0.14/specification/xlang_serialization_spec>0.14</a><li><a class=dropdown__link href=/docs/0.13/specification/xlang_serialization_spec>0.13</a><li><a class=dropdown__link href=/docs/0.12/specification/xlang_serialization_spec>0.12</a><li><a class=dropdown__link href=/docs/0.11/specification/xlang_serialization_spec>0.11</a><li><a class=dropdown__link href=/docs/0.10/specification/xlang_serialization_spec>0.10</a></ul></div><a href=https://github.com/apache/fory target=_blank rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href=# aria-haspopup=true aria-expanded=false role=button class=navbar__link><svg viewBox="0 0 24 24" width=20 height=20 aria-hidden=true class=iconLanguage_nlXk><path fill=currentColor d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"/></svg>English</a><ul class=dropdown__menu><li><a href=/docs/0.14/specification/xlang_serialization_spec target=_self rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang=en-US>English</a><li><a href=/zh-CN/docs/0.14/specification/xlang_serialization_spec target=_self rel="noopener noreferrer" class=dropdown__link lang=zh-CN>简体中文</a></ul></div><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type=button disabled title="system mode" aria-label="Switch between dark and light mode (currently system mode)"><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP lightToggleIcon_pyhR"><path fill=currentColor d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP darkToggleIcon_wfgR"><path fill=currentColor d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP systemToggleIcon_QzmC"><path fill=currentColor d="m12 21c4.971 0 9-4.029 9-9s-4.029-9-9-9-9 4.029-9 9 4.029 9 9 9zm4.95-13.95c1.313 1.313 2.05 3.093 2.05 4.95s-0.738 3.637-2.05 4.95c-1.313 1.313-3.093 2.05-4.95 2.05v-14c1.857 0 3.637 0.737 4.95 2.05z"/></svg></button></div><div class=navbarSearchContainer_Bca1><div class=navbar__search><span aria-label="expand searchbar" role=button class=search-icon tabindex=0></span><input id=search_input_react type=search placeholder=Loading... aria-label=Search class="navbar__search-input search-bar" disabled/></div></div></div></div><div role=presentation class=navbar-sidebar__backdrop></div></nav><div id=__docusaurus_skipToContent_fallback class="theme-layout-main main-wrapper mainWrapper_z2l0"><div class=docsWrapper_hBAB><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type=button></button><div class=docRoot_UBD9><aside class="theme-doc-sidebar-container docSidebarContainer_YfHR"><div class=sidebarViewport_aRkj><div class=sidebar_njMd><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current=page href=/docs/0.14/specification/xlang_serialization_spec><span title="Xlang Serialization Format" class=linkLabel_WmDU>Xlang Serialization Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.14/specification/java_serialization_spec><span title="Java Serialization Format" class=linkLabel_WmDU>Java Serialization Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.14/specification/row_format_spec><span title="Row Format" class=linkLabel_WmDU>Row Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.14/specification/xlang_type_mapping><span title="Xlang Type Mapping" class=linkLabel_WmDU>Xlang Type Mapping</span></a></ul></nav></div></div></aside><main class=docMainContainer_TBSr><div class="container padding-top--md padding-bottom--lg"><div class=row><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role=alert><div>This is documentation for <!-- -->Apache Fory™<!-- --> <b>0.14</b>, which is no longer actively maintained.</div><div class=margin-top--md>For up-to-date documentation, see the <b><a href=/docs/specification/xlang_serialization_spec>latest version</a></b> (<!-- -->1.3.0<!-- -->).</div></div><div class=docItemContainer_Djhp><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label=Breadcrumbs><ul class=breadcrumbs><li class=breadcrumbs__item><a aria-label="Home page" class=breadcrumbs__link href=/><svg viewBox="0 0 24 24" class=breadcrumbHomeIcon_YNFT><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill=currentColor /></svg></a><li class="breadcrumbs__item breadcrumbs__item--active"><span class=breadcrumbs__link>Xlang Serialization Format</span></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.14</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type=button class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Xlang Serialization Format</h1></header><h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=cross-language-serialization-specification>Cross-language Serialization Specification<a href=#cross-language-serialization-specification class=hash-link aria-label="Direct link to Cross-language Serialization Specification" title="Direct link to Cross-language Serialization Specification" translate=no></a></h2> |
| <p>Apache Fory™ xlang serialization enables automatic cross-language object serialization with support for shared references, circular references, and polymorphism. Unlike traditional serialization frameworks that require IDL definitions and schema compilation, Fory serializes objects directly without any intermediate steps.</p> |
| <p>Key characteristics:</p> |
| <ul> |
| <li class=""><strong>Automatic</strong>: No IDL definition, no schema compilation, no manual object-to-protocol conversion</li> |
| <li class=""><strong>Cross-language</strong>: Same binary format works seamlessly across Java, Python, C++, Rust, Go, JavaScript, and more</li> |
| <li class=""><strong>Reference-aware</strong>: Handles shared references and circular references without duplication or infinite recursion</li> |
| <li class=""><strong>Polymorphic</strong>: Supports object polymorphism with runtime type resolution</li> |
| </ul> |
| <p>This specification defines the Fory xlang binary format. The format is dynamic rather than static, which enables flexibility and ease of use at the cost of additional complexity in the wire format.</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-systems>Type Systems<a href=#type-systems class=hash-link aria-label="Direct link to Type Systems" title="Direct link to Type Systems" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=data-types>Data Types<a href=#data-types class=hash-link aria-label="Direct link to Data Types" title="Direct link to Data Types" translate=no></a></h3> |
| <ul> |
| <li class="">bool: a boolean value (true or false).</li> |
| <li class="">int8: a 8-bit signed integer.</li> |
| <li class="">int16: a 16-bit signed integer.</li> |
| <li class="">int32: a 32-bit signed integer.</li> |
| <li class="">var_int32: a 32-bit signed integer which use fory var_int32 encoding.</li> |
| <li class="">int64: a 64-bit signed integer.</li> |
| <li class="">var_int64: a 64-bit signed integer which use fory PVL encoding.</li> |
| <li class="">sli_int64: a 64-bit signed integer which use fory SLI encoding.</li> |
| <li class="">float16: a 16-bit floating point number.</li> |
| <li class="">float32: a 32-bit floating point number.</li> |
| <li class="">float64: a 64-bit floating point number including NaN and Infinity.</li> |
| <li class="">string: a text string encoded using Latin1/UTF16/UTF-8 encoding.</li> |
| <li class="">enum: a data type consisting of a set of named values. Rust enum with non-predefined field values are not supported as |
| an enum.</li> |
| <li class="">named_enum: an enum whose value will be serialized as the registered name.</li> |
| <li class="">struct: a morphic(final) type serialized by Fory Struct serializer. i.e. it doesn't have subclasses. Suppose we're |
| deserializing <code>List<SomeClass></code>, we can save dynamic serializer dispatch since <code>SomeClass</code> is morphic(final).</li> |
| <li class="">compatible_struct: a morphic(final) type serialized by Fory compatible Struct serializer.</li> |
| <li class="">named_struct: a <code>struct</code> whose type mapping will be encoded as a name.</li> |
| <li class="">named_compatible_struct: a <code>compatible_struct</code> whose type mapping will be encoded as a name.</li> |
| <li class="">ext: a type which will be serialized by a customized serializer.</li> |
| <li class="">named_ext: an <code>ext</code> type whose type mapping will be encoded as a name.</li> |
| <li class="">list: a sequence of objects.</li> |
| <li class="">set: an unordered set of unique elements.</li> |
| <li class="">map: a map of key-value pairs. Mutable types such as <code>list/map/set/array</code> are not allowed as key of map.</li> |
| <li class="">duration: an absolute length of time, independent of any calendar/timezone, as a count of nanoseconds.</li> |
| <li class="">timestamp: a point in time, independent of any calendar/timezone, as a count of nanoseconds. The count is relative |
| to an epoch at UTC midnight on January 1, 1970.</li> |
| <li class="">local_date: a naive date without timezone. The count is days relative to an epoch at UTC midnight on Jan 1, 1970.</li> |
| <li class="">decimal: exact decimal value represented as an integer value in two's complement.</li> |
| <li class="">binary: an variable-length array of bytes.</li> |
| <li class="">array: only allow 1d numeric components. Other arrays will be taken as List. The implementation should support the |
| interoperability between array and list.<!-- --> |
| <ul> |
| <li class="">bool_array: one dimensional bool array.</li> |
| <li class="">int8_array: one dimensional int8 array.</li> |
| <li class="">int16_array: one dimensional int16 array.</li> |
| <li class="">int32_array: one dimensional int32 array.</li> |
| <li class="">int64_array: one dimensional int64 array.</li> |
| <li class="">float16_array: one dimensional half_float_16 array.</li> |
| <li class="">float32_array: one dimensional float32 array.</li> |
| <li class="">float64_array: one dimensional float64 array.</li> |
| </ul> |
| </li> |
| <li class="">union: a tagged union type that can hold one of several alternative types. The active alternative is identified by an index.</li> |
| <li class="">none: represents an empty/unit value with no data (e.g., for empty union alternatives).</li> |
| </ul> |
| <p>Note:</p> |
| <ul> |
| <li class="">Unsigned int/long are not added here, since not every language support those types.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=polymorphisms>Polymorphisms<a href=#polymorphisms class=hash-link aria-label="Direct link to Polymorphisms" title="Direct link to Polymorphisms" translate=no></a></h3> |
| <p>For polymorphism, if one non-final class is registered, and only one subclass is registered, then we can take all |
| elements in List/Map have same type, thus reduce runtime check cost.</p> |
| <p>Collection/Array polymorphism are not fully supported, since some languages such as golang have only one collection |
| type. If users want to get exactly the type he passed, he must pass that type when deserializing or annotate that type |
| to the field of struct.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-disambiguation>Type disambiguation<a href=#type-disambiguation class=hash-link aria-label="Direct link to Type disambiguation" title="Direct link to Type disambiguation" translate=no></a></h3> |
| <p>Due to differences between type systems of languages, those types can't be mapped one-to-one between languages. When |
| deserializing, Fory use the target data structure type and the data type in the data jointly to determine how to |
| deserialize and populate the target data structure. For example:</p> |
| <div class="language-java codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-java codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token keyword" style=color:#00009f>class</span><span class="token plain"> </span><span class="token class-name">Foo</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>{</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>int</span><span class="token punctuation" style=color:#393A34>[</span><span class="token punctuation" style=color:#393A34>]</span><span class="token plain"> intArray</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token class-name">Object</span><span class="token punctuation" style=color:#393A34>[</span><span class="token punctuation" style=color:#393A34>]</span><span class="token plain"> objects</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token class-name">List</span><span class="token generics punctuation" style=color:#393A34><</span><span class="token generics class-name">Object</span><span class="token generics punctuation" style=color:#393A34>></span><span class="token plain"> objectList</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token punctuation" style=color:#393A34>}</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>class</span><span class="token plain"> </span><span class="token class-name">Foo2</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>{</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>int</span><span class="token punctuation" style=color:#393A34>[</span><span class="token punctuation" style=color:#393A34>]</span><span class="token plain"> intArray</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token class-name">List</span><span class="token generics punctuation" style=color:#393A34><</span><span class="token generics class-name">Object</span><span class="token generics punctuation" style=color:#393A34>></span><span class="token plain"> objects</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token class-name">List</span><span class="token generics punctuation" style=color:#393A34><</span><span class="token generics class-name">Object</span><span class="token generics punctuation" style=color:#393A34>></span><span class="token plain"> objectList</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token punctuation" style=color:#393A34>}</span><br/></div></code></pre></div></div> |
| <p><code>intArray</code> has an <code>int32_array</code> type. But both <code>objects</code> and <code>objectList</code> fields in the serialize data have <code>list</code> data |
| type. When deserializing, the implementation will create an <code>Object</code> array for <code>objects</code>, but create a <code>ArrayList</code> |
| for <code>objectList</code> to populate its elements. And the serialized data of <code>Foo</code> can be deserialized into <code>Foo2</code> too.</p> |
| <p>Users can also provide meta hints for fields of a type, or the type whole. Here is an example in java which use |
| annotation to provide such information.</p> |
| <div class="language-java codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-java codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token annotation punctuation" style=color:#393A34>@ForyObject</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">fieldsNullable </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>false</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> trackingRef </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>false</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>class</span><span class="token plain"> </span><span class="token class-name">Foo</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>{</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token annotation punctuation" style=color:#393A34>@ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">trackingRef </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>false</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>int</span><span class="token punctuation" style=color:#393A34>[</span><span class="token punctuation" style=color:#393A34>]</span><span class="token plain"> intArray</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token annotation punctuation" style=color:#393A34>@ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">polymorphic </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>true</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token class-name">Object</span><span class="token plain"> object</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token annotation punctuation" style=color:#393A34>@ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">tagId </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token number" style=color:#36acaa>1</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> nullable </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>true</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token class-name">List</span><span class="token generics punctuation" style=color:#393A34><</span><span class="token generics class-name">Object</span><span class="token generics punctuation" style=color:#393A34>></span><span class="token plain"> objectList</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token punctuation" style=color:#393A34>}</span><br/></div></code></pre></div></div> |
| <p>Such information can be provided in other languages too:</p> |
| <ul> |
| <li class="">cpp: use macro and template.</li> |
| <li class="">golang: use struct tag.</li> |
| <li class="">python: use typehint.</li> |
| <li class="">rust: use macro.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-id>Type ID<a href=#type-id class=hash-link aria-label="Direct link to Type ID" title="Direct link to Type ID" translate=no></a></h3> |
| <p>All internal data types are expressed using an ID in range <code>0~64</code>. Users can use IDs in range <code>0~8192</code> for registering their |
| custom types (struct/ext/enum). User type IDs are in a separate namespace and combined with internal type IDs via bit shifting: |
| <code>(user_type_id << 8) | internal_type_id</code>.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=internal-type-id-table>Internal Type ID Table<a href=#internal-type-id-table class=hash-link aria-label="Direct link to Internal Type ID Table" title="Direct link to Internal Type ID Table" translate=no></a></h4> |
| <table><thead><tr><th>Type ID<th>Name<th>Description<tbody><tr><td>0<td>UNKNOWN<td>Unknown type, used for dynamic typing<tr><td>1<td>BOOL<td>Boolean value<tr><td>2<td>INT8<td>8-bit signed integer<tr><td>3<td>INT16<td>16-bit signed integer<tr><td>4<td>INT32<td>32-bit signed integer<tr><td>5<td>VAR_INT32<td>Variable-length encoded 32-bit signed integer<tr><td>6<td>INT64<td>64-bit signed integer<tr><td>7<td>VAR_INT64<td>Variable-length encoded 64-bit signed integer<tr><td>8<td>SLI_INT64<td>Small Long as Int encoded 64-bit signed integer<tr><td>9<td>FLOAT16<td>16-bit floating point (half precision)<tr><td>10<td>FLOAT32<td>32-bit floating point (single precision)<tr><td>11<td>FLOAT64<td>64-bit floating point (double precision)<tr><td>12<td>STRING<td>UTF-8/UTF-16/Latin1 encoded string<tr><td>13<td>ENUM<td>Enum registered by numeric ID<tr><td>14<td>NAMED_ENUM<td>Enum registered by namespace + type name<tr><td>15<td>STRUCT<td>Struct registered by numeric ID (schema consistent)<tr><td>16<td>COMPATIBLE_STRUCT<td>Struct with schema evolution support (by ID)<tr><td>17<td>NAMED_STRUCT<td>Struct registered by namespace + type name<tr><td>18<td>NAMED_COMPATIBLE_STRUCT<td>Struct with schema evolution (by name)<tr><td>19<td>EXT<td>Extension type registered by numeric ID<tr><td>20<td>NAMED_EXT<td>Extension type registered by namespace + type name<tr><td>21<td>LIST<td>Ordered collection (List, Array, Vector)<tr><td>22<td>SET<td>Unordered collection of unique elements<tr><td>23<td>MAP<td>Key-value mapping<tr><td>24<td>DURATION<td>Time duration (seconds + nanoseconds)<tr><td>25<td>TIMESTAMP<td>Point in time (nanoseconds since epoch)<tr><td>26<td>LOCAL_DATE<td>Date without timezone (days since epoch)<tr><td>27<td>DECIMAL<td>Arbitrary precision decimal<tr><td>28<td>BINARY<td>Raw binary data<tr><td>29<td>ARRAY<td>Generic array type<tr><td>30<td>BOOL_ARRAY<td>1D boolean array<tr><td>31<td>INT8_ARRAY<td>1D int8 array<tr><td>32<td>INT16_ARRAY<td>1D int16 array<tr><td>33<td>INT32_ARRAY<td>1D int32 array<tr><td>34<td>INT64_ARRAY<td>1D int64 array<tr><td>35<td>FLOAT16_ARRAY<td>1D float16 array<tr><td>36<td>FLOAT32_ARRAY<td>1D float32 array<tr><td>37<td>FLOAT64_ARRAY<td>1D float64 array<tr><td>38<td>UNION<td>Tagged union type (one of several alternatives)<tr><td>39<td>NONE<td>Empty/unit type (no data)</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-id-encoding-for-user-types>Type ID Encoding for User Types<a href=#type-id-encoding-for-user-types class=hash-link aria-label="Direct link to Type ID Encoding for User Types" title="Direct link to Type ID Encoding for User Types" translate=no></a></h4> |
| <p>When registering user types (struct/ext/enum), the full type ID combines user ID and internal type ID:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Full Type ID = (user_type_id << 8) | internal_type_id</span><br/></div></code></pre></div></div> |
| <p><strong>Examples:</strong></p> |
| <table><thead><tr><th>User ID<th>Type<th>Internal ID<th>Full Type ID<th>Decimal<tbody><tr><td>0<td>STRUCT<td>15<td><code>(0 << 8) | 15</code><td>15<tr><td>0<td>ENUM<td>13<td><code>(0 << 8) | 13</code><td>13<tr><td>1<td>STRUCT<td>15<td><code>(1 << 8) | 15</code><td>271<tr><td>1<td>COMPATIBLE_STRUCT<td>16<td><code>(1 << 8) | 16</code><td>272<tr><td>2<td>NAMED_STRUCT<td>17<td><code>(2 << 8) | 17</code><td>529</table> |
| <p>When reading type IDs:</p> |
| <ul> |
| <li class="">Extract internal type: <code>internal_type_id = full_type_id & 0xFF</code></li> |
| <li class="">Extract user type ID: <code>user_type_id = full_type_id >> 8</code></li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-mapping>Type mapping<a href=#type-mapping class=hash-link aria-label="Direct link to Type mapping" title="Direct link to Type mapping" translate=no></a></h3> |
| <p>See <a class="" href=/docs/0.14/specification/xlang_type_mapping>Type mapping</a></p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=spec-overview>Spec overview<a href=#spec-overview class=hash-link aria-label="Direct link to Spec overview" title="Direct link to Spec overview" translate=no></a></h2> |
| <p>Here is the overall format:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| fory header | object ref meta | object type meta | object value data |</span><br/></div></code></pre></div></div> |
| <p>The data are serialized using little endian byte order overall. If bytes swap is costly for some object, |
| Fory will write the byte order for that object into the data instead of converting it to little endian.</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=fory-header>Fory header<a href=#fory-header class=hash-link aria-label="Direct link to Fory header" title="Direct link to Fory header" translate=no></a></h2> |
| <p>Fory header format for xlang serialization:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| 2 bytes | 1 byte bitmap | 1 byte | optional 4 bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+--------------+--------------------------------+------------+------------------------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| magic number | 4 bits reserved | 4 bits meta | language | unsigned int for meta start offset |</span><br/></div></code></pre></div></div> |
| <p>Detailed byte layout:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Byte 0-1: Magic number (0x62d4) - little endian</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">Byte 2: Bitmap flags</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> - Bit 0: null flag (0x01)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> - Bit 1: endian flag (0x02)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> - Bit 2: xlang flag (0x04)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> - Bit 3: oob flag (0x08)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> - Bits 4-7: reserved</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">Byte 3: Language ID (only present when xlang flag is set)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">Byte 4-7: Meta start offset (only present when meta share mode is enabled)</span><br/></div></code></pre></div></div> |
| <ul> |
| <li class=""><strong>magic number</strong>: <code>0x62d4</code> (2 bytes, little endian) - used to identify fory xlang serialization protocol.</li> |
| <li class=""><strong>null flag</strong> (bit 0): 1 when object is null, 0 otherwise. If an object is null, only this flag and endian flag are set.</li> |
| <li class=""><strong>endian flag</strong> (bit 1): 1 when data is encoded by little endian, 0 for big endian. Modern implementations always use little endian.</li> |
| <li class=""><strong>xlang flag</strong> (bit 2): 1 when serialization uses Fory xlang format, 0 when serialization uses Fory language-native format.</li> |
| <li class=""><strong>oob flag</strong> (bit 3): 1 when out-of-band serialization is enabled (BufferCallback is not null), 0 otherwise.</li> |
| <li class=""><strong>language</strong>: 1 byte indicating the source language. This allows deserializers to optimize for specific language characteristics.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=language-ids>Language IDs<a href=#language-ids class=hash-link aria-label="Direct link to Language IDs" title="Direct link to Language IDs" translate=no></a></h3> |
| <table><thead><tr><th>Language<th>ID<tbody><tr><td>XLANG<td>0<tr><td>JAVA<td>1<tr><td>PYTHON<td>2<tr><td>CPP<td>3<tr><td>GO<td>4<tr><td>JAVASCRIPT<td>5<tr><td>RUST<td>6<tr><td>DART<td>7</table> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=meta-start-offset>Meta Start Offset<a href=#meta-start-offset class=hash-link aria-label="Direct link to Meta Start Offset" title="Direct link to Meta Start Offset" translate=no></a></h3> |
| <p>If compatible mode is enabled, an uncompressed unsigned int32 (4 bytes, little endian) is appended to indicate the start offset of metadata. During serialization, this is initially written as a placeholder (e.g., <code>-1</code> or <code>0</code>), then updated after all objects are serialized and metadata is collected.</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=reference-meta>Reference Meta<a href=#reference-meta class=hash-link aria-label="Direct link to Reference Meta" title="Direct link to Reference Meta" translate=no></a></h2> |
| <p>Reference tracking handles whether the object is null, and whether to track reference for the object by writing |
| corresponding flags and maintaining internal state.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=reference-flags>Reference Flags<a href=#reference-flags class=hash-link aria-label="Direct link to Reference Flags" title="Direct link to Reference Flags" translate=no></a></h3> |
| <table><thead><tr><th>Flag<th>Byte Value (int8)<th>Hex<th>Description<tbody><tr><td>NULL FLAG<td><code>-3</code><td><code>0xFD</code><td>Object is null. No further bytes are written for this object.<tr><td>REF FLAG<td><code>-2</code><td><code>0xFE</code><td>Object was already serialized. Followed by unsigned varint32 reference ID.<tr><td>NOT_NULL VALUE FLAG<td><code>-1</code><td><code>0xFF</code><td>Object is non-null but reference tracking is disabled for this type. Object data follows immediately.<tr><td>REF VALUE FLAG<td><code>0</code><td><code>0x00</code><td>Object is referencable and this is its first occurrence. Object data follows. Assigns next reference ID.</table> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=reference-tracking-algorithm>Reference Tracking Algorithm<a href=#reference-tracking-algorithm class=hash-link aria-label="Direct link to Reference Tracking Algorithm" title="Direct link to Reference Tracking Algorithm" translate=no></a></h3> |
| <p><strong>Writing:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function write_ref_or_null(buffer, obj):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if obj is null:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_int8(NULL_FLAG) // -3</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return true // done, no more data to write</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if reference_tracking_enabled:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> ref_id = lookup_written_objects(obj)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if ref_id exists:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_int8(REF_FLAG) // -2</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_varuint32(ref_id)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return true // done, reference written</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_int8(REF_VALUE_FLAG) // 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> add_to_written_objects(obj, next_ref_id++)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return false // continue to serialize object data</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_int8(NOT_NULL_VALUE_FLAG) // -1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return false // continue to serialize object data</span><br/></div></code></pre></div></div> |
| <p><strong>Reading:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function read_ref_or_null(buffer):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> flag = buffer.read_int8()</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> switch flag:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case NULL_FLAG (-3):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return (null, true) // null object, done</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case REF_FLAG (-2):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> ref_id = buffer.read_varuint32()</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> obj = get_from_read_objects(ref_id)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return (obj, true) // referenced object, done</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case NOT_NULL_VALUE_FLAG (-1):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return (null, false) // non-null, continue reading</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case REF_VALUE_FLAG (0):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> reserve_ref_slot() // will be filled after reading</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return (null, false) // non-null, continue reading</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=reference-id-assignment>Reference ID Assignment<a href=#reference-id-assignment class=hash-link aria-label="Direct link to Reference ID Assignment" title="Direct link to Reference ID Assignment" translate=no></a></h3> |
| <ul> |
| <li class="">Reference IDs are assigned sequentially starting from <code>0</code></li> |
| <li class="">The ID is assigned when <code>REF_VALUE_FLAG</code> is written (first occurrence)</li> |
| <li class="">Objects are stored in a list/map indexed by their reference ID</li> |
| <li class="">For reading, a placeholder slot is reserved before deserializing the object, then filled after</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=when-reference-tracking-is-disabled>When Reference Tracking is Disabled<a href=#when-reference-tracking-is-disabled class=hash-link aria-label="Direct link to When Reference Tracking is Disabled" title="Direct link to When Reference Tracking is Disabled" translate=no></a></h3> |
| <p>When reference tracking is disabled globally or for specific types, only the <code>NULL</code> and <code>NOT_NULL VALUE</code> flags |
| will be used for reference meta. This reduces overhead for types that are known not to have references.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=language-specific-considerations>Language-Specific Considerations<a href=#language-specific-considerations class=hash-link aria-label="Direct link to Language-Specific Considerations" title="Direct link to Language-Specific Considerations" translate=no></a></h3> |
| <p><strong>Languages with nullable and reference types by default (Java, Python, JavaScript):</strong></p> |
| <p>In xlang mode, for cross-language compatibility:</p> |
| <ul> |
| <li class="">All fields are treated as <strong>not-null</strong> by default</li> |
| <li class="">Reference tracking is <strong>disabled</strong> by default</li> |
| <li class="">Users can explicitly mark fields as nullable or enable reference tracking via annotations</li> |
| <li class=""><code>Optional</code> types (e.g., <code>java.util.Optional</code>, <code>typing.Optional</code>) are treated as nullable</li> |
| </ul> |
| <p><strong>Annotation examples:</strong></p> |
| <div class="language-java codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-java codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token comment" style=color:#999988;font-style:italic>// Java: use @ForyField annotation</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>public</span><span class="token plain"> </span><span class="token keyword" style=color:#00009f>class</span><span class="token plain"> </span><span class="token class-name">MyClass</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>{</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token annotation punctuation" style=color:#393A34>@ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">nullable </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>true</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> ref </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>true</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>private</span><span class="token plain"> </span><span class="token class-name">Object</span><span class="token plain"> refField</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token annotation punctuation" style=color:#393A34>@ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">nullable </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>false</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>private</span><span class="token plain"> </span><span class="token class-name">String</span><span class="token plain"> requiredField</span><span class="token punctuation" style=color:#393A34>;</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token punctuation" style=color:#393A34>}</span><br/></div></code></pre></div></div> |
| <div class="language-python codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-python codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token comment" style=color:#999988;font-style:italic># Python: use typing with fory field descriptors</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>from</span><span class="token plain"> pyfory </span><span class="token keyword" style=color:#00009f>import</span><span class="token plain"> Fory</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> ForyField</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>class</span><span class="token plain"> </span><span class="token class-name">MyClass</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> ref_field</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"> ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">SomeType</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> nullable</span><span class="token operator" style=color:#393A34>=</span><span class="token boolean" style=color:#36acaa>True</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> ref</span><span class="token operator" style=color:#393A34>=</span><span class="token boolean" style=color:#36acaa>True</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> required_field</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"> ForyField</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">str</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> nullable</span><span class="token operator" style=color:#393A34>=</span><span class="token boolean" style=color:#36acaa>False</span><span class="token punctuation" style=color:#393A34>)</span><br/></div></code></pre></div></div> |
| <p><strong>Languages with non-nullable types by default:</strong></p> |
| <table><thead><tr><th>Language<th>Null Representation<th>Reference Tracking Support<tbody><tr><td>Rust<td><code>Option::None</code><td>Via <code>Rc<T></code>, <code>Arc<T></code>, <code>Weak<T></code><tr><td>C++<td><code>std::nullopt</code>, <code>nullptr</code><td>Via <code>std::shared_ptr<T></code>, <code>weak_ptr<T></code><tr><td>Go<td><code>nil</code> interface/pointer<td>Via pointer/interface types</table> |
| <p><strong>Important:</strong> For languages like Rust that don't have implicit reference semantics, reference tracking must use |
| explicit smart pointers (<code>Rc</code>, <code>Arc</code>).</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-meta>Type Meta<a href=#type-meta class=hash-link aria-label="Direct link to Type Meta" title="Direct link to Type Meta" translate=no></a></h2> |
| <p>For every type to be serialized, it have a type id to indicate its type.</p> |
| <ul> |
| <li class="">basic types: the type id</li> |
| <li class="">enum:<!-- --> |
| <ul> |
| <li class=""><code>Type.ENUM</code> + registered id</li> |
| <li class=""><code>Type.NAMED_ENUM</code> + registered namespace+typename</li> |
| </ul> |
| </li> |
| <li class="">list: <code>Type.List</code></li> |
| <li class="">set: <code>Type.SET</code></li> |
| <li class="">map: <code>Type.MAP</code></li> |
| <li class="">ext:<!-- --> |
| <ul> |
| <li class=""><code>Type.EXT</code> + registered id</li> |
| <li class=""><code>Type.NAMED_EXT</code> + registered namespace+typename</li> |
| </ul> |
| </li> |
| <li class="">struct:<!-- --> |
| <ul> |
| <li class=""><code>Type.STRUCT</code> + struct meta</li> |
| <li class=""><code>Type.NAMED_STRUCT</code> + struct meta</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Every type must be registered with an ID or name first. The registration can be used for security check and type |
| identification.</p> |
| <p>Struct is a special type, depending whether schema compatibility is enabled, Fory will write struct meta |
| differently.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=struct-schema-consistent>Struct Schema consistent<a href=#struct-schema-consistent class=hash-link aria-label="Direct link to Struct Schema consistent" title="Direct link to Struct Schema consistent" translate=no></a></h3> |
| <ul> |
| <li class="">If schema consistent mode is enabled globally when creating fory, type meta will be written as a fory unsigned varint |
| of <code>type_id</code>. Schema evolution related meta will be ignored.</li> |
| <li class="">If schema evolution mode is enabled globally when creating fory, and current class is configured to use schema |
| consistent mode like <code>struct</code> vs <code>table</code> in flatbuffers:<!-- --> |
| <ul> |
| <li class="">Type meta will be add to <code>captured_type_defs</code>: <code>captured_type_defs[type def stub] = map size</code> ahead when |
| registering type.</li> |
| <li class="">Get index of the meta in <code>captured_type_defs</code>, write that index as <code>| unsigned varint: index |</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=struct-schema-evolution>Struct Schema evolution<a href=#struct-schema-evolution class=hash-link aria-label="Direct link to Struct Schema evolution" title="Direct link to Struct Schema evolution" translate=no></a></h3> |
| <p>If schema evolution mode is enabled globally when creating fory, and enabled for current type, type meta will be written |
| using one of the following mode. Which mode to use is configured when creating fory.</p> |
| <ul> |
| <li class=""> |
| <p>Normal mode(meta share not enabled):</p> |
| <ul> |
| <li class="">If type meta hasn't been written before, add <code>type def</code> |
| to <code>captured_type_defs</code>: <code>captured_type_defs[type def] = map size</code>.</li> |
| <li class="">Get index of the meta in <code>captured_type_defs</code>, write that index as <code>| unsigned varint: index |</code>.</li> |
| <li class="">After finished the serialization of the object graph, fory will start to write <code>captured_type_defs</code>:<!-- --> |
| <ul> |
| <li class=""> |
| <p>Firstly, set current to <code>meta start offset</code> of fory header</p> |
| </li> |
| <li class=""> |
| <p>Then write <code>captured_type_defs</code> one by one:</p> |
| <div class="language-python codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-python codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_var_uint32</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">len</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">writting_type_defs</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"> </span><span class="token operator" style=color:#393A34>-</span><span class="token plain"> </span><span class="token builtin">len</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">schema_consistent_type_def_stubs</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> type_meta </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> writting_type_defs</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> </span><span class="token keyword" style=color:#00009f>not</span><span class="token plain"> type_meta</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">is_stub</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> type_meta</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_type_def</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">writing_type_defs </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> copy</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">schema_consistent_type_def_stubs</span><span class="token punctuation" style=color:#393A34>)</span><br/></div></code></pre></div></div> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class=""> |
| <p>Meta share mode: the writing steps are same as the normal mode, but <code>captured_type_defs</code> will be shared across |
| multiple serializations of different objects. For example, suppose we have a batch to serialize:</p> |
| <div class="language-python codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-python codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">captured_type_defs </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>{</span><span class="token punctuation" style=color:#393A34>}</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">stream </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token comment" style=color:#999988;font-style:italic># add `Type1` to `captured_type_defs` and write `Type1`</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">serialize</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">stream</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>[</span><span class="token plain">Type1</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>]</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token comment" style=color:#999988;font-style:italic># add `Type2` to `captured_type_defs` and write `Type2`, `Type1` is written before.</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">serialize</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">stream</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>[</span><span class="token plain">Type1</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> Type2</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>]</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token comment" style=color:#999988;font-style:italic># `Type1` and `Type2` are written before, no need to write meta.</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">serialize</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">stream</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>[</span><span class="token plain">Type1</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> Type2</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>]</span><span class="token punctuation" style=color:#393A34>)</span><br/></div></code></pre></div></div> |
| </li> |
| <li class=""> |
| <p>Streaming mode(streaming mode doesn't support meta share):</p> |
| <ul> |
| <li class=""> |
| <p>If type meta hasn't been written before, the data will be written as:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| unsigned varint: 0b11111111 | type def |</span><br/></div></code></pre></div></div> |
| </li> |
| <li class=""> |
| <p>If type meta has been written before, the data will be written as:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| unsigned varint: written index << 1 |</span><br/></div></code></pre></div></div> |
| <p><code>written index</code> is the id in <code>captured_type_defs</code>.</p> |
| </li> |
| <li class=""> |
| <p>With this mode, <code>meta start offset</code> can be omitted.</p> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <blockquote> |
| <p>The normal mode and meta share mode will forbid streaming writing since it needs to look back for update the start |
| offset after the whole object graph writing and meta collecting is finished. Only in this way we can ensure |
| deserialization failure in meta share mode doesn't lost shared meta.</p> |
| </blockquote> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-def>Type Def<a href=#type-def class=hash-link aria-label="Direct link to Type Def" title="Direct link to Type Def" translate=no></a></h4> |
| <p>Here we mainly describe the meta layout for schema evolution mode:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| 8 bytes header | variable bytes | variable bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------------+--------------------+-------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| global binary header | meta header | fields meta |</span><br/></div></code></pre></div></div> |
| <p>For languages which support inheritance, if parent class and subclass has fields with same name, using field in |
| subclass.</p> |
| <h5 class="anchor anchorTargetStickyNavbar_Vzrq" id=global-binary-header>Global binary header<a href=#global-binary-header class=hash-link aria-label="Direct link to Global binary header" title="Direct link to Global binary header" translate=no></a></h5> |
| <p><code>50 bits hash + 1bit compress flag + write fields meta + 12 bits meta size</code>. Right is the lower bits.</p> |
| <ul> |
| <li class="">lower 12 bits are used to encode meta size. If meta size <code>>= 0b1111_1111_1111</code>, then write |
| <code>meta_ size - 0b1111_1111_1111</code> next.</li> |
| <li class="">13rd bit is used to indicate whether to write fields meta. When this class is schema-consistent or use registered |
| serializer, fields meta will be skipped. Class Meta will be used for share namespace + type name only.</li> |
| <li class="">14rd bit is used to indicate whether meta is compressed.</li> |
| <li class="">Other 50 bits is used to store the unique hash of <code>flags + all layers class meta</code>.</li> |
| </ul> |
| <h5 class="anchor anchorTargetStickyNavbar_Vzrq" id=meta-header>Meta header<a href=#meta-header class=hash-link aria-label="Direct link to Meta header" title="Direct link to Meta header" translate=no></a></h5> |
| <p>Meta header is a 8 bits number value.</p> |
| <ul> |
| <li class="">Lowest 5 digits <code>0b00000~0b11110</code> are used to record num fields. <code>0b11111</code> is preserved to indicate that Fory need to |
| read more bytes for length using Fory unsigned int encoding. Note that num_fields is the number of compatible fields. |
| Users can use tag id to mark some fields as compatible fields in schema consistent context. In such cases, schema |
| consistent fields will be serialized first, then compatible fields will be serialized next. At deserialization, |
| Fory will use fields info of those fields which aren't annotated by tag id for deserializing schema consistent |
| fields, then use fields info in meta for deserializing compatible fields.</li> |
| <li class="">The 6th bit: 0 for registered by id, 1 for registered by name.</li> |
| <li class="">Remaining 2 bits are reserved for future extension.</li> |
| </ul> |
| <h5 class="anchor anchorTargetStickyNavbar_Vzrq" id=fields-meta>Fields meta<a href=#fields-meta class=hash-link aria-label="Direct link to Fields meta" title="Direct link to Fields meta" translate=no></a></h5> |
| <p>Format:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| field info: variable bytes | variable bytes | ... |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+---------------------------------+-----------------+-----+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| header + type info + field name | next field info | ... |</span><br/></div></code></pre></div></div> |
| <h6 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-header>Field Header<a href=#field-header class=hash-link aria-label="Direct link to Field Header" title="Direct link to Field Header" translate=no></a></h6> |
| <p>Field Header is 8 bits, annotation can be used to provide more specific info. If annotation not exists, fory will infer |
| those info automatically.</p> |
| <p>The format for field header is:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">2 bits field name encoding + 4 bits size + nullability flag + ref tracking flag</span><br/></div></code></pre></div></div> |
| <p>Detailed spec:</p> |
| <ul> |
| <li class="">2 bits field name encoding:<!-- --> |
| <ul> |
| <li class="">encoding: <code>UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID</code></li> |
| <li class="">If tag id is used, field name will be written by an unsigned varint tag id, and 2 bits encoding will be <code>11</code>.</li> |
| </ul> |
| </li> |
| <li class="">size of field name:<!-- --> |
| <ul> |
| <li class="">The <code>4 bits size: 0~14</code> will be used to indicate length <code>1~15</code>, the value <code>15</code> indicates to read more bytes, |
| the encoding will encode <code>size - 15</code> as a varint next.</li> |
| <li class="">If encoding is <code>TAG_ID</code>, then num_bytes of field name will be used to store tag id.</li> |
| </ul> |
| </li> |
| <li class="">ref tracking: when set to 1, ref tracking will be enabled for this field.</li> |
| <li class="">nullability: when set to 1, this field can be null.</li> |
| </ul> |
| <h6 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-type-info>Field Type Info<a href=#field-type-info class=hash-link aria-label="Direct link to Field Type Info" title="Direct link to Field Type Info" translate=no></a></h6> |
| <p>Field type info is written as unsigned int8. Detailed id spec is:</p> |
| <ul> |
| <li class="">For struct registered by id, it will be <code>Type.STRUCT</code>.</li> |
| <li class="">For struct registered by name, it will be <code>Type.NAMED_STRUCT</code>.</li> |
| <li class="">For enum registered by id, it will be <code>Type.ENUM</code>.</li> |
| <li class="">For enum registered by name, it will be <code>Type.NAMED_ENUM</code>.</li> |
| <li class="">For ext type registered by id, it will be <code>Type.EXT</code>.</li> |
| <li class="">For ext type registered by name, it will be <code>Type.NAMED_EXT</code>.</li> |
| <li class="">For list/set type, it will be written as <code>Type.LIST/SET</code>, then write element type recursively.</li> |
| <li class="">For 1D primitive array type, it will be written as <code>Type.XXX_ARRAY</code>.</li> |
| <li class="">For multi-dimensional primitive array type with same size on each dim, it will be written as <code>Type.TENSOR</code>.</li> |
| <li class="">For other array type, it will be written as <code>Type.LIST</code>, then write element type recursively.</li> |
| <li class="">For map type, it will be written as <code>Type.MAP</code>, then write key and value type recursively.</li> |
| <li class="">For other types supported by fory directly, it will be fory type id for that type.</li> |
| <li class="">For other types not determined at compile time, write <code>Type.UNKNOWN</code> instead. For such types, actual type |
| will be written when serializing such field values.</li> |
| </ul> |
| <p>Polymorphism spec:</p> |
| <ul> |
| <li class=""><code>struct/named_struct/ext/named_ext</code> are taken as polymorphic, the meta for those types are written separately |
| instead of inlining here to reduce meta space cost if object of this type is serialized in current object graph |
| multiple times, and the field value may be null too.</li> |
| <li class=""><code>enum</code> is taken as morphic, if deserialization doesn't have this field, or the type is not enum, enum value |
| will be skipped.</li> |
| <li class=""><code>list/map/set</code> are taken as morphic, when serializing values of those type, the concrete types won't be written |
| again.</li> |
| <li class="">Other types that fory supported are taken as morphic too.</li> |
| </ul> |
| <p>List/Set/Map nested type spec:</p> |
| <ul> |
| <li class=""><code>list</code>: <code>| list type id | nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |</code></li> |
| <li class=""><code>set</code>: <code>| set type id | nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |</code></li> |
| <li class=""><code>map</code>: <code>| set type id | key type info | value type info |</code> |
| <ul> |
| <li class="">Key type format: <code>| nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |</code></li> |
| <li class="">Value type format: <code>| nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |</code></li> |
| </ul> |
| </li> |
| </ul> |
| <h6 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-name>Field Name<a href=#field-name class=hash-link aria-label="Direct link to Field Name" title="Direct link to Field Name" translate=no></a></h6> |
| <p>If tag id is set, tag id will be used instead. Otherwise meta string of field name will be written instead.</p> |
| <h6 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-order>Field order<a href=#field-order class=hash-link aria-label="Direct link to Field order" title="Direct link to Field order" translate=no></a></h6> |
| <p>Field order are left as implementation details, which is not exposed to specification, the deserialization need to |
| resort fields based on Fory fields sort algorithms. In this way, fory can compute statistics for field names or types and |
| using a more compact encoding.</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=extended-type-meta-with-inheritance-support>Extended Type Meta with Inheritance support<a href=#extended-type-meta-with-inheritance-support class=hash-link aria-label="Direct link to Extended Type Meta with Inheritance support" title="Direct link to Extended Type Meta with Inheritance support" translate=no></a></h2> |
| <p>If one want to support inheritance for struct, one can implement following spec.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=schema-consistent>Schema consistent<a href=#schema-consistent class=hash-link aria-label="Direct link to Schema consistent" title="Direct link to Schema consistent" translate=no></a></h3> |
| <p>Fields are serialized from parent type to leaf type. Fields are sorted using fory struct fields sort algorithms.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=schema-evolution>Schema Evolution<a href=#schema-evolution class=hash-link aria-label="Direct link to Schema Evolution" title="Direct link to Schema Evolution" translate=no></a></h3> |
| <p>Meta layout for schema evolution mode:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| 8 bytes header | variable bytes | variable bytes | variable bytes | variable bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------------+----------------+----------------+--------------------+--------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| global binary header | meta header | fields meta | parent meta header | parent fields meta |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=meta-header-1>Meta header<a href=#meta-header-1 class=hash-link aria-label="Direct link to Meta header" title="Direct link to Meta header" translate=no></a></h4> |
| <p>Meta header is a 64 bits number value encoded in little endian order.</p> |
| <ul> |
| <li class="">Lowest 4 digits <code>0b0000~0b1110</code> are used to record num classes. <code>0b1111</code> is preserved to indicate that Fory need to |
| read more bytes for length using Fory unsigned int encoding. If current type doesn't has parent type, or parent |
| type doesn't have fields to serialize, or we're in a context which serialize fields of current type |
| only, num classes will be 1.</li> |
| <li class="">The 5th bit is used to indicate whether this type needs schema evolution.</li> |
| <li class="">Other 56 bits are used to store the unique hash of <code>flags + all layers type meta</code>.</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=single-layer-type-meta>Single layer type meta<a href=#single-layer-type-meta class=hash-link aria-label="Direct link to Single layer type meta" title="Direct link to Single layer type meta" translate=no></a></h4> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| unsigned varint | var uint | field info: variable bytes | variable bytes | ... |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------------+----------+-------------------------------+-----------------+-----+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| num_fields | type id | header + type id + field name | next field info | ... |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=other-layers-type-meta>Other layers type meta<a href=#other-layers-type-meta class=hash-link aria-label="Direct link to Other layers type meta" title="Direct link to Other layers type meta" translate=no></a></h4> |
| <p>Same encoding algorithm as the previous layer.</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=meta-string>Meta String<a href=#meta-string class=hash-link aria-label="Direct link to Meta String" title="Direct link to Meta String" translate=no></a></h2> |
| <p>Meta string is a compressed encoding for metadata strings such as field names, type names, and namespaces. |
| This compression significantly reduces the size of type metadata in serialized data.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=encoding-type-ids>Encoding Type IDs<a href=#encoding-type-ids class=hash-link aria-label="Direct link to Encoding Type IDs" title="Direct link to Encoding Type IDs" translate=no></a></h3> |
| <table><thead><tr><th>ID<th>Name<th>Bits/Char<th>Character Set<tbody><tr><td>0<td>UTF8<td>8<td>Any UTF-8 character<tr><td>1<td>LOWER_SPECIAL<td>5<td><code>a-z . _ $ |</code><tr><td>2<td>LOWER_UPPER_DIGIT_SPECIAL<td>6<td><code>a-z A-Z 0-9 . _</code><tr><td>3<td>FIRST_TO_LOWER_SPECIAL<td>5<td>First char uppercase, rest <code>a-z . _</code><tr><td>4<td>ALL_TO_LOWER_SPECIAL<td>5<td><code>a-z A-Z . _</code> (uppercase escaped)</table> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=character-mapping-tables>Character Mapping Tables<a href=#character-mapping-tables class=hash-link aria-label="Direct link to Character Mapping Tables" title="Direct link to Character Mapping Tables" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=lower_special-5-bits-per-character>LOWER_SPECIAL (5 bits per character)<a href=#lower_special-5-bits-per-character class=hash-link aria-label="Direct link to LOWER_SPECIAL (5 bits per character)" title="Direct link to LOWER_SPECIAL (5 bits per character)" translate=no></a></h4> |
| <table><thead><tr><th>Character<th>Code (binary)<th>Code (decimal)<tbody><tr><td>a-z<td>00000-11001<td>0-25<tr><td>.<td>11010<td>26<tr><td>_<td>11011<td>27<tr><td>$<td>11100<td>28<tr><td>|<td>11101<td>29</table> |
| <p><strong>Note:</strong> The <code>|</code> character is used as an escape sequence in ALL_TO_LOWER_SPECIAL encoding.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=lower_upper_digit_special-6-bits-per-character>LOWER_UPPER_DIGIT_SPECIAL (6 bits per character)<a href=#lower_upper_digit_special-6-bits-per-character class=hash-link aria-label="Direct link to LOWER_UPPER_DIGIT_SPECIAL (6 bits per character)" title="Direct link to LOWER_UPPER_DIGIT_SPECIAL (6 bits per character)" translate=no></a></h4> |
| <table><thead><tr><th>Character<th>Code (binary)<th>Code (decimal)<tbody><tr><td>a-z<td>000000-011001<td>0-25<tr><td>A-Z<td>011010-110011<td>26-51<tr><td>0-9<td>110100-111101<td>52-61<tr><td>.<td>111110<td>62<tr><td>_<td>111111<td>63</table> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=encoding-algorithms>Encoding Algorithms<a href=#encoding-algorithms class=hash-link aria-label="Direct link to Encoding Algorithms" title="Direct link to Encoding Algorithms" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=lower_special-encoding>LOWER_SPECIAL Encoding<a href=#lower_special-encoding class=hash-link aria-label="Direct link to LOWER_SPECIAL Encoding" title="Direct link to LOWER_SPECIAL Encoding" translate=no></a></h4> |
| <p>For strings containing only <code>a-z</code>, <code>.</code>, <code>_</code>, <code>$</code>, <code>|</code>:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function encode_lower_special(str):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bits = []</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> for char in str:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bits.append(lookup_lower_special[char]) // 5 bits each</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // Pad to byte boundary</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> total_bits = len(str) * 5</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> padding_bits = (8 - (total_bits % 8)) % 8</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // First bit indicates if last char should be stripped (due to padding)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> strip_last = (padding_bits >= 5)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if strip_last:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> prepend bit 1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> prepend bit 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return pack_bits_to_bytes(bits)</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=first_to_lower_special-encoding>FIRST_TO_LOWER_SPECIAL Encoding<a href=#first_to_lower_special-encoding class=hash-link aria-label="Direct link to FIRST_TO_LOWER_SPECIAL Encoding" title="Direct link to FIRST_TO_LOWER_SPECIAL Encoding" translate=no></a></h4> |
| <p>For strings like <code>MyFieldName</code> where only the first character is uppercase:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function encode_first_to_lower_special(str):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // Convert first char to lowercase</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> modified = str[0].lower() + str[1:]</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // Then use LOWER_SPECIAL encoding</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return encode_lower_special(modified)</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=all_to_lower_special-encoding>ALL_TO_LOWER_SPECIAL Encoding<a href=#all_to_lower_special-encoding class=hash-link aria-label="Direct link to ALL_TO_LOWER_SPECIAL Encoding" title="Direct link to ALL_TO_LOWER_SPECIAL Encoding" translate=no></a></h4> |
| <p>For strings with multiple uppercase characters like <code>MyTypeName</code>:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function encode_all_to_lower_special(str):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> result = ""</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> for char in str:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if char.is_upper():</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> result += "|" + char.lower() // Escape uppercase with |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> result += char</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return encode_lower_special(result)</span><br/></div></code></pre></div></div> |
| <p>Example: <code>MyType</code> → <code>|my|type</code> → encoded with LOWER_SPECIAL</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=encoding-selection-algorithm>Encoding Selection Algorithm<a href=#encoding-selection-algorithm class=hash-link aria-label="Direct link to Encoding Selection Algorithm" title="Direct link to Encoding Selection Algorithm" translate=no></a></h3> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function choose_encoding(str):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if all chars in str are in [a-z . _ $ |]:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return LOWER_SPECIAL</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if first char is uppercase AND rest are in [a-z . _]:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return FIRST_TO_LOWER_SPECIAL</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if all chars are in [a-z A-Z . _]:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> lower_special_size = encode_all_to_lower_special(str).size</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> luds_size = encode_lower_upper_digit_special(str).size</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if lower_special_size <= luds_size:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return ALL_TO_LOWER_SPECIAL</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return LOWER_UPPER_DIGIT_SPECIAL</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if all chars are in [a-z A-Z 0-9 . _]:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return LOWER_UPPER_DIGIT_SPECIAL</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return UTF8</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=meta-string-header-format>Meta String Header Format<a href=#meta-string-header-format class=hash-link aria-label="Direct link to Meta String Header Format" title="Direct link to Meta String Header Format" translate=no></a></h3> |
| <p>Meta strings are written with a header that includes the encoding type:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| 3 bits encoding | 5+ bits length | encoded bytes |</span><br/></div></code></pre></div></div> |
| <p>Or for larger strings:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| varuint: (length << 3) | encoding | encoded bytes |</span><br/></div></code></pre></div></div> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=special-character-sets-by-context>Special Character Sets by Context<a href=#special-character-sets-by-context class=hash-link aria-label="Direct link to Special Character Sets by Context" title="Direct link to Special Character Sets by Context" translate=no></a></h3> |
| <p>Different contexts use different special characters:</p> |
| <table><thead><tr><th>Context<th>Special Chars<th>Notes<tbody><tr><td>Field Name<td>. _ $ |<td>$ for inner classes, | for escape<tr><td>Namespace<td>. _<td>Package/module separators<tr><td>Type Name<td>$ _<td>$ for inner classes in Java</table> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=deduplication>Deduplication<a href=#deduplication class=hash-link aria-label="Direct link to Deduplication" title="Direct link to Deduplication" translate=no></a></h3> |
| <p>Meta strings are deduplicated within a serialization session:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">First occurrence: | (length << 1) | [hash if large] | encoding | bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">Reference: | ((id + 1) << 1) | 1 |</span><br/></div></code></pre></div></div> |
| <ul> |
| <li class="">Bit 0 of the header indicates: 0 = new string, 1 = reference to previous</li> |
| <li class="">Large strings (> 16 bytes) include 64-bit hash for content-based deduplication</li> |
| <li class="">Small strings use exact byte comparison</li> |
| </ul> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=value-format>Value Format<a href=#value-format class=hash-link aria-label="Direct link to Value Format" title="Direct link to Value Format" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=basic-types>Basic types<a href=#basic-types class=hash-link aria-label="Direct link to Basic types" title="Direct link to Basic types" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=bool>bool<a href=#bool class=hash-link aria-label="Direct link to bool" title="Direct link to bool" translate=no></a></h4> |
| <ul> |
| <li class="">size: 1 byte</li> |
| <li class="">format: 0 for <code>false</code>, 1 for <code>true</code></li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=int8>int8<a href=#int8 class=hash-link aria-label="Direct link to int8" title="Direct link to int8" translate=no></a></h4> |
| <ul> |
| <li class="">size: 1 byte</li> |
| <li class="">format: write as pure byte.</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=int16>int16<a href=#int16 class=hash-link aria-label="Direct link to int16" title="Direct link to int16" translate=no></a></h4> |
| <ul> |
| <li class="">size: 2 byte</li> |
| <li class="">byte order: raw bytes of little endian order</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=unsigned-int32>unsigned int32<a href=#unsigned-int32 class=hash-link aria-label="Direct link to unsigned int32" title="Direct link to unsigned int32" translate=no></a></h4> |
| <ul> |
| <li class="">size: 4 byte</li> |
| <li class="">byte order: raw bytes of little endian order</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=unsigned-varint32>unsigned varint32<a href=#unsigned-varint32 class=hash-link aria-label="Direct link to unsigned varint32" title="Direct link to unsigned varint32" translate=no></a></h4> |
| <ul> |
| <li class="">size: 1~5 bytes</li> |
| <li class="">Format: The most significant bit (MSB) in every byte indicates whether to have the next byte. If the continuation |
| bit is set (i.e. <code>b & 0x80 == 0x80</code>), then the next byte should be read until a byte with unset continuation bit.</li> |
| </ul> |
| <p><strong>Encoding Algorithm:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function write_varuint32(value):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> while value >= 0x80:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_byte((value & 0x7F) | 0x80) // 7 bits of data + continuation bit</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> value = value >> 7</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_byte(value) // final byte without continuation bit</span><br/></div></code></pre></div></div> |
| <p><strong>Decoding Algorithm:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function read_varuint32():</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> result = 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> shift = 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> while true:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> byte = buffer.read_byte()</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> result = result | ((byte & 0x7F) << shift)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if (byte & 0x80) == 0:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> break</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> shift = shift + 7</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return result</span><br/></div></code></pre></div></div> |
| <p><strong>Byte sizes by value range:</strong></p> |
| <table><thead><tr><th>Value Range<th>Bytes<tbody><tr><td>0 ~ 127<td>1<tr><td>128 ~ 16383<td>2<tr><td>16384 ~ 2097151<td>3<tr><td>2097152 ~ 268435455<td>4<tr><td>268435456 ~ 4294967295<td>5</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=signed-int32>signed int32<a href=#signed-int32 class=hash-link aria-label="Direct link to signed int32" title="Direct link to signed int32" translate=no></a></h4> |
| <ul> |
| <li class="">size: 4 bytes</li> |
| <li class="">byte order: raw bytes of little endian order</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=signed-varint32>signed varint32<a href=#signed-varint32 class=hash-link aria-label="Direct link to signed varint32" title="Direct link to signed varint32" translate=no></a></h4> |
| <ul> |
| <li class="">size: 1~5 bytes</li> |
| <li class="">Format: First convert the number into positive unsigned int using ZigZag encoding, then encode as unsigned varint.</li> |
| </ul> |
| <p><strong>ZigZag Encoding:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">// Encode: convert signed to unsigned</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">zigzag_value = (value << 1) ^ (value >> 31)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">// Decode: convert unsigned back to signed</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">original = (zigzag_value >> 1) ^ (-(zigzag_value & 1))</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">// Or equivalently:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">original = (zigzag_value >> 1) ^ (~(zigzag_value & 1) + 1)</span><br/></div></code></pre></div></div> |
| <p>ZigZag encoding maps signed integers to unsigned integers so that small absolute values (positive or negative) |
| have small encoded values:</p> |
| <table><thead><tr><th>Original<th>ZigZag Encoded<tbody><tr><td>0<td>0<tr><td>-1<td>1<tr><td>1<td>2<tr><td>-2<td>3<tr><td>2<td>4<tr><td>...<td>...</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=unsigned-int64>unsigned int64<a href=#unsigned-int64 class=hash-link aria-label="Direct link to unsigned int64" title="Direct link to unsigned int64" translate=no></a></h4> |
| <ul> |
| <li class="">size: 8 bytes</li> |
| <li class="">byte order: raw bytes of little endian order</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=unsigned-varint64>unsigned varint64<a href=#unsigned-varint64 class=hash-link aria-label="Direct link to unsigned varint64" title="Direct link to unsigned varint64" translate=no></a></h4> |
| <ul> |
| <li class="">size: 1~9 bytes</li> |
| </ul> |
| <p>Fory supports two encoding schemes for 64-bit integers:</p> |
| <p><strong>Fory SLI (Small Long as Int) Encoding:</strong></p> |
| <p>Optimized for values that fit in 31 bits (common case for IDs, timestamps, etc.):</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">if value in [0, 2147483647]: // fits in 31 bits</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write 4 bytes: ((int32) value) << 1 // bit 0 is 0, indicating 4-byte encoding</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write 1 byte: 0x01 // bit 0 is 1, indicating 9-byte encoding</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write 8 bytes: value as little-endian int64</span><br/></div></code></pre></div></div> |
| <p>Reading:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">first_int32 = read_int32_le()</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">if (first_int32 & 1) == 0:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return first_int32 >> 1 // 4-byte encoding</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return read_int64_le() // read remaining 8 bytes</span><br/></div></code></pre></div></div> |
| <p><strong>Fory PVL (Progressive Variable-Length) Encoding:</strong></p> |
| <p>Standard varint encoding extended to 64 bits:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function write_varuint64(value):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> while value >= 0x80:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_byte((value & 0x7F) | 0x80)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> value = value >> 7</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_byte(value)</span><br/></div></code></pre></div></div> |
| <table><thead><tr><th>Value Range<th>Bytes<tbody><tr><td>0 ~ 127<td>1<tr><td>128 ~ 16383<td>2<tr><td>...<td>...<tr><td>2^56 ~ 2^63-1<td>9</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=varuint36small>VarUint36Small<a href=#varuint36small class=hash-link aria-label="Direct link to VarUint36Small" title="Direct link to VarUint36Small" translate=no></a></h4> |
| <p>A specialized encoding used for string headers that combines size (up to 36 bits) with encoding flags:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">// Write: encodes (size << 2) | encoding_flags</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">function write_varuint36_small(value):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> if value < 0x80:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_byte(value)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // Standard varint encoding for values >= 128</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write_varuint64(value)</span><br/></div></code></pre></div></div> |
| <p>This encoding is optimized for the common case where string length fits in 7 bits (strings < 32 characters).</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=signed-int64>signed int64<a href=#signed-int64 class=hash-link aria-label="Direct link to signed int64" title="Direct link to signed int64" translate=no></a></h4> |
| <ul> |
| <li class="">size: 8 bytes</li> |
| <li class="">byte order: raw bytes of little endian order</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=signed-varint64>signed varint64<a href=#signed-varint64 class=hash-link aria-label="Direct link to signed varint64" title="Direct link to signed varint64" translate=no></a></h4> |
| <ul> |
| <li class="">size: 1~9 bytes</li> |
| </ul> |
| <p><strong>Fory SLI (Small Long as Int) Encoding for signed:</strong></p> |
| <p>Optimized for small signed values:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">if value in [-1073741824, 1073741823]: // fits in 31 bits signed</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write 4 bytes: ((int32) value) << 1 // bit 0 is 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">else:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write 1 byte: 0x01 // bit 0 is 1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> write 8 bytes: value as little-endian int64</span><br/></div></code></pre></div></div> |
| <p><strong>Fory PVL (Progressive Variable-Length) Encoding for signed:</strong></p> |
| <p>Uses ZigZag encoding first, then varint:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">// Encode</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">zigzag_value = (value << 1) ^ (value >> 63)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">write_varuint64(zigzag_value)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain" style=display:inline-block></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">// Decode</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">zigzag_value = read_varuint64()</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">value = (zigzag_value >> 1) ^ (-(zigzag_value & 1))</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=float32>float32<a href=#float32 class=hash-link aria-label="Direct link to float32" title="Direct link to float32" translate=no></a></h4> |
| <ul> |
| <li class="">size: 4 byte</li> |
| <li class="">format: encode the specified floating-point value according to the IEEE 754 floating-point "single format" bit layout, |
| preserving Not-a-Number (NaN) values, then write as binary by little endian order.</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=float64>float64<a href=#float64 class=hash-link aria-label="Direct link to float64" title="Direct link to float64" translate=no></a></h4> |
| <ul> |
| <li class="">size: 8 byte</li> |
| <li class="">format: encode the specified floating-point value according to the IEEE 754 floating-point "double format" bit layout, |
| preserving Not-a-Number (NaN) values. then write as binary by little endian order.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=string>string<a href=#string class=hash-link aria-label="Direct link to string" title="Direct link to string" translate=no></a></h3> |
| <p>Format:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| varuint36_small: (size << 2) | encoding | binary data |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=string-header>String Header<a href=#string-header class=hash-link aria-label="Direct link to String Header" title="Direct link to String Header" translate=no></a></h4> |
| <p>The header is encoded using <code>varuint36_small</code> format, which combines the byte length and encoding type:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">header = (byte_length << 2) | encoding_type</span><br/></div></code></pre></div></div> |
| <table><thead><tr><th>Encoding Type<th>Value<th>Description<tbody><tr><td>LATIN1<td>0<td>ISO-8859-1 single-byte encoding<tr><td>UTF16<td>1<td>UTF-16 encoding (2 bytes per code unit)<tr><td>UTF8<td>2<td>UTF-8 variable-length encoding<tr><td>Reserved<td>3<td>Reserved for future use</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=encoding-algorithm>Encoding Algorithm<a href=#encoding-algorithm class=hash-link aria-label="Direct link to Encoding Algorithm" title="Direct link to Encoding Algorithm" translate=no></a></h4> |
| <p><strong>Writing:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function write_string(str):</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bytes = encode_to_bytes(str, chosen_encoding)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> header = (bytes.length << 2) | encoding_type</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_varuint36_small(header)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> buffer.write_bytes(bytes)</span><br/></div></code></pre></div></div> |
| <p><strong>Reading:</strong></p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">function read_string():</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> header = buffer.read_varuint36_small()</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> encoding = header & 0x03</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> byte_length = header >> 2</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bytes = buffer.read_bytes(byte_length)</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> return decode_bytes(bytes, encoding)</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=encoding-selection-by-language>Encoding Selection by Language<a href=#encoding-selection-by-language class=hash-link aria-label="Direct link to Encoding Selection by Language" title="Direct link to Encoding Selection by Language" translate=no></a></h4> |
| <p><strong>Writing:</strong></p> |
| <table><thead><tr><th>Language<th>Encoding Strategy<tbody><tr><td>Java (JDK8)<td>Detect at runtime: LATIN1 if all chars < 256, else UTF16<tr><td>Java (JDK9+)<td>Use String's internal coder: LATIN1 or UTF16<tr><td>Python<td>Can write LATIN1, UTF16, or UTF8 based on string content<tr><td>C++<td>UTF8 (<code>std::string</code>) or UTF16 (<code>std::u16string</code>)<tr><td>Rust<td>UTF8 (<code>String</code>)<tr><td>Go<td>UTF8 (<code>string</code>)<tr><td>JavaScript<td>UTF8</table> |
| <p><strong>Reading:</strong> All languages support decoding all three encodings (LATIN1, UTF16, UTF8).</p> |
| <p><strong>Recommendation:</strong> Select encoding based on maximum performance - use the encoding that matches the language's native string representation to avoid conversion overhead.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=empty-string>Empty String<a href=#empty-string class=hash-link aria-label="Direct link to Empty String" title="Direct link to Empty String" translate=no></a></h4> |
| <p>Empty strings are encoded with header <code>0</code> (length 0, any encoding) followed by no data bytes.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=duration>duration<a href=#duration class=hash-link aria-label="Direct link to duration" title="Direct link to duration" translate=no></a></h3> |
| <p>Duration is an absolute length of time, independent of any calendar/timezone, as a count of seconds and nanoseconds.</p> |
| <p>Format:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| signed varint64: seconds | signed int32: nanoseconds |</span><br/></div></code></pre></div></div> |
| <ul> |
| <li class=""><code>seconds</code>: Number of seconds in the duration, encoded as a signed varint64. Can be positive or negative.</li> |
| <li class=""><code>nanoseconds</code>: Nanosecond adjustment to the duration, encoded as a signed int32. Value range is [0, 999,999,999] for positive durations, and [-999,999,999, 0] for negative durations.</li> |
| </ul> |
| <p>Notes:</p> |
| <ul> |
| <li class="">The duration is stored as two separate fields to maintain precision and avoid overflow issues.</li> |
| <li class="">Seconds are encoded using varint64 for compact representation of common duration values.</li> |
| <li class="">Nanoseconds are stored as a fixed int32 since the range is limited.</li> |
| <li class="">The sign of the duration is determined by the seconds field. When seconds is 0, the sign is determined by nanoseconds.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=collectionlist>collection/list<a href=#collectionlist class=hash-link aria-label="Direct link to collection/list" title="Direct link to collection/list" translate=no></a></h3> |
| <p>Format:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| varuint32: length | 1 byte elements header | [optional type info] | elements data |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=elements-header>Elements Header<a href=#elements-header class=hash-link aria-label="Direct link to Elements Header" title="Direct link to Elements Header" translate=no></a></h4> |
| <p>The elements header is a single byte that encodes metadata about the collection elements to optimize serialization:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| bit 7-4 (reserved) | bit 3 | bit 2 | bit 1 | bit 0 |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+--------------------+-------------+------------------+----------+-----------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| reserved | is_same_type| is_decl_elem_type| has_null | track_ref |</span><br/></div></code></pre></div></div> |
| <table><thead><tr><th>Bit<th>Name<th>Value<th>Meaning when SET (1)<th>Meaning when UNSET (0)<tbody><tr><td>0<td>track_ref<td>0x01<td>Track references for elements<td>Don't track element references<tr><td>1<td>has_null<td>0x02<td>Collection may contain null elements<td>No null elements (skip null checks)<tr><td>2<td>is_decl_elem_type<td>0x04<td>Elements are the declared generic type<td>Element types differ from declared type<tr><td>3<td>is_same_type<td>0x08<td>All elements have the same runtime type<td>Elements have different runtime types</table> |
| <p><strong>Common header values:</strong></p> |
| <table><thead><tr><th>Header<th>Hex<th>Meaning<tbody><tr><td>0x0C<td>12<td>Declared type + same type, non-null, no ref tracking (optimal)<tr><td>0x0D<td>13<td>Declared type + same type, non-null, with ref tracking<tr><td>0x0E<td>14<td>Declared type + same type, may have nulls, no ref tracking<tr><td>0x08<td>8<td>Same type but not declared type (type info written once)<tr><td>0x00<td>0<td>Different types, non-null, no ref tracking (type per element)</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=type-info-after-header>Type Info After Header<a href=#type-info-after-header class=hash-link aria-label="Direct link to Type Info After Header" title="Direct link to Type Info After Header" translate=no></a></h4> |
| <p>When <code>is_decl_elem_type</code> (bit 2) is NOT set, the element type info is written once after the header if <code>is_same_type</code> (bit 3) is set:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| header (0x08) | type_id (varuint32) | elements... |</span><br/></div></code></pre></div></div> |
| <p>When both <code>is_decl_elem_type</code> and <code>is_same_type</code> are NOT set, type info is written per element.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=element-serialization-based-on-header>Element Serialization Based on Header<a href=#element-serialization-based-on-header class=hash-link aria-label="Direct link to Element Serialization Based on Header" title="Direct link to Element Serialization Based on Header" translate=no></a></h4> |
| <p>The header determines how each element is serialized:</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=elements-data>elements data<a href=#elements-data class=hash-link aria-label="Direct link to elements data" title="Direct link to elements data" translate=no></a></h4> |
| <p>Based on the elements header, the serialization of elements data may skip <code>ref flag</code>/<code>null flag</code>/<code>element type info</code>.</p> |
| <div class="language-python codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-python codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">fory </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token builtin">buffer</span><span class="token plain"> </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain">elems </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> </span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> element_type_is_same</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> </span><span class="token keyword" style=color:#00009f>not</span><span class="token plain"> is_declared_type</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_type</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem_type</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> elem_serializer </span><span class="token operator" style=color:#393A34>=</span><span class="token plain"> get_serializer</span><span class="token punctuation" style=color:#393A34>(</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>.</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> track_ref</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> elems</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> </span><span class="token keyword" style=color:#00009f>not</span><span class="token plain"> ref_resolver</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_ref_or_null</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> elem_serializer</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>elif</span><span class="token plain"> has_null</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> elems</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>is</span><span class="token plain"> </span><span class="token boolean" style=color:#36acaa>None</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_byte</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">null_flag</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>else</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_byte</span><span class="token punctuation" style=color:#393A34>(</span><span class="token plain">not_null_flag</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> elem_serializer</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>else</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> elems</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> elem_serializer</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"></span><span class="token keyword" style=color:#00009f>else</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>if</span><span class="token plain"> track_ref</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> elems</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_ref</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>elif</span><span class="token plain"> has_null</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> elems</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_nullable</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>else</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> </span><span class="token keyword" style=color:#00009f>for</span><span class="token plain"> elem </span><span class="token keyword" style=color:#00009f>in</span><span class="token plain"> elems</span><span class="token punctuation" style=color:#393A34>:</span><span class="token plain"></span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fory</span><span class="token punctuation" style=color:#393A34>.</span><span class="token plain">write_value</span><span class="token punctuation" style=color:#393A34>(</span><span class="token builtin">buffer</span><span class="token punctuation" style=color:#393A34>,</span><span class="token plain"> elem</span><span class="token punctuation" style=color:#393A34>)</span><br/></div></code></pre></div></div> |
| <p><a href=https://github.com/apache/fory/blob/20a1a78b17a75a123a6f5b7094c06ff77defc0fe/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java#L302 target=_blank rel="noopener noreferrer" class=""><code>CollectionSerializer#writeElements</code></a> |
| can be taken as an example.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=array>array<a href=#array class=hash-link aria-label="Direct link to array" title="Direct link to array" translate=no></a></h3> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=primitive-array>primitive array<a href=#primitive-array class=hash-link aria-label="Direct link to primitive array" title="Direct link to primitive array" translate=no></a></h4> |
| <p>Primitive array are taken as a binary buffer, serialization will just write the length of array size as an unsigned int, |
| then copy the whole buffer into the stream.</p> |
| <p>Such serialization won't compress the array. If users want to compress primitive array, users need to register custom |
| serializers for such types or mark it as list type.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=tensor>Tensor<a href=#tensor class=hash-link aria-label="Direct link to Tensor" title="Direct link to Tensor" translate=no></a></h4> |
| <p>Tensor is a special primitive multi-dimensional array which all dimensions have same size and type. The serialization |
| format is:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| num_dims(unsigned varint) | shape[0](unsigned varint) | shape[...] | shape[N] | element type | data |</span><br/></div></code></pre></div></div> |
| <p>The data is continuous to reduce copy and may zero-copy in some cases.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=object-array>object array<a href=#object-array class=hash-link aria-label="Direct link to object array" title="Direct link to object array" translate=no></a></h4> |
| <p>Object array is serialized using the list format. Object component type will be taken as list element |
| generic type.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=map>map<a href=#map class=hash-link aria-label="Direct link to map" title="Direct link to map" translate=no></a></h3> |
| <p>Map uses a chunk-based format to handle heterogeneous key-value pairs efficiently:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| varuint32: total_size | chunk_1 | chunk_2 | ... | chunk_n |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=map-chunk-format>Map Chunk Format<a href=#map-chunk-format class=hash-link aria-label="Direct link to Map Chunk Format" title="Direct link to Map Chunk Format" translate=no></a></h4> |
| <p>Each chunk contains up to 255 key-value pairs with the same metadata characteristics:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| 1 byte | 1 byte | variable bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+--------------+----------------+------------------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| KV header | chunk size N | N key-value pairs (N*2 obj) |</span><br/></div></code></pre></div></div> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=kv-header-bits>KV Header Bits<a href=#kv-header-bits class=hash-link aria-label="Direct link to KV Header Bits" title="Direct link to KV Header Bits" translate=no></a></h4> |
| <p>The KV header is a single byte encoding metadata for both keys and values:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| bit 7-6 | bit 5 | bit 4 | bit 3 | bit 2 | bit 1 | bit 0 |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+------------+---------------+--------------+---------------+---------------+--------------+---------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| reserved | val_decl_type | val_has_null | val_track_ref | key_decl_type | key_has_null | key_track_ref |</span><br/></div></code></pre></div></div> |
| <table><thead><tr><th>Bit<th>Name<th>Value<th>Meaning when SET (1)<tbody><tr><td>0<td>key_track_ref<td>0x01<td>Track references for keys<tr><td>1<td>key_has_null<td>0x02<td>Keys may be null (rare, usually invalid)<tr><td>2<td>key_decl_type<td>0x04<td>Key is the declared generic type<tr><td>3<td>val_track_ref<td>0x08<td>Track references for values<tr><td>4<td>val_has_null<td>0x10<td>Values may be null<tr><td>5<td>val_decl_type<td>0x20<td>Value is the declared generic type</table> |
| <p><strong>Common KV header values:</strong></p> |
| <table><thead><tr><th>Header<th>Hex<th>Meaning<tbody><tr><td>0x24<td>36<td>Key + value are declared types, non-null, no ref tracking (optimal)<tr><td>0x2C<td>44<td>Key + value declared types, value tracks refs<tr><td>0x34<td>52<td>Key + value declared types, value may be null<tr><td>0x00<td>0<td>Key + value not declared types, non-null, no ref tracking</table> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=chunk-size>Chunk Size<a href=#chunk-size class=hash-link aria-label="Direct link to Chunk Size" title="Direct link to Chunk Size" translate=no></a></h4> |
| <ul> |
| <li class="">Maximum chunk size: 255 pairs (fits in 1 byte)</li> |
| <li class="">When key or value is null, that entry is serialized as a separate chunk with implicit size 1 (chunk size byte is skipped)</li> |
| <li class="">Reader tracks accumulated count against total map size to know when to stop reading chunks</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=why-chunk-based-format>Why Chunk-Based Format?<a href=#why-chunk-based-format class=hash-link aria-label="Direct link to Why Chunk-Based Format?" title="Direct link to Why Chunk-Based Format?" translate=no></a></h4> |
| <p>Map iteration is expensive. Computing a single header for all pairs would require two passes. The chunk-based |
| approach allows:</p> |
| <ol> |
| <li class=""><strong>Optimistic prediction</strong>: Use first key-value pair to predict header</li> |
| <li class=""><strong>Adaptive chunking</strong>: Start new chunk if prediction fails for a pair</li> |
| <li class=""><strong>Efficient reading</strong>: Most maps fit in single chunk (< 255 pairs)</li> |
| <li class=""><strong>Memory efficiency</strong>: Minimal overhead for common homogeneous maps</li> |
| </ol> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=why-serialize-chunk-by-chunk>Why serialize chunk by chunk?<a href=#why-serialize-chunk-by-chunk class=hash-link aria-label="Direct link to Why serialize chunk by chunk?" title="Direct link to Why serialize chunk by chunk?" translate=no></a></h4> |
| <p>When fory will use first key-value pair to predict header optimistically, it can't know how many pairs have same |
| meta(tracking kef ref, key has null and so on). If we don't write chunk by chunk with max chunk size, we must write at |
| least <code>X</code> bytes to take up a place for later to update the number which has same elements, <code>X</code> is the num_bytes for |
| encoding varint encoding of map size.</p> |
| <p>And most map size are smaller than 255, if all pairs have same data, the chunk will be 1. This is common in golang/rust, |
| which object are not reference by default.</p> |
| <p>Also, if only one or two keys have different meta, we can make it into a different chunk, so that most pairs can share |
| meta.</p> |
| <p>The implementation can accumulate read count with map size to decide whether to read more chunks.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=enum>enum<a href=#enum class=hash-link aria-label="Direct link to enum" title="Direct link to enum" translate=no></a></h3> |
| <p>Enums are serialized as an unsigned var int. If the order of enum values change, the deserialized enum value may not be |
| the value users expect. In such cases, users must register enum serializer by make it write enum value as an enumerated |
| string with unique hash disabled.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=decimal>decimal<a href=#decimal class=hash-link aria-label="Direct link to decimal" title="Direct link to decimal" translate=no></a></h3> |
| <p>Not supported for now.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=struct>struct<a href=#struct class=hash-link aria-label="Direct link to struct" title="Direct link to struct" translate=no></a></h3> |
| <p>Struct means object of <code>class/pojo/struct/bean/record</code> type. |
| Struct will be serialized by writing its fields data in fory order.</p> |
| <p>Depending on schema compatibility, structs will have different formats.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=field-order-1>field order<a href=#field-order-1 class=hash-link aria-label="Direct link to field order" title="Direct link to field order" translate=no></a></h4> |
| <p>Field will be ordered as following, every group of fields will have its own order:</p> |
| <ul> |
| <li class="">primitive fields:<!-- --> |
| <ul> |
| <li class="">larger size type first, smaller later, variable size type last.</li> |
| <li class="">when same size, sort by type id</li> |
| <li class="">when same size and type id, sort by snake case field name</li> |
| <li class="">types: bool/int8/int16/int32/varint32/int64/varint64/sliint64/float16/float32/float64</li> |
| </ul> |
| </li> |
| <li class="">nullable primitive fields: same order as primitive fields</li> |
| <li class="">other internal type fields: sort by type id then snake case field name</li> |
| <li class="">list fields: sort by snake case field name</li> |
| <li class="">set fields: sort by snake case field name</li> |
| <li class="">map fields: sort by snake case field name</li> |
| <li class="">other fields: sort by snake case field name</li> |
| </ul> |
| <p>If two fields have same type, then sort by snake_case styled field name.</p> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=schema-consistent-1>schema consistent<a href=#schema-consistent-1 class=hash-link aria-label="Direct link to schema consistent" title="Direct link to schema consistent" translate=no></a></h4> |
| <p>Object will be written as:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">| 4 byte | variable bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+---------------+------------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| type hash | field values |</span><br/></div></code></pre></div></div> |
| <p>Type hash is used to check the type schema consistency across languages. Type hash will be the first 32 bits of 56 bits |
| value of the type meta.</p> |
| <p>Object fields will be serialized one by one using following format:</p> |
| <div class="language-text codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-text codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">not null primitive field value:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| var bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| value data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+----------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">nullable primitive field value:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| one byte | var bytes |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+---------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| null flag | field value |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+---------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">other interal types supported by fory</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| var bytes | var objects |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+-------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| null flag | value data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+-------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">list field type:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| one byte | var objects |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+-------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| ref meta | value data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">set field type:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| one byte | var objects |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+-------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| ref meta | value data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">map field type:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| one byte | var objects |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+-------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| ref meta | value data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+-------------+-------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">other types such as enum/struct/ext</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| one byte | var bytes | var objects |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+------------+------------+</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">| ref flag | type meta | value data |</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">+-----------+------------+------------+</span><br/></div></code></pre></div></div> |
| <p>Type hash algorithm:</p> |
| <ul> |
| <li class="">Sort fields by fields sort algorithm</li> |
| <li class="">Start with string <code>""</code></li> |
| <li class="">Iterate every field, append string by:<!-- --> |
| <ul> |
| <li class=""><code>snow_case(field_name),</code>. For camelcase name, convert it to snow_case first.</li> |
| <li class=""><code>$type_id,</code>, for other fields, use type id <code>TypeId::UNKNOWN</code> instead.</li> |
| <li class=""><code>$nullable;</code>, <code>1</code> if nullable, <code>0</code> otherwise.</li> |
| </ul> |
| </li> |
| <li class="">Then convert string to utf8 bytes</li> |
| <li class="">Compute murmurhash3_x64_128, and use first 32 bits</li> |
| </ul> |
| <h4 class="anchor anchorTargetStickyNavbar_Vzrq" id=schema-evolution-1>Schema evolution<a href=#schema-evolution-1 class=hash-link aria-label="Direct link to Schema evolution" title="Direct link to Schema evolution" translate=no></a></h4> |
| <p>Schema evolution have similar format as schema consistent mode for object except:</p> |
| <ul> |
| <li class="">For the object type, <code>schema consistent</code> mode will write type by id only, but <code>schema evolution</code> mode will |
| write type consisting of field names, types and other meta too, see <a href=#type-meta class="">Type meta</a>.</li> |
| <li class="">Type meta of <code>final custom type</code> needs to be written too, because peers may not have this type defined.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=type>Type<a href=#type class=hash-link aria-label="Direct link to Type" title="Direct link to Type" translate=no></a></h3> |
| <p>Type will be serialized using type meta format.</p> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=implementation-guidelines>Implementation guidelines<a href=#implementation-guidelines class=hash-link aria-label="Direct link to Implementation guidelines" title="Direct link to Implementation guidelines" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=how-to-reduce-memory-readwrite-code>How to reduce memory read/write code<a href=#how-to-reduce-memory-readwrite-code class=hash-link aria-label="Direct link to How to reduce memory read/write code" title="Direct link to How to reduce memory read/write code" translate=no></a></h3> |
| <ul> |
| <li class="">Try to merge multiple bytes into an int/long write before writing to reduce memory IO and bound check cost.</li> |
| <li class="">Read multiple bytes as an int/long, then split into multiple bytes to reduce memory IO and bound check cost.</li> |
| <li class="">Try to use one varint/long to write flags and length together to save one byte cost and reduce memory io.</li> |
| <li class="">Condition branches are less expensive compared to memory IO cost unless there are too many branches.</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=fast-deserialization-for-static-languages-without-runtime-codegen-support>Fast deserialization for static languages without runtime codegen support<a href=#fast-deserialization-for-static-languages-without-runtime-codegen-support class=hash-link aria-label="Direct link to Fast deserialization for static languages without runtime codegen support" title="Direct link to Fast deserialization for static languages without runtime codegen support" translate=no></a></h3> |
| <p>For type evolution, the serializer will encode the type meta into the serialized data. The deserializer will compare |
| this meta with class meta in the current process, and use the diff to determine how to deserialize the data.</p> |
| <p>For java/javascript/python, we can use the diff to generate serializer code at runtime and load it as class/function for |
| deserialization. In this way, the type evolution will be as fast as type consist mode.</p> |
| <p>For C++/Rust, we can't generate the serializer code at runtime. So we need to generate the code at compile-time using |
| meta programming. But at that time, we don't know the type schema in other processes, so we can't generate the |
| serializer code for such inconsistent types. We may need to generate the code which has a loop and compare field name |
| one by one to decide whether to deserialize and assign the field or skip the field value.</p> |
| <p>One fast way is that we can optimize the string comparison into <code>jump</code> instructions:</p> |
| <ul> |
| <li class="">Assume the current type has <code>n</code> fields, and the peer type has <code>n1</code> fields.</li> |
| <li class="">Generate an auto growing <code>field id</code> from <code>0</code> for every sorted field in the current type at the compile time.</li> |
| <li class="">Compare the received type meta with current type, generate same id if the field name is same, otherwise generate an |
| auto growing id starting from <code>n</code>, cache this meta at runtime.</li> |
| <li class="">Iterate the fields of received type meta, use a <code>switch</code> to compare the <code>field id</code> to deserialize data |
| and <code>assign/skip</code> field value. <strong>Continuous</strong> field id will be optimized into <code>jump</code> in <code>switch</code> block, so it will |
| very fast.</li> |
| </ul> |
| <p>Here is an example, suppose process A has a class <code>Foo</code> with version 1 defined as <code>Foo1</code>, process B has a class <code>Foo</code> |
| with version 2 defined as <code>Foo2</code>:</p> |
| <div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-c++ codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">// class Foo with version 1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">class Foo1 {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> int32_t v1; // id 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> std::string v2; // id 1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">};</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">// class Foo with version 2</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">class Foo2 {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 0, but will have id 2 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bool v0;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 1, but will have id 0 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> int32_t v1;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 2, but will have id 3 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> int64_t long_value;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 3, but will have id 1 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> std::string v2;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 4, but will have id 4 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> std::vector<std::string> list;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">};</span><br/></div></code></pre></div></div> |
| <p>When process A received serialized <code>Foo2</code> from process B, here is how it deserialize the data:</p> |
| <div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-c++ codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Foo1 foo1 = ...;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">const std::vector<fory::FieldInfo> &field_infos = type_meta.field_infos;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">for (const auto &field_info : field_infos) {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> switch (field_info.field_id) {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case 0:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> foo1.v1 = buffer.read_varint32();</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> break;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case 1:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> foo1.v2 = fory.read_string();</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> break;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> default:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fory.skip_data(field_info);</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> }</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">}</span><br/></div></code></pre></div></div> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=implementation-checklist-for-new-languages>Implementation Checklist for New Languages<a href=#implementation-checklist-for-new-languages class=hash-link aria-label="Direct link to Implementation Checklist for New Languages" title="Direct link to Implementation Checklist for New Languages" translate=no></a></h2> |
| <p>This section provides a step-by-step guide for implementing Fory xlang serialization in a new language.</p> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-1-core-infrastructure>Phase 1: Core Infrastructure<a href=#phase-1-core-infrastructure class=hash-link aria-label="Direct link to Phase 1: Core Infrastructure" title="Direct link to Phase 1: Core Infrastructure" translate=no></a></h3> |
| <ol> |
| <li class=""> |
| <p><strong>Buffer Implementation</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Create a byte buffer with read/write cursor tracking</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement little-endian byte order for all multi-byte writes</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_int8</code>, <code>write_int16</code>, <code>write_int32</code>, <code>write_int64</code></li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_float32</code>, <code>write_float64</code></li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>read_*</code> counterparts for all write methods</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement buffer growth strategy (e.g., doubling)</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Varint Encoding</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varuint32</code> / <code>read_varuint32</code></li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varint32</code> / <code>read_varint32</code> (with ZigZag)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varuint64</code> / <code>read_varuint64</code></li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varint64</code> / <code>read_varint64</code> (with ZigZag)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varuint36_small</code> / <code>read_varuint36_small</code> (for strings)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Optionally implement SLI encoding for int64</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Header Handling</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write magic number <code>0x62d4</code></li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write/read bitmap flags (null, endian, xlang, oob)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write/read language ID</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle meta start offset placeholder (for schema evolution)</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-2-basic-type-serializers>Phase 2: Basic Type Serializers<a href=#phase-2-basic-type-serializers class=hash-link aria-label="Direct link to Phase 2: Basic Type Serializers" title="Direct link to Phase 2: Basic Type Serializers" translate=no></a></h3> |
| <ol start=4> |
| <li class=""> |
| <p><strong>Primitive Types</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->bool (1 byte: 0 or 1)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->int8, int16, int32, int64 (little endian)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->float32, float64 (IEEE 754, little endian)</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>String Serialization</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement string header: <code>(byte_length << 2) | encoding</code></li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Support UTF-8 encoding (required for xlang)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Optionally support LATIN1 and UTF-16</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Temporal Types</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Duration (seconds + nanoseconds)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Timestamp (nanoseconds since epoch)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->LocalDate (days since epoch)</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Reference Tracking</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement write-side object tracking (object → ref_id map)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement read-side object tracking (ref_id → object list)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle all four reference flags: NULL(-3), REF(-2), NOT_NULL(-1), REF_VALUE(0)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Support disabling reference tracking per-type or globally</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-3-collection-types>Phase 3: Collection Types<a href=#phase-3-collection-types class=hash-link aria-label="Direct link to Phase 3: Collection Types" title="Direct link to Phase 3: Collection Types" translate=no></a></h3> |
| <ol start=8> |
| <li class=""> |
| <p><strong>List/Array Serialization</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write length as varuint32</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write elements header byte</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle homogeneous vs heterogeneous elements</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle null elements</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Map Serialization</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write total size as varuint32</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement chunk-based format (max 255 pairs per chunk)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write KV header byte per chunk</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle key and value type variations</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Set Serialization</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Same format as List (reuse implementation)</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-4-meta-string-encoding>Phase 4: Meta String Encoding<a href=#phase-4-meta-string-encoding class=hash-link aria-label="Direct link to Phase 4: Meta String Encoding" title="Direct link to Phase 4: Meta String Encoding" translate=no></a></h3> |
| <p>Meta strings are required for enum and struct serialization (encoding field names, type names, namespaces).</p> |
| <ol start=11> |
| <li class=""><strong>Meta String Compression</strong> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement LOWER_SPECIAL encoding (5 bits/char)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement LOWER_UPPER_DIGIT_SPECIAL encoding (6 bits/char)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement FIRST_TO_LOWER_SPECIAL encoding</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement ALL_TO_LOWER_SPECIAL encoding</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement encoding selection algorithm</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement meta string deduplication</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-5-enum-serialization>Phase 5: Enum Serialization<a href=#phase-5-enum-serialization class=hash-link aria-label="Direct link to Phase 5: Enum Serialization" title="Direct link to Phase 5: Enum Serialization" translate=no></a></h3> |
| <ol start=12> |
| <li class=""><strong>Enum Serialization</strong> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write ordinal as varuint32</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Support named enum (namespace + type name)</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-6-struct-serialization>Phase 6: Struct Serialization<a href=#phase-6-struct-serialization class=hash-link aria-label="Direct link to Phase 6: Struct Serialization" title="Direct link to Phase 6: Struct Serialization" translate=no></a></h3> |
| <ol start=13> |
| <li class=""> |
| <p><strong>Type Registration</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Support registration by numeric ID</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Support registration by namespace + type name</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Maintain type → serializer mapping</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Generate type IDs: <code>(user_id << 8) | internal_type_id</code></li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Field Ordering</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement Fory field ordering algorithm</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Sort primitives by size (larger first), then type ID, then name</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle nullable vs non-nullable fields</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Convert field names to snake_case for sorting</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Schema Consistent Mode</strong></p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Compute type hash (MurmurHash3 of field info string)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Write 4-byte type hash before fields</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Serialize fields in Fory order</li> |
| </ul> |
| </li> |
| <li class=""> |
| <p><strong>Schema Evolution Mode</strong> (Optional)</p> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement type meta writing</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Support field addition/removal</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle unknown fields (skip during read)</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-7-other-types>Phase 7: Other types<a href=#phase-7-other-types class=hash-link aria-label="Direct link to Phase 7: Other types" title="Direct link to Phase 7: Other types" translate=no></a></h3> |
| <ol start=17> |
| <li class=""><strong>Binary/Array Types</strong> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Primitive arrays (direct buffer copy)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Tensor (multi-dimensional arrays)</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=testing-strategy>Testing Strategy<a href=#testing-strategy class=hash-link aria-label="Direct link to Testing Strategy" title="Direct link to Testing Strategy" translate=no></a></h3> |
| <ol start=18> |
| <li class=""><strong>Cross-Language Compatibility Tests</strong> |
| <ul class="contains-task-list containsTaskList_mC6p"> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Serialize in new language, deserialize in Java/Python</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Serialize in Java/Python, deserialize in new language</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Test all primitive types</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Test strings with various encodings</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Test collections (empty, single, multiple elements)</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Test maps with various key/value types</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Test nested structs</li> |
| <li class=task-list-item><input type=checkbox disabled/> <!-- -->Test circular references (if supported)</li> |
| </ul> |
| </li> |
| </ol> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=language-specific-implementation-notes>Language-Specific Implementation Notes<a href=#language-specific-implementation-notes class=hash-link aria-label="Direct link to Language-Specific Implementation Notes" title="Direct link to Language-Specific Implementation Notes" translate=no></a></h2> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=java>Java<a href=#java class=hash-link aria-label="Direct link to Java" title="Direct link to Java" translate=no></a></h3> |
| <ul> |
| <li class="">Uses runtime code generation (JIT) for maximum performance</li> |
| <li class="">Supports all reference tracking modes</li> |
| <li class="">Uses internal String coder for encoding selection</li> |
| <li class="">Thread-safe via <code>ThreadSafeFory</code> wrapper</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=python>Python<a href=#python class=hash-link aria-label="Direct link to Python" title="Direct link to Python" translate=no></a></h3> |
| <ul> |
| <li class="">Two modes: Pure Python (debugging) and Cython (performance)</li> |
| <li class="">Uses <code>id(obj)</code> for reference tracking</li> |
| <li class="">Latin1/UTF-16/UTF-8 encoding for all strings in xlang mode</li> |
| <li class=""><code>dataclass</code> support via code generation</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=c>C++<a href=#c class=hash-link aria-label="Direct link to C++" title="Direct link to C++" translate=no></a></h3> |
| <ul> |
| <li class="">Compile-time reflection via macros (<code>FORY_STRUCT</code>, <code>FORY_FIELD_INFO</code>)</li> |
| <li class="">Template meta programming for type dispatch and serializer selection</li> |
| <li class="">Uses <code>std::shared_ptr</code> for reference tracking</li> |
| <li class="">Compile-time field ordering</li> |
| <li class="">No runtime code generation</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=rust>Rust<a href=#rust class=hash-link aria-label="Direct link to Rust" title="Direct link to Rust" translate=no></a></h3> |
| <ul> |
| <li class="">Derive macros for automatic serialization (<code>#[derive(ForyObject)]</code>)</li> |
| <li class="">Uses <code>Rc<T></code> / <code>Arc<T></code> for reference tracking</li> |
| <li class="">Thread-local context caching for performance</li> |
| <li class="">Compile-time field ordering</li> |
| </ul> |
| <h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=go>Go<a href=#go class=hash-link aria-label="Direct link to Go" title="Direct link to Go" translate=no></a></h3> |
| <ul> |
| <li class="">Reflection-based and codegen-based modes</li> |
| <li class="">Struct tags for field annotations</li> |
| <li class="">Interface types for polymorphism</li> |
| </ul> |
| <h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=common-pitfalls>Common Pitfalls<a href=#common-pitfalls class=hash-link aria-label="Direct link to Common Pitfalls" title="Direct link to Common Pitfalls" translate=no></a></h2> |
| <ol> |
| <li class=""><strong>Byte Order</strong>: Always use little-endian for multi-byte values</li> |
| <li class=""><strong>Varint Sign Extension</strong>: Ensure proper handling of signed vs unsigned varints</li> |
| <li class=""><strong>Reference ID Ordering</strong>: IDs must be assigned in serialization order</li> |
| <li class=""><strong>Field Order Consistency</strong>: Must match exactly across languages (schema consistent mode only; in evolution mode, deserialization follows serialization field order from type meta)</li> |
| <li class=""><strong>String Encoding</strong>: Use best encoding for current language</li> |
| <li class=""><strong>Null Handling</strong>: Different languages represent null differently</li> |
| <li class=""><strong>Empty Collections</strong>: Still write length (0) and header byte</li> |
| <li class=""><strong>Type Hash Calculation</strong>: Must use exact same algorithm across languages</li> |
| </ol></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="row margin-top--sm theme-doc-footer-edit-meta-row"><div class="col noPrint_WFHX"><a href=https://github.com/apache/fory-site/tree/main/docs/specification/xlang_serialization_spec.md target=_blank rel="noopener noreferrer" class=theme-edit-this-page><svg fill=currentColor height=20 width=20 viewBox="0 0 40 40" class=iconEdit_Z9Sw aria-hidden=true><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"/></g></svg>Edit this page</a></div><div class="col lastUpdated_JAkA"></div></div></footer></article><nav class="docusaurus-mt-lg pagination-nav" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href=/docs/0.14/specification/java_serialization_spec><div class=pagination-nav__sublabel>Next</div><div class=pagination-nav__label>Java Serialization Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href=#cross-language-serialization-specification class="table-of-contents__link toc-highlight">Cross-language Serialization Specification</a><li><a href=#type-systems class="table-of-contents__link toc-highlight">Type Systems</a><ul><li><a href=#data-types class="table-of-contents__link toc-highlight">Data Types</a><li><a href=#polymorphisms class="table-of-contents__link toc-highlight">Polymorphisms</a><li><a href=#type-disambiguation class="table-of-contents__link toc-highlight">Type disambiguation</a><li><a href=#type-id class="table-of-contents__link toc-highlight">Type ID</a><li><a href=#type-mapping class="table-of-contents__link toc-highlight">Type mapping</a></ul><li><a href=#spec-overview class="table-of-contents__link toc-highlight">Spec overview</a><li><a href=#fory-header class="table-of-contents__link toc-highlight">Fory header</a><ul><li><a href=#language-ids class="table-of-contents__link toc-highlight">Language IDs</a><li><a href=#meta-start-offset class="table-of-contents__link toc-highlight">Meta Start Offset</a></ul><li><a href=#reference-meta class="table-of-contents__link toc-highlight">Reference Meta</a><ul><li><a href=#reference-flags class="table-of-contents__link toc-highlight">Reference Flags</a><li><a href=#reference-tracking-algorithm class="table-of-contents__link toc-highlight">Reference Tracking Algorithm</a><li><a href=#reference-id-assignment class="table-of-contents__link toc-highlight">Reference ID Assignment</a><li><a href=#when-reference-tracking-is-disabled class="table-of-contents__link toc-highlight">When Reference Tracking is Disabled</a><li><a href=#language-specific-considerations class="table-of-contents__link toc-highlight">Language-Specific Considerations</a></ul><li><a href=#type-meta class="table-of-contents__link toc-highlight">Type Meta</a><ul><li><a href=#struct-schema-consistent class="table-of-contents__link toc-highlight">Struct Schema consistent</a><li><a href=#struct-schema-evolution class="table-of-contents__link toc-highlight">Struct Schema evolution</a></ul><li><a href=#extended-type-meta-with-inheritance-support class="table-of-contents__link toc-highlight">Extended Type Meta with Inheritance support</a><ul><li><a href=#schema-consistent class="table-of-contents__link toc-highlight">Schema consistent</a><li><a href=#schema-evolution class="table-of-contents__link toc-highlight">Schema Evolution</a></ul><li><a href=#meta-string class="table-of-contents__link toc-highlight">Meta String</a><ul><li><a href=#encoding-type-ids class="table-of-contents__link toc-highlight">Encoding Type IDs</a><li><a href=#character-mapping-tables class="table-of-contents__link toc-highlight">Character Mapping Tables</a><li><a href=#encoding-algorithms class="table-of-contents__link toc-highlight">Encoding Algorithms</a><li><a href=#encoding-selection-algorithm class="table-of-contents__link toc-highlight">Encoding Selection Algorithm</a><li><a href=#meta-string-header-format class="table-of-contents__link toc-highlight">Meta String Header Format</a><li><a href=#special-character-sets-by-context class="table-of-contents__link toc-highlight">Special Character Sets by Context</a><li><a href=#deduplication class="table-of-contents__link toc-highlight">Deduplication</a></ul><li><a href=#value-format class="table-of-contents__link toc-highlight">Value Format</a><ul><li><a href=#basic-types class="table-of-contents__link toc-highlight">Basic types</a><li><a href=#string class="table-of-contents__link toc-highlight">string</a><li><a href=#duration class="table-of-contents__link toc-highlight">duration</a><li><a href=#collectionlist class="table-of-contents__link toc-highlight">collection/list</a><li><a href=#array class="table-of-contents__link toc-highlight">array</a><li><a href=#map class="table-of-contents__link toc-highlight">map</a><li><a href=#enum class="table-of-contents__link toc-highlight">enum</a><li><a href=#decimal class="table-of-contents__link toc-highlight">decimal</a><li><a href=#struct class="table-of-contents__link toc-highlight">struct</a><li><a href=#type class="table-of-contents__link toc-highlight">Type</a></ul><li><a href=#implementation-guidelines class="table-of-contents__link toc-highlight">Implementation guidelines</a><ul><li><a href=#how-to-reduce-memory-readwrite-code class="table-of-contents__link toc-highlight">How to reduce memory read/write code</a><li><a href=#fast-deserialization-for-static-languages-without-runtime-codegen-support class="table-of-contents__link toc-highlight">Fast deserialization for static languages without runtime codegen support</a></ul><li><a href=#implementation-checklist-for-new-languages class="table-of-contents__link toc-highlight">Implementation Checklist for New Languages</a><ul><li><a href=#phase-1-core-infrastructure class="table-of-contents__link toc-highlight">Phase 1: Core Infrastructure</a><li><a href=#phase-2-basic-type-serializers class="table-of-contents__link toc-highlight">Phase 2: Basic Type Serializers</a><li><a href=#phase-3-collection-types class="table-of-contents__link toc-highlight">Phase 3: Collection Types</a><li><a href=#phase-4-meta-string-encoding class="table-of-contents__link toc-highlight">Phase 4: Meta String Encoding</a><li><a href=#phase-5-enum-serialization class="table-of-contents__link toc-highlight">Phase 5: Enum Serialization</a><li><a href=#phase-6-struct-serialization class="table-of-contents__link toc-highlight">Phase 6: Struct Serialization</a><li><a href=#phase-7-other-types class="table-of-contents__link toc-highlight">Phase 7: Other types</a><li><a href=#testing-strategy class="table-of-contents__link toc-highlight">Testing Strategy</a></ul><li><a href=#language-specific-implementation-notes class="table-of-contents__link toc-highlight">Language-Specific Implementation Notes</a><ul><li><a href=#java class="table-of-contents__link toc-highlight">Java</a><li><a href=#python class="table-of-contents__link toc-highlight">Python</a><li><a href=#c class="table-of-contents__link toc-highlight">C++</a><li><a href=#rust class="table-of-contents__link toc-highlight">Rust</a><li><a href=#go class="table-of-contents__link toc-highlight">Go</a></ul><li><a href=#common-pitfalls class="table-of-contents__link toc-highlight">Common Pitfalls</a></ul></div></div></div></div></main></div></div></div><footer class="theme-layout-footer footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Community</div><ul class="footer__items clean-list"><li class=footer__item><a href=https://lists.apache.org/list.html?dev@fory.apache.org target=_blank rel="noopener noreferrer" class=footer__link-item>Mailing list<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://join.slack.com/t/fory-project/shared_invite/zt-1u8soj4qc-ieYEu7ciHOqA2mo47llS8A target=_blank rel="noopener noreferrer" class=footer__link-item>Slack<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://twitter.com/ApacheFory target=_blank rel="noopener noreferrer" class=footer__link-item>Twitter<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a></ul></div><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Docs</div><ul class="footer__items clean-list"><li class=footer__item><a class=footer__link-item href=/docs/start/install>Install</a><li class=footer__item><a class=footer__link-item href=/docs/start/usage>Usage</a><li class=footer__item><a class=footer__link-item href=/docs/introduction/benchmark>Benchmark</a></ul></div><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Repositories</div><ul class="footer__items clean-list"><li class=footer__item><a href=https://github.com/apache/fory target=_blank rel="noopener noreferrer" class=footer__link-item>Apache Fory™<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://github.com/apache/fory-site target=_blank rel="noopener noreferrer" class=footer__link-item>Website<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a></ul></div></div><div class="footer__bottom text--center"><div class=margin-bottom--sm><a href=https://apache.org/ rel="noopener noreferrer" class=footerLogoLink_BH7S><img src=/img/asf_logo.svg alt="ASF Logo" class="footer__logo themedComponent_mlkZ themedComponent--light_NVdE" width=200 /><img src=/img/asf_logo.svg alt="ASF Logo" class="footer__logo themedComponent_mlkZ themedComponent--dark_xIcU" width=200 /></a></div><div class=footer__copyright><div> |
| <p> |
| Copyright © 2026 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br/> |
| Apache Fory, Fory, Apache, the Apache Logo and the Apache Fory logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. |
| </p> |
| </div></div></div></div></footer></div></body> |
