Google Git
Sign in
apache/fory-site/refs/heads/deploy/./docs/0.16/specification/xlang_implementation_guide/index.html
blob: bb4a8f56c2cff64578660c8c85b80b1a1663e31e [file]
<!doctype html><html lang=en-US dir=ltr class="docs-wrapper plugin-docs plugin-id-default docs-version-0.16 docs-doc-page docs-doc-id-specification/xlang_implementation_guide" data-has-hydrated=false><head><meta charset=UTF-8><meta name=generator content="Docusaurus v3.10.1"><title data-rh=true>Xlang Implementation Guide | Apache Fory™</title><meta data-rh=true name=viewport content="width=device-width, initial-scale=1.0"/><meta data-rh=true property=og:url content=https://fory.apache.org/docs/0.16/specification/xlang_implementation_guide /><meta data-rh=true property=og:locale content=en_US /><meta data-rh=true property=og:locale:alternate content=zh_CN /><meta data-rh=true name=docusaurus_locale content=en-US /><meta data-rh=true name=docsearch:language content=en-US /><meta data-rh=true http-equiv=Content-Security-Policy content="frame-src 'self' https://ghbtns.com/;"/><meta data-rh=true property=og:image content=https://fory.apache.org/img/logo.png /><meta data-rh=true property=og:image:width content=1500 /><meta data-rh=true property=og:image:height content=1500 /><meta data-rh=true name=twitter:card content=summary_large_image /><meta data-rh=true name=twitter:image content=https://fory.apache.org/img/logo.png /><meta data-rh=true name=docusaurus_version content=0.16 /><meta data-rh=true name=docusaurus_tag content=docs-default-0.16 /><meta data-rh=true name=docsearch:version content=0.16 /><meta data-rh=true name=docsearch:docusaurus_tag content=docs-default-0.16 /><meta data-rh=true property=og:title content="Xlang Implementation Guide | Apache Fory™"/><meta data-rh=true name=description content="Implementation guidelines"/><meta data-rh=true property=og:description content="Implementation guidelines"/><link data-rh=true rel=icon href=/img/favicon.ico /><link data-rh=true rel=canonical href=https://fory.apache.org/docs/0.16/specification/xlang_implementation_guide /><link data-rh=true rel=alternate href=https://fory.apache.org/docs/0.16/specification/xlang_implementation_guide hreflang=en-US /><link data-rh=true rel=alternate href=https://fory.apache.org/zh-CN/docs/0.16/specification/xlang_implementation_guide hreflang=zh-CN /><link data-rh=true rel=alternate href=https://fory.apache.org/docs/0.16/specification/xlang_implementation_guide hreflang=x-default /><script data-rh=true type=application/ld+json>{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","item":"https://fory.apache.org/docs/0.16/specification/xlang_implementation_guide","name":"Xlang Implementation Guide","position":1}]}</script><link rel=alternate type=application/rss+xml href=/blog/rss.xml title="Apache Fory™ RSS Feed"><link rel=alternate type=application/atom+xml href=/blog/atom.xml title="Apache Fory™ Atom Feed"><script type=text/javascript>"fury.apache.org"===window.location.host&&(window.location.href="https://fory.apache.org"),function(){for(var o=window.location.pathname,a=["guide","introduction","start"],n=0;n<a.length;n++){var t=a[n],i=RegExp("^((?:/[a-z]{2}-[A-Z]{2})?/docs(?:/(?:next|[0-9]+.[0-9]+))?)/docs/"+t+"(/.*)?$"),c=o.match(i);if(c){var r=c[1]+"/"+t+(c[2]||"");window.location.replace(r+window.location.search+window.location.hash);return}}}()</script><link rel=stylesheet href=/assets/css/styles.3a1edaa1.css /><script src=/assets/js/runtime~main.bd6d811b.js defer></script><script src=/assets/js/main.a118c2a5.js defer></script></head><body><svg style="display: none;"><defs>
<symbol id=theme-svg-external-link viewBox="0 0 24 24"><path fill=currentColor d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"/></symbol>
</defs></svg>
<script>!function(){var t=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return window.localStorage.getItem("theme")}catch(t){}}();document.documentElement.setAttribute("data-theme",t||"light"),document.documentElement.setAttribute("data-theme-choice",t||"light")}(),function(){try{for(var[t,e]of new URLSearchParams(window.location.search).entries())if(t.startsWith("docusaurus-data-")){var a=t.replace("docusaurus-data-","data-");document.documentElement.setAttribute(a,e)}}catch(t){}}()</script><div id=__docusaurus><div role=region aria-label="Skip to main content"><a class=skipToContent_fXgn href=#__docusaurus_skipToContent_fallback>Skip to main content</a></div><nav aria-label=Main class="theme-layout-navbar navbar navbar--fixed-top"><div class=navbar__inner><div class="theme-layout-navbar-left navbar__items"><button aria-label="Toggle navigation bar" aria-expanded=false class="navbar__toggle clean-btn" type=button><svg width=30 height=30 viewBox="0 0 30 30" aria-hidden=true><path stroke=currentColor stroke-linecap=round stroke-miterlimit=10 stroke-width=2 d="M4 7h22M4 15h22M4 23h22"/></svg></button><a class=navbar__brand href=/><div class=navbar__logo><img src=/img/fory-logo-light.png alt="Apache Fory™ Logo" class="themedComponent_mlkZ themedComponent--light_NVdE"/><img src=/img/fory-logo-dark.png alt="Apache Fory™ Logo" class="themedComponent_mlkZ themedComponent--dark_xIcU"/></div><b class="navbar__title text--truncate"></b></a><a class="navbar__item navbar__link" href=/docs/0.16/introduction/overview>Docs</a><a aria-current=page class="navbar__item navbar__link navbar__link--active" href=/docs/0.16/specification/xlang_serialization_spec>Specification</a><a class="navbar__item navbar__link" href=/docs/0.16/community/>Community</a><a class="navbar__item navbar__link" href=/user>Users</a><a class="navbar__item navbar__link" href=/download>Download</a><a class="navbar__item navbar__link" href=/blog>Blog</a></div><div class="theme-layout-navbar-right navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href=# aria-haspopup=true aria-expanded=false role=button class=navbar__link>ASF</a><ul class=dropdown__menu><li><a href=https://www.apache.org/ target=_blank rel="noopener noreferrer" class=dropdown__link>Foundation</a><li><a href=https://www.apache.org/licenses/ target=_blank rel="noopener noreferrer" class=dropdown__link>License</a><li><a href=https://www.apache.org/events/current-event.html target=_blank rel="noopener noreferrer" class=dropdown__link>Events</a><li><a href=https://privacy.apache.org/policies/privacy-policy-public.html target=_blank rel="noopener noreferrer" class=dropdown__link>Privacy</a><li><a href=https://www.apache.org/security/ target=_blank rel="noopener noreferrer" class=dropdown__link>Security</a><li><a href=https://www.apache.org/foundation/sponsorship.html target=_blank rel="noopener noreferrer" class=dropdown__link>Sponsorship</a><li><a href=https://www.apache.org/foundation/thanks.html target=_blank rel="noopener noreferrer" class=dropdown__link>Thanks</a><li><a href=https://www.apache.org/foundation/policies/conduct.html target=_blank rel="noopener noreferrer" class=dropdown__link>Code of Conduct</a></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class=navbar__link aria-haspopup=true aria-expanded=false role=button href=/docs/0.16/specification/xlang_implementation_guide>0.16</a><ul class=dropdown__menu><li><a class=dropdown__link href=/docs/next/specification/xlang_implementation_guide>dev</a><li><a class=dropdown__link href=/docs/specification/xlang_implementation_guide>1.3.0</a><li><a class=dropdown__link href=/docs/1.2.0/specification/xlang_implementation_guide>1.2.0</a><li><a class=dropdown__link href=/docs/1.1.0/specification/xlang_implementation_guide>1.1.0</a><li><a class=dropdown__link href=/docs/1.0.0/specification/xlang_implementation_guide>1.0.0</a><li><a class=dropdown__link href=/docs/0.17/specification/xlang_implementation_guide>0.17</a><li><a aria-current=page class="dropdown__link dropdown__link--active" href=/docs/0.16/specification/xlang_implementation_guide>0.16</a><li><a class=dropdown__link href=/docs/0.15/specification/xlang_implementation_guide>0.15</a><li><a class=dropdown__link href=/docs/0.14/introduction/overview>0.14</a><li><a class=dropdown__link href=/docs/0.13/introduction/overview>0.13</a><li><a class=dropdown__link href=/docs/0.12/introduction/overview>0.12</a><li><a class=dropdown__link href=/docs/0.11/introduction/overview>0.11</a><li><a class=dropdown__link href=/docs/0.10/introduction/overview>0.10</a></ul></div><a href=https://github.com/apache/fory target=_blank rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href=# aria-haspopup=true aria-expanded=false role=button class=navbar__link><svg viewBox="0 0 24 24" width=20 height=20 aria-hidden=true class=iconLanguage_nlXk><path fill=currentColor d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"/></svg>English</a><ul class=dropdown__menu><li><a href=/docs/0.16/specification/xlang_implementation_guide target=_self rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang=en-US>English</a><li><a href=/zh-CN/docs/0.16/specification/xlang_implementation_guide target=_self rel="noopener noreferrer" class=dropdown__link lang=zh-CN>简体中文</a></ul></div><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type=button disabled title="system mode" aria-label="Switch between dark and light mode (currently system mode)"><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP lightToggleIcon_pyhR"><path fill=currentColor d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP darkToggleIcon_wfgR"><path fill=currentColor d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg><svg viewBox="0 0 24 24" width=24 height=24 aria-hidden=true class="toggleIcon_g3eP systemToggleIcon_QzmC"><path fill=currentColor d="m12 21c4.971 0 9-4.029 9-9s-4.029-9-9-9-9 4.029-9 9 4.029 9 9 9zm4.95-13.95c1.313 1.313 2.05 3.093 2.05 4.95s-0.738 3.637-2.05 4.95c-1.313 1.313-3.093 2.05-4.95 2.05v-14c1.857 0 3.637 0.737 4.95 2.05z"/></svg></button></div><div class=navbarSearchContainer_Bca1><div class=navbar__search><span aria-label="expand searchbar" role=button class=search-icon tabindex=0></span><input id=search_input_react type=search placeholder=Loading... aria-label=Search class="navbar__search-input search-bar" disabled/></div></div></div></div><div role=presentation class=navbar-sidebar__backdrop></div></nav><div id=__docusaurus_skipToContent_fallback class="theme-layout-main main-wrapper mainWrapper_z2l0"><div class=docsWrapper_hBAB><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type=button></button><div class=docRoot_UBD9><aside class="theme-doc-sidebar-container docSidebarContainer_YfHR"><div class=sidebarViewport_aRkj><div class=sidebar_njMd><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.16/specification/xlang_serialization_spec><span title="Xlang Serialization Format" class=linkLabel_WmDU>Xlang Serialization Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.16/specification/java_serialization_spec><span title="Java Serialization Format" class=linkLabel_WmDU>Java Serialization Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.16/specification/row_format_spec><span title="Row Format" class=linkLabel_WmDU>Row Format</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class=menu__link href=/docs/0.16/specification/xlang_type_mapping><span title="Xlang Type Mapping" class=linkLabel_WmDU>Xlang Type Mapping</span></a><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current=page href=/docs/0.16/specification/xlang_implementation_guide><span title="Xlang Implementation Guide" class=linkLabel_WmDU>Xlang Implementation Guide</span></a></ul></nav></div></div></aside><main class=docMainContainer_TBSr><div class="container padding-top--md padding-bottom--lg"><div class=row><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role=alert><div>This is documentation for <!-- -->Apache Fory™<!-- --> <b>0.16</b>, which is no longer actively maintained.</div><div class=margin-top--md>For up-to-date documentation, see the <b><a href=/docs/specification/xlang_implementation_guide>latest version</a></b> (<!-- -->1.3.0<!-- -->).</div></div><div class=docItemContainer_Djhp><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label=Breadcrumbs><ul class=breadcrumbs><li class=breadcrumbs__item><a aria-label="Home page" class=breadcrumbs__link href=/><svg viewBox="0 0 24 24" class=breadcrumbHomeIcon_YNFT><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill=currentColor /></svg></a><li class="breadcrumbs__item breadcrumbs__item--active"><span class=breadcrumbs__link>Xlang Implementation Guide</span></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.16</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type=button class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Xlang Implementation Guide</h1></header><h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=implementation-guidelines>Implementation guidelines<a href=#implementation-guidelines class=hash-link aria-label="Direct link to Implementation guidelines" title="Direct link to Implementation guidelines" translate=no>​</a></h2>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=how-to-reduce-memory-readwrite-code>How to reduce memory read/write code<a href=#how-to-reduce-memory-readwrite-code class=hash-link aria-label="Direct link to How to reduce memory read/write code" title="Direct link to How to reduce memory read/write code" translate=no>​</a></h3>
<ul>
<li class="">Try to merge multiple bytes into an int/long write before writing to reduce memory IO and bound check cost.</li>
<li class="">Read multiple bytes as an int/long, then split into multiple bytes to reduce memory IO and bound check cost.</li>
<li class="">Try to use one varint/long to write flags and length together to save one byte cost and reduce memory io.</li>
<li class="">Condition branches are less expensive compared to memory IO cost unless there are too many branches.</li>
</ul>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=fast-deserialization-for-static-languages-without-runtime-codegen-support>Fast deserialization for static languages without runtime codegen support<a href=#fast-deserialization-for-static-languages-without-runtime-codegen-support class=hash-link aria-label="Direct link to Fast deserialization for static languages without runtime codegen support" title="Direct link to Fast deserialization for static languages without runtime codegen support" translate=no>​</a></h3>
<p>For type evolution, the serializer will encode the type meta into the serialized data. The deserializer will compare
this meta with class meta in the current process, and use the diff to determine how to deserialize the data.</p>
<p>For java/javascript/python, we can use the diff to generate serializer code at runtime and load it as class/function for
deserialization. In this way, the type evolution will be as fast as type consist mode.</p>
<p>For C++/Rust, we can't generate the serializer code at runtime. So we need to generate the code at compile-time using
meta programming. But at that time, we don't know the type schema in other processes, so we can't generate the
serializer code for such inconsistent types. We may need to generate the code which has a loop and compare field name
one by one to decide whether to deserialize and assign the field or skip the field value.</p>
<p>One fast way is that we can optimize the string comparison into <code>jump</code> instructions:</p>
<ul>
<li class="">Assume the current type has <code>n</code> fields, and the peer type has <code>n1</code> fields.</li>
<li class="">Generate an auto growing <code>field id</code> from <code>0</code> for every sorted field in the current type at the compile time.</li>
<li class="">Compare the received type meta with current type, generate same id if the field name is same, otherwise generate an
auto growing id starting from <code>n</code>, cache this meta at runtime.</li>
<li class="">Iterate the fields of received type meta, use a <code>switch</code> to compare the <code>field id</code> to deserialize data
and <code>assign/skip</code> field value. <strong>Continuous</strong> field id will be optimized into <code>jump</code> in <code>switch</code> block, so it will
very fast.</li>
</ul>
<p>Here is an example, suppose process A has a class <code>Foo</code> with version 1 defined as <code>Foo1</code>, process B has a class <code>Foo</code>
with version 2 defined as <code>Foo2</code>:</p>
<div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-c++ codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">// class Foo with version 1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">class Foo1 {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> int32_t v1; // id 0</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> std::string v2; // id 1</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">};</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">// class Foo with version 2</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">class Foo2 {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 0, but will have id 2 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> bool v0;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 1, but will have id 0 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> int32_t v1;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 2, but will have id 3 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> int64_t long_value;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 3, but will have id 1 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> std::string v2;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> // id 4, but will have id 4 in process A</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> std::vector&lt;std::string> list;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">};</span><br/></div></code></pre></div></div>
<p>When process A received serialized <code>Foo2</code> from process B, here is how it deserialize the data:</p>
<div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style=--prism-color:#393A34;--prism-background-color:#f6f8fa><div class=codeBlockContent_QJqH><pre tabindex=0 class="prism-code language-c++ codeBlock_bY9V thin-scrollbar" style=color:#393A34;background-color:#f6f8fa><code class=codeBlockLines_e6Vv><div class=token-line style=color:#393A34><span class="token plain">Foo1 foo1 = ...;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">const std::vector&lt;fory::FieldInfo> &field_infos = type_meta.field_infos;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">for (const auto &field_info : field_infos) {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> switch (field_info.field_id) {</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case 0:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> foo1.v1 = buffer.read_varint32();</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> break;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> case 1:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> foo1.v2 = fory.read_string();</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> break;</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> default:</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> fory.skip_data(field_info);</span><br/></div><div class=token-line style=color:#393A34><span class="token plain"> }</span><br/></div><div class=token-line style=color:#393A34><span class="token plain">}</span><br/></div></code></pre></div></div>
<h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=implementation-checklist-for-new-languages>Implementation Checklist for New Languages<a href=#implementation-checklist-for-new-languages class=hash-link aria-label="Direct link to Implementation Checklist for New Languages" title="Direct link to Implementation Checklist for New Languages" translate=no>​</a></h2>
<p>This section provides a step-by-step guide for implementing Fory xlang serialization in a new language.</p>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-1-core-infrastructure>Phase 1: Core Infrastructure<a href=#phase-1-core-infrastructure class=hash-link aria-label="Direct link to Phase 1: Core Infrastructure" title="Direct link to Phase 1: Core Infrastructure" translate=no>​</a></h3>
<ol>
<li class="">
<p><strong>Buffer Implementation</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Create a byte buffer with read/write cursor tracking</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement little-endian byte order for all multi-byte writes</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_int8</code>, <code>write_int16</code>, <code>write_int32</code>, <code>write_int64</code></li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_float32</code>, <code>write_float64</code></li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>read_*</code> counterparts for all write methods</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement buffer growth strategy (e.g., doubling)</li>
</ul>
</li>
<li class="">
<p><strong>Varint Encoding</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varuint32</code> / <code>read_varuint32</code></li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varint32</code> / <code>read_varint32</code> (with ZigZag)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varuint64</code> / <code>read_varuint64</code></li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varint64</code> / <code>read_varint64</code> (with ZigZag)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement <code>write_varuint36_small</code> / <code>read_varuint36_small</code> (for strings)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Optionally implement Hybrid encoding (TAGGED_INT64/TAGGED_UINT64) for int64</li>
</ul>
</li>
<li class="">
<p><strong>Header Handling</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write/read bitmap flags (null, xlang, oob)</li>
</ul>
</li>
</ol>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-2-basic-type-serializers>Phase 2: Basic Type Serializers<a href=#phase-2-basic-type-serializers class=hash-link aria-label="Direct link to Phase 2: Basic Type Serializers" title="Direct link to Phase 2: Basic Type Serializers" translate=no>​</a></h3>
<ol start=4>
<li class="">
<p><strong>Primitive Types</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->bool (1 byte: 0 or 1)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->int8, int16, int32, int64 (little endian)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->float32, float64 (IEEE 754, little endian)</li>
</ul>
</li>
<li class="">
<p><strong>String Serialization</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement string header: <code>(byte_length &lt;&lt; 2) | encoding</code></li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Support UTF-8 encoding (required for xlang)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Optionally support LATIN1 and UTF-16</li>
</ul>
</li>
<li class="">
<p><strong>Temporal Types</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Duration (seconds + nanoseconds)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Timestamp (seconds + nanoseconds since epoch)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Date (days since epoch)</li>
</ul>
</li>
<li class="">
<p><strong>Reference Tracking</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement write-side object tracking (object → ref_id map)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement read-side object tracking (ref_id → object list)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle all four reference flags: NULL(-3), REF(-2), NOT_NULL(-1), REF_VALUE(0)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Support disabling reference tracking per-type or globally</li>
</ul>
</li>
</ol>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-3-collection-types>Phase 3: Collection Types<a href=#phase-3-collection-types class=hash-link aria-label="Direct link to Phase 3: Collection Types" title="Direct link to Phase 3: Collection Types" translate=no>​</a></h3>
<ol start=8>
<li class="">
<p><strong>List/Array Serialization</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write length as varuint32</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write elements header byte</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle homogeneous vs heterogeneous elements</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle null elements</li>
</ul>
</li>
<li class="">
<p><strong>Map Serialization</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write total size as varuint32</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement chunk-based format (max 255 pairs per chunk)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write KV header byte per chunk</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Handle key and value type variations</li>
</ul>
</li>
<li class="">
<p><strong>Set Serialization</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Same format as List (reuse implementation)</li>
</ul>
</li>
</ol>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-4-meta-string-encoding>Phase 4: Meta String Encoding<a href=#phase-4-meta-string-encoding class=hash-link aria-label="Direct link to Phase 4: Meta String Encoding" title="Direct link to Phase 4: Meta String Encoding" translate=no>​</a></h3>
<p>Meta strings are required for enum and struct serialization (encoding field names, type names, namespaces).</p>
<ol start=11>
<li class=""><strong>Meta String Compression</strong>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement LOWER_SPECIAL encoding (5 bits/char)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement LOWER_UPPER_DIGIT_SPECIAL encoding (6 bits/char)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement FIRST_TO_LOWER_SPECIAL encoding</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement ALL_TO_LOWER_SPECIAL encoding</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement encoding selection algorithm</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement meta string deduplication</li>
</ul>
</li>
</ol>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-5-enum-serialization>Phase 5: Enum Serialization<a href=#phase-5-enum-serialization class=hash-link aria-label="Direct link to Phase 5: Enum Serialization" title="Direct link to Phase 5: Enum Serialization" translate=no>​</a></h3>
<ol start=12>
<li class=""><strong>Enum Serialization</strong>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write ordinal as varuint32</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Support named enum (namespace + type name)</li>
</ul>
</li>
</ol>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-6-struct-serialization>Phase 6: Struct Serialization<a href=#phase-6-struct-serialization class=hash-link aria-label="Direct link to Phase 6: Struct Serialization" title="Direct link to Phase 6: Struct Serialization" translate=no>​</a></h3>
<ol start=13>
<li class="">
<p><strong>Type Registration</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Support registration by numeric ID</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Support registration by namespace + type name</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Maintain type → serializer mapping</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Generate type IDs: write internal type ID, then <code>user_type_id</code> as varuint32</li>
</ul>
</li>
<li class="">
<p><strong>Field Ordering</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement the spec-defined grouping and ordering (primitive/boxed/built-in, collections/maps, other)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Use a stable comparator within each group (type ID and name)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Use tag ID or snake_case field name as field identifier for fingerprints</li>
</ul>
</li>
<li class="">
<p><strong>Schema Consistent Mode</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->If class-version check is enabled, compute schema hash from field identifiers</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Write 4-byte schema hash before fields</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Serialize fields in Fory order</li>
</ul>
</li>
<li class="">
<p><strong>Compatible/Meta Share Mode</strong></p>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Implement shared TypeDef stream (inline new TypeDefs, index references)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Map fields by name or tag ID, skip unknown fields</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Apply nullable/ref flags from TypeDef metadata</li>
</ul>
</li>
</ol>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=phase-7-other-types>Phase 7: Other types<a href=#phase-7-other-types class=hash-link aria-label="Direct link to Phase 7: Other types" title="Direct link to Phase 7: Other types" translate=no>​</a></h3>
<ol start=17>
<li class=""><strong>Binary/Array Types</strong></li>
</ol>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Primitive arrays (direct buffer copy)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Multi-dimensional arrays as nested lists (no tensor encoding)</li>
</ul>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=testing-strategy>Testing Strategy<a href=#testing-strategy class=hash-link aria-label="Direct link to Testing Strategy" title="Direct link to Testing Strategy" translate=no>​</a></h3>
<ol start=18>
<li class=""><strong>Cross-Language Compatibility Tests</strong>
<ul class="contains-task-list containsTaskList_mC6p">
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Serialize in new language, deserialize in Java/Python</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Serialize in Java/Python, deserialize in new language</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Test all primitive types</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Test strings with various encodings</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Test collections (empty, single, multiple elements)</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Test maps with various key/value types</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Test nested structs</li>
<li class=task-list-item><input type=checkbox disabled/> <!-- -->Test circular references (if supported)</li>
</ul>
</li>
</ol>
<h2 class="anchor anchorTargetStickyNavbar_Vzrq" id=language-specific-implementation-notes>Language-Specific Implementation Notes<a href=#language-specific-implementation-notes class=hash-link aria-label="Direct link to Language-Specific Implementation Notes" title="Direct link to Language-Specific Implementation Notes" translate=no>​</a></h2>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=java>Java<a href=#java class=hash-link aria-label="Direct link to Java" title="Direct link to Java" translate=no>​</a></h3>
<ul>
<li class="">Uses runtime code generation (JIT) for maximum performance</li>
<li class="">Supports all reference tracking modes</li>
<li class="">Uses internal String coder for encoding selection</li>
<li class="">Thread-safe via <code>ThreadSafeFory</code> wrapper</li>
</ul>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=python>Python<a href=#python class=hash-link aria-label="Direct link to Python" title="Direct link to Python" translate=no>​</a></h3>
<ul>
<li class="">Two modes: Pure Python (debugging) and Cython (performance)</li>
<li class="">Uses <code>id(obj)</code> for reference tracking</li>
<li class="">Latin1/UTF-16/UTF-8 encoding for all strings in xlang mode</li>
<li class=""><code>dataclass</code> support via code generation</li>
</ul>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=c>C++<a href=#c class=hash-link aria-label="Direct link to C++" title="Direct link to C++" translate=no>​</a></h3>
<ul>
<li class="">Compile-time reflection via macros (<code>FORY_STRUCT</code>)</li>
<li class="">Template meta programming for type dispatch and serializer selection</li>
<li class="">Uses <code>std::shared_ptr</code> for reference tracking</li>
<li class="">Compile-time field ordering</li>
<li class="">No runtime code generation</li>
</ul>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=rust>Rust<a href=#rust class=hash-link aria-label="Direct link to Rust" title="Direct link to Rust" translate=no>​</a></h3>
<ul>
<li class="">Derive macros for automatic serialization (<code>#[derive(ForyObject)]</code>)</li>
<li class="">Uses <code>Rc&lt;T></code> / <code>Arc&lt;T></code> for reference tracking</li>
<li class="">Thread-local context caching for performance</li>
<li class="">Compile-time field ordering</li>
</ul>
<h3 class="anchor anchorTargetStickyNavbar_Vzrq" id=go>Go<a href=#go class=hash-link aria-label="Direct link to Go" title="Direct link to Go" translate=no>​</a></h3>
<ul>
<li class="">Reflection-based and codegen-based modes</li>
<li class="">Struct tags for field annotations</li>
<li class="">Interface types for polymorphism</li>
</ul></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="row margin-top--sm theme-doc-footer-edit-meta-row"><div class="col noPrint_WFHX"><a href=https://github.com/apache/fory-site/tree/main/docs/specification/xlang_implementation_guide.md target=_blank rel="noopener noreferrer" class=theme-edit-this-page><svg fill=currentColor height=20 width=20 viewBox="0 0 40 40" class=iconEdit_Z9Sw aria-hidden=true><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"/></g></svg>Edit this page</a></div><div class="col lastUpdated_JAkA"></div></div></footer></article><nav class="docusaurus-mt-lg pagination-nav" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href=/docs/0.16/specification/xlang_type_mapping><div class=pagination-nav__sublabel>Previous</div><div class=pagination-nav__label>Xlang Type Mapping</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href=#implementation-guidelines class="table-of-contents__link toc-highlight">Implementation guidelines</a><ul><li><a href=#how-to-reduce-memory-readwrite-code class="table-of-contents__link toc-highlight">How to reduce memory read/write code</a><li><a href=#fast-deserialization-for-static-languages-without-runtime-codegen-support class="table-of-contents__link toc-highlight">Fast deserialization for static languages without runtime codegen support</a></ul><li><a href=#implementation-checklist-for-new-languages class="table-of-contents__link toc-highlight">Implementation Checklist for New Languages</a><ul><li><a href=#phase-1-core-infrastructure class="table-of-contents__link toc-highlight">Phase 1: Core Infrastructure</a><li><a href=#phase-2-basic-type-serializers class="table-of-contents__link toc-highlight">Phase 2: Basic Type Serializers</a><li><a href=#phase-3-collection-types class="table-of-contents__link toc-highlight">Phase 3: Collection Types</a><li><a href=#phase-4-meta-string-encoding class="table-of-contents__link toc-highlight">Phase 4: Meta String Encoding</a><li><a href=#phase-5-enum-serialization class="table-of-contents__link toc-highlight">Phase 5: Enum Serialization</a><li><a href=#phase-6-struct-serialization class="table-of-contents__link toc-highlight">Phase 6: Struct Serialization</a><li><a href=#phase-7-other-types class="table-of-contents__link toc-highlight">Phase 7: Other types</a><li><a href=#testing-strategy class="table-of-contents__link toc-highlight">Testing Strategy</a></ul><li><a href=#language-specific-implementation-notes class="table-of-contents__link toc-highlight">Language-Specific Implementation Notes</a><ul><li><a href=#java class="table-of-contents__link toc-highlight">Java</a><li><a href=#python class="table-of-contents__link toc-highlight">Python</a><li><a href=#c class="table-of-contents__link toc-highlight">C++</a><li><a href=#rust class="table-of-contents__link toc-highlight">Rust</a><li><a href=#go class="table-of-contents__link toc-highlight">Go</a></ul></ul></div></div></div></div></main></div></div></div><footer class="theme-layout-footer footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Community</div><ul class="footer__items clean-list"><li class=footer__item><a href=https://lists.apache.org/list.html?dev@fory.apache.org target=_blank rel="noopener noreferrer" class=footer__link-item>Mailing list<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://join.slack.com/t/fory-project/shared_invite/zt-1u8soj4qc-ieYEu7ciHOqA2mo47llS8A target=_blank rel="noopener noreferrer" class=footer__link-item>Slack<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://twitter.com/ApacheFory target=_blank rel="noopener noreferrer" class=footer__link-item>Twitter<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a></ul></div><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Docs</div><ul class="footer__items clean-list"><li class=footer__item><a class=footer__link-item href=/docs/start/install>Install</a><li class=footer__item><a class=footer__link-item href=/docs/start/usage>Usage</a><li class=footer__item><a class=footer__link-item href=/docs/introduction/benchmark>Benchmark</a></ul></div><div class="theme-layout-footer-column col footer__col"><div class=footer__title>Repositories</div><ul class="footer__items clean-list"><li class=footer__item><a href=https://github.com/apache/fory target=_blank rel="noopener noreferrer" class=footer__link-item>Apache Fory™<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a><li class=footer__item><a href=https://github.com/apache/fory-site target=_blank rel="noopener noreferrer" class=footer__link-item>Website<svg width=13.5 height=13.5 aria-label="(opens in new tab)" class=iconExternalLink_nPIU><use href=#theme-svg-external-link /></svg></a></ul></div></div><div class="footer__bottom text--center"><div class=margin-bottom--sm><a href=https://apache.org/ rel="noopener noreferrer" class=footerLogoLink_BH7S><img src=/img/asf_logo.svg alt="ASF Logo" class="footer__logo themedComponent_mlkZ themedComponent--light_NVdE" width=200 /><img src=/img/asf_logo.svg alt="ASF Logo" class="footer__logo themedComponent_mlkZ themedComponent--dark_xIcU" width=200 /></a></div><div class=footer__copyright><div>
<p>
Copyright © 2026 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br/>
Apache Fory, Fory, Apache, the Apache Logo and the Apache Fory logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.
</p>
</div></div></div></div></footer></div></body>