| <!doctype html> |
| <html lang="en-US" data-theme="light"> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width,initial-scale=1" /> |
| <meta name="generator" content="VuePress 2.0.0-rc.9" /> |
| <meta name="theme" content="VuePress Theme Hope 2.0.0-rc.34" /> |
| <style> |
| html { |
| background: var(--bg-color, #fff); |
| } |
| |
| html[data-theme="dark"] { |
| background: var(--bg-color, #1d1e1f); |
| } |
| |
| body { |
| background: var(--bg-color); |
| } |
| </style> |
| <script> |
| const userMode = localStorage.getItem("vuepress-theme-hope-scheme"); |
| const systemDarkMode = |
| window.matchMedia && |
| window.matchMedia("(prefers-color-scheme: dark)").matches; |
| |
| if (userMode === "dark" || (userMode !== "light" && systemDarkMode)) { |
| document.documentElement.setAttribute("data-theme", "dark"); |
| } |
| </script> |
| <link rel="alternate" hreflang="zh-cn" href="https://iotdb.apache.org/zh/UserGuide/latest/User-Manual/Data-Sync.html"><meta property="og:url" content="https://iotdb.apache.org/UserGuide/latest/User-Manual/Data-Sync.html"><meta property="og:site_name" content="IoTDB Website"><meta property="og:title" content="IoTDB Data Sync"><meta property="og:description" content="IoTDB Data Sync The IoTDB data sync transfers data from IoTDB to another data platform, and . A Pipe consists of three subtasks (plugins): Source Process Sink Pipe allows users ..."><meta property="og:type" content="article"><meta property="og:image" content="https://alioss.timecho.com/docs/img/1706698537700.jpg"><meta property="og:locale" content="en-US"><meta property="og:locale:alternate" content="zh-CN"><meta property="og:updated_time" content="2024-04-08T07:45:44.000Z"><meta property="article:modified_time" content="2024-04-08T07:45:44.000Z"><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","headline":"IoTDB Data Sync","image":["https://alioss.timecho.com/docs/img/1706698537700.jpg","https://alioss.timecho.com/docs/img/%E7%8A%B6%E6%80%81%E8%BF%81%E7%A7%BB%E5%9B%BE.png"],"dateModified":"2024-04-08T07:45:44.000Z","author":[]}</script><link rel="icon" href="/favicon.ico"><meta name="Description" content="Apache IoTDB: Time Series Database for IoT"><meta name="Keywords" content="TSDB, time series, time series database, IoTDB, IoT database, IoT data management,时序数据库, 时间序列管理, IoTDB, 物联网数据库, 实时数据库, 物联网数据管理, 物联网数据"><meta name="baidu-site-verification" content="wfKETzB3OT"><meta name="google-site-verification" content="mZWAoRY0yj_HAr-s47zHCGHzx5Ju-RVm5wDbPnwQYFo"><script type="text/javascript"> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| _paq.push(["setDoNotTrack", true]); |
| _paq.push(["disableCookies"]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '56']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script><title>IoTDB Data Sync | IoTDB Website</title><meta name="description" content="IoTDB Data Sync The IoTDB data sync transfers data from IoTDB to another data platform, and . A Pipe consists of three subtasks (plugins): Source Process Sink Pipe allows users ..."> |
| <link rel="preload" href="/assets/style-DnEHAOmf.css" as="style"><link rel="stylesheet" href="/assets/style-DnEHAOmf.css"> |
| <link rel="modulepreload" href="/assets/app-DrPcRZG6.js"><link rel="modulepreload" href="/assets/Data-Sync.html-DeQ437tp.js"> |
| |
| </head> |
| <body> |
| <div id="app"><!--[--><!--[--><!--[--><span tabindex="-1"></span><a href="#main-content" class="vp-skip-link sr-only">Skip to main content</a><!--]--><!--[--><div class="theme-container has-toc"><!--[--><header id="navbar" class="vp-navbar hide-icon"><div class="vp-navbar-start"><button type="button" class="vp-toggle-sidebar-button" title="Toggle Sidebar"><span class="icon"></span></button><!--[--><!----><!--]--><!--[--><a class="route-link vp-brand" href="/"><img class="vp-nav-logo" src="/logo.png" alt><!----><span class="vp-site-name hide-in-pad">IoTDB Website</span></a><!--]--><!--[--><!----><!--]--></div><div class="vp-navbar-center"><!--[--><!----><!--]--><!--[--><!--]--><!--[--><!----><!--]--></div><div class="vp-navbar-end"><!--[--><!----><!--]--><!--[--><!--[--><div id="docsearch-container" style="display:none;"></div><div><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"><svg width="15" height="15" class="DocSearch-Control-Key-Icon"><path d="M4.505 4.496h2M5.505 5.496v5M8.216 4.496l.055 5.993M10 7.5c.333.333.5.667.5 1v2M12.326 4.5v5.996M8.384 4.496c1.674 0 2.116 0 2.116 1.5s-.442 1.5-2.116 1.5M3.205 9.303c-.09.448-.277 1.21-1.241 1.203C1 10.5.5 9.513.5 8V7c0-1.57.5-2.5 1.464-2.494.964.006 1.134.598 1.24 1.342M12.553 10.5h1.953" stroke-width="1.2" stroke="currentColor" fill="none" stroke-linecap="square"></path></svg></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--><nav class="vp-nav-links"><div class="vp-nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="Documentation"><span class="title"><!---->Documentation</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a class="route-link nav-link" href="/UserGuide/latest/QuickStart/QuickStart.html" aria-label="v1.3.x"><!---->v1.3.x<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/UserGuide/V1.2.x/QuickStart/QuickStart.html" aria-label="v1.2.x"><!---->v1.2.x<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/UserGuide/V1.1.x/QuickStart/QuickStart.html" aria-label="v1.1.x"><!---->v1.1.x<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/UserGuide/V1.0.x/QuickStart/QuickStart.html" aria-label="v1.0.x"><!---->v1.0.x<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/UserGuide/V0.13.x/QuickStart/QuickStart.html" aria-label="v0.13.x"><!---->v0.13.x<!----></a></li></ul></button></div></div><div class="vp-nav-item hide-in-mobile"><a href="https://cwiki.apache.org/confluence/display/IOTDB/System+Design" rel="noopener noreferrer" target="_blank" aria-label="Design" class="nav-link"><!---->Design<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></div><div class="vp-nav-item hide-in-mobile"><a class="route-link nav-link" href="/Download/" aria-label="Download"><!---->Download<!----></a></div><div class="vp-nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="Community"><span class="title"><!---->Community</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a class="route-link nav-link" href="/Community/About.html" aria-label="About"><!---->About<!----></a></li><li class="dropdown-item"><a href="https://cwiki.apache.org/confluence/display/iotdb" rel="noopener noreferrer" target="_blank" aria-label="Wiki" class="nav-link"><!---->Wiki<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Community/Community-Project-Committers.html" aria-label="People"><!---->People<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Community/Community-Powered-By.html" aria-label="Powered By"><!---->Powered By<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Community/Materials.html" aria-label="Resources"><!---->Resources<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Community/Feedback.html" aria-label="Feedback"><!---->Feedback<!----></a></li></ul></button></div></div><div class="vp-nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="Development"><span class="title"><!---->Development</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a class="route-link nav-link" href="/Development/VoteRelease.html" aria-label="How to vote"><!---->How to vote<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/HowToCommit.html" aria-label="How to Commit"><!---->How to Commit<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/HowToJoin.html" aria-label="Become a Contributor"><!---->Become a Contributor<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/Committer.html" aria-label="Become a Committer"><!---->Become a Committer<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/ContributeGuide.html" aria-label="ContributeGuide"><!---->ContributeGuide<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/HowtoContributeCode.html" aria-label="How to Contribute Code"><!---->How to Contribute Code<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/format-changelist.html" aria-label="Changelist of TsFile"><!---->Changelist of TsFile<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/Development/rpc-changelist.html" aria-label="Changelist of RPC"><!---->Changelist of RPC<!----></a></li></ul></button></div></div><div class="vp-nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="ASF"><span class="title"><!---->ASF</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a href="https://www.apache.org/" rel="noopener noreferrer" target="_blank" aria-label="Foundation" class="nav-link"><!---->Foundation<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/licenses/" rel="noopener noreferrer" target="_blank" aria-label="License" class="nav-link"><!---->License<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/security/" rel="noopener noreferrer" target="_blank" aria-label="Security" class="nav-link"><!---->Security<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/foundation/sponsorship.html" rel="noopener noreferrer" target="_blank" aria-label="Sponsorship" class="nav-link"><!---->Sponsorship<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/foundation/thanks.html" rel="noopener noreferrer" target="_blank" aria-label="Thanks" class="nav-link"><!---->Thanks<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/events/current-event" rel="noopener noreferrer" target="_blank" aria-label="Current Events" class="nav-link"><!---->Current Events<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" rel="noopener noreferrer" target="_blank" aria-label="Privacy" class="nav-link"><!---->Privacy<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li></ul></button></div></div></nav><div class="vp-nav-item"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="Select language"><!--[--><svg xmlns="http://www.w3.org/2000/svg" class="icon i18n-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="i18n icon" style="width:1rem;height:1rem;vertical-align:middle;"><path d="M379.392 460.8 494.08 575.488l-42.496 102.4L307.2 532.48 138.24 701.44l-71.68-72.704L234.496 460.8l-45.056-45.056c-27.136-27.136-51.2-66.56-66.56-108.544h112.64c7.68 14.336 16.896 27.136 26.112 35.84l45.568 46.08 45.056-45.056C382.976 312.32 409.6 247.808 409.6 204.8H0V102.4h256V0h102.4v102.4h256v102.4H512c0 70.144-37.888 161.28-87.04 210.944L378.88 460.8zM576 870.4 512 1024H409.6l256-614.4H768l256 614.4H921.6l-64-153.6H576zM618.496 768h196.608L716.8 532.48 618.496 768z"></path></svg><!--]--><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a class="route-link nav-link active" href="/UserGuide/latest/User-Manual/Data-Sync.html" aria-label="English"><!---->English<!----></a></li><li class="dropdown-item"><a class="route-link nav-link" href="/zh/UserGuide/latest/User-Manual/Data-Sync.html" aria-label="简体中文"><!---->简体中文<!----></a></li></ul></button></div></div><div class="vp-nav-item hide-in-mobile"><button type="button" id="appearance-switch"><svg xmlns="http://www.w3.org/2000/svg" class="icon auto-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="auto icon" style="display:none;"><path d="M512 992C246.92 992 32 777.08 32 512S246.92 32 512 32s480 214.92 480 480-214.92 480-480 480zm0-840c-198.78 0-360 161.22-360 360 0 198.84 161.22 360 360 360s360-161.16 360-360c0-198.78-161.22-360-360-360zm0 660V212c165.72 0 300 134.34 300 300 0 165.72-134.28 300-300 300z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" class="icon dark-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="dark icon" style="display:none;"><path d="M524.8 938.667h-4.267a439.893 439.893 0 0 1-313.173-134.4 446.293 446.293 0 0 1-11.093-597.334A432.213 432.213 0 0 1 366.933 90.027a42.667 42.667 0 0 1 45.227 9.386 42.667 42.667 0 0 1 10.24 42.667 358.4 358.4 0 0 0 82.773 375.893 361.387 361.387 0 0 0 376.747 82.774 42.667 42.667 0 0 1 54.187 55.04 433.493 433.493 0 0 1-99.84 154.88 438.613 438.613 0 0 1-311.467 128z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" class="icon light-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="light icon" style="display:block;"><path d="M952 552h-80a40 40 0 0 1 0-80h80a40 40 0 0 1 0 80zM801.88 280.08a41 41 0 0 1-57.96-57.96l57.96-58a41.04 41.04 0 0 1 58 58l-58 57.96zM512 752a240 240 0 1 1 0-480 240 240 0 0 1 0 480zm0-560a40 40 0 0 1-40-40V72a40 40 0 0 1 80 0v80a40 40 0 0 1-40 40zm-289.88 88.08-58-57.96a41.04 41.04 0 0 1 58-58l57.96 58a41 41 0 0 1-57.96 57.96zM192 512a40 40 0 0 1-40 40H72a40 40 0 0 1 0-80h80a40 40 0 0 1 40 40zm30.12 231.92a41 41 0 0 1 57.96 57.96l-57.96 58a41.04 41.04 0 0 1-58-58l58-57.96zM512 832a40 40 0 0 1 40 40v80a40 40 0 0 1-80 0v-80a40 40 0 0 1 40-40zm289.88-88.08 58 57.96a41.04 41.04 0 0 1-58 58l-57.96-58a41 41 0 0 1 57.96-57.96z"></path></svg></button></div><div class="vp-nav-item vp-action"><a class="vp-action-link" href="https://github.com/apache/iotdb" target="_blank" rel="noopener noreferrer" aria-label="GitHub"><svg xmlns="http://www.w3.org/2000/svg" class="icon github-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="github icon" style="width:1.25rem;height:1.25rem;vertical-align:middle;"><path d="M511.957 21.333C241.024 21.333 21.333 240.981 21.333 512c0 216.832 140.544 400.725 335.574 465.664 24.49 4.395 32.256-10.07 32.256-23.083 0-11.69.256-44.245 0-85.205-136.448 29.61-164.736-64.64-164.736-64.64-22.315-56.704-54.4-71.765-54.4-71.765-44.587-30.464 3.285-29.824 3.285-29.824 49.195 3.413 75.179 50.517 75.179 50.517 43.776 75.008 114.816 53.333 142.762 40.79 4.523-31.66 17.152-53.377 31.19-65.537-108.971-12.458-223.488-54.485-223.488-242.602 0-53.547 19.114-97.323 50.517-131.67-5.035-12.33-21.93-62.293 4.779-129.834 0 0 41.258-13.184 134.912 50.346a469.803 469.803 0 0 1 122.88-16.554c41.642.213 83.626 5.632 122.88 16.554 93.653-63.488 134.784-50.346 134.784-50.346 26.752 67.541 9.898 117.504 4.864 129.834 31.402 34.347 50.474 78.123 50.474 131.67 0 188.586-114.73 230.016-224.042 242.09 17.578 15.232 33.578 44.672 33.578 90.454v135.85c0 13.142 7.936 27.606 32.854 22.87C862.25 912.597 1002.667 728.747 1002.667 512c0-271.019-219.648-490.667-490.71-490.667z"></path></svg></a></div><!--]--><!--[--><!----><!--]--><button type="button" class="vp-toggle-navbar-button" aria-label="Toggle Navbar" aria-expanded="false" aria-controls="nav-screen"><span><span class="vp-top"></span><span class="vp-middle"></span><span class="vp-bottom"></span></span></button></div></header><!----><!--]--><!----><div class="toggle-sidebar-wrapper"><span class="arrow start"></span></div><aside id="sidebar" class="vp-sidebar"><!--[--><!----><!--]--><ul class="vp-sidebar-links"><li><section class="vp-sidebar-group"><p class="vp-sidebar-header"><!----><span class="vp-sidebar-title">IoTDB User Guide (V1.3.x)</span><!----></p><ul class="vp-sidebar-links"></ul></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">About IoTDB</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">Quick Start</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">Basic Concept</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">Deployment & Maintenance</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable active" type="button"><!----><span class="vp-sidebar-title">User Manual</span><span class="vp-arrow down"></span></button><ul class="vp-sidebar-links"><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Syntax-Rule.html" aria-label="Syntax Rule"><!---->Syntax Rule<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Operate-Metadata.html" aria-label="Operate Metadata"><!---->Operate Metadata<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Write-Delete-Data.html" aria-label="Write Delete Data"><!---->Write Delete Data<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Query-Data.html" aria-label="Query Data"><!---->Query Data<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Operator-and-Expression.html" aria-label="Operator and Expression"><!---->Operator and Expression<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Streaming.html" aria-label="Streaming"><!---->Streaming<!----></a></li><li><a class="route-link nav-link active vp-sidebar-link vp-sidebar-page active" href="/UserGuide/latest/User-Manual/Data-Sync.html" aria-label="Data Sync"><!---->Data Sync<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Database-Programming.html" aria-label="Database Programming"><!---->Database Programming<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Authority-Management.html" aria-label="Authority Management"><!---->Authority Management<!----></a></li><li><a class="route-link nav-link vp-sidebar-link vp-sidebar-page" href="/UserGuide/latest/User-Manual/Maintennance.html" aria-label="Maintennance"><!---->Maintennance<!----></a></li></ul></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">Tools System</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">API</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">Ecosystem Integration</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">SQL Manual</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">FAQ</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-header clickable" type="button"><!----><span class="vp-sidebar-title">Reference</span><span class="vp-arrow end"></span></button><!----></section></li></ul><!--[--><!----><!--]--></aside><!--[--><main id="main-content" class="vp-page"><!--[--><!--[--><!----><!--]--><!----><nav class="vp-breadcrumb disable"></nav><div class="vp-page-title"><h1><!---->IoTDB Data Sync</h1><div class="page-info"><!----><!----><span class="page-date-info" aria-label="Writing Date"><svg xmlns="http://www.w3.org/2000/svg" class="icon calendar-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="calendar icon"><path d="M716.4 110.137c0-18.753-14.72-33.473-33.472-33.473-18.753 0-33.473 14.72-33.473 33.473v33.473h66.993v-33.473zm-334.87 0c0-18.753-14.72-33.473-33.473-33.473s-33.52 14.72-33.52 33.473v33.473h66.993v-33.473zm468.81 33.52H716.4v100.465c0 18.753-14.72 33.473-33.472 33.473a33.145 33.145 0 01-33.473-33.473V143.657H381.53v100.465c0 18.753-14.72 33.473-33.473 33.473a33.145 33.145 0 01-33.473-33.473V143.657H180.6A134.314 134.314 0 0046.66 277.595v535.756A134.314 134.314 0 00180.6 947.289h669.74a134.36 134.36 0 00133.94-133.938V277.595a134.314 134.314 0 00-133.94-133.938zm33.473 267.877H147.126a33.145 33.145 0 01-33.473-33.473c0-18.752 14.72-33.473 33.473-33.473h736.687c18.752 0 33.472 14.72 33.472 33.473a33.145 33.145 0 01-33.472 33.473z"></path></svg><span><!----></span><meta property="datePublished" content="2023-10-10T10:08:46.000Z"></span><span class="page-pageview-info" aria-label="Page views"><svg xmlns="http://www.w3.org/2000/svg" class="icon eye-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="eye icon"><path d="M992 512.096c0-5.76-.992-10.592-1.28-11.136-.192-2.88-1.152-8.064-2.08-10.816-.256-.672-.544-1.376-.832-2.08-.48-1.568-1.024-3.104-1.6-4.32C897.664 290.112 707.104 160 512 160c-195.072 0-385.632 130.016-473.76 322.592-1.056 2.112-1.792 4.096-2.272 5.856a55.512 55.512 0 00-.64 1.6c-1.76 5.088-1.792 8.64-1.632 7.744-.832 3.744-1.568 11.168-1.568 11.168-.224 2.272-.224 4.032.032 6.304 0 0 .736 6.464 1.088 7.808.128 1.824.576 4.512 1.12 6.976h-.032c.448 2.08 1.12 4.096 1.984 6.08.48 1.536.992 2.976 1.472 4.032C126.432 733.856 316.992 864 512 864c195.136 0 385.696-130.048 473.216-321.696 1.376-2.496 2.24-4.832 2.848-6.912.256-.608.48-1.184.672-1.728 1.536-4.48 1.856-8.32 1.728-8.32l-.032.032c.608-3.104 1.568-7.744 1.568-13.28zM512 672c-88.224 0-160-71.776-160-160s71.776-160 160-160 160 71.776 160 160-71.776 160-160 160z"></path></svg><span id="ArtalkPV" class="vp-pageview waline-pageview-count" data-path="/UserGuide/latest/User-Manual/Data-Sync.html" data-page-key="/UserGuide/latest/User-Manual/Data-Sync.html">...</span></span><span class="page-reading-time-info" aria-label="Reading Time"><svg xmlns="http://www.w3.org/2000/svg" class="icon timer-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="timer icon"><path d="M799.387 122.15c4.402-2.978 7.38-7.897 7.38-13.463v-1.165c0-8.933-7.38-16.312-16.312-16.312H256.33c-8.933 0-16.311 7.38-16.311 16.312v1.165c0 5.825 2.977 10.874 7.637 13.592 4.143 194.44 97.22 354.963 220.201 392.763-122.204 37.542-214.893 196.511-220.2 389.397-4.661 5.049-7.638 11.651-7.638 19.03v5.825h566.49v-5.825c0-7.379-2.849-13.981-7.509-18.9-5.049-193.016-97.867-351.985-220.2-389.527 123.24-37.67 216.446-198.453 220.588-392.892zM531.16 450.445v352.632c117.674 1.553 211.787 40.778 211.787 88.676H304.097c0-48.286 95.149-87.382 213.728-88.676V450.445c-93.077-3.107-167.901-81.297-167.901-177.093 0-8.803 6.99-15.793 15.793-15.793 8.803 0 15.794 6.99 15.794 15.793 0 80.261 63.69 145.635 142.01 145.635s142.011-65.374 142.011-145.635c0-8.803 6.99-15.793 15.794-15.793s15.793 6.99 15.793 15.793c0 95.019-73.789 172.82-165.96 177.093z"></path></svg><span>About 12 min</span><meta property="timeRequired" content="PT12M"></span><!----><!----></div><hr></div><div class="vp-toc-placeholder"><aside id="toc"><!--[--><!----><!--]--><div class="vp-toc-header">On This Page<button type="button" class="print-button" title="Print"><svg xmlns="http://www.w3.org/2000/svg" class="icon print-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="print icon"><path d="M819.2 364.8h-44.8V128c0-17.067-14.933-32-32-32H281.6c-17.067 0-32 14.933-32 32v236.8h-44.8C145.067 364.8 96 413.867 96 473.6v192c0 59.733 49.067 108.8 108.8 108.8h44.8V896c0 17.067 14.933 32 32 32h460.8c17.067 0 32-14.933 32-32V774.4h44.8c59.733 0 108.8-49.067 108.8-108.8v-192c0-59.733-49.067-108.8-108.8-108.8zM313.6 160h396.8v204.8H313.6V160zm396.8 704H313.6V620.8h396.8V864zM864 665.6c0 25.6-19.2 44.8-44.8 44.8h-44.8V588.8c0-17.067-14.933-32-32-32H281.6c-17.067 0-32 14.933-32 32v121.6h-44.8c-25.6 0-44.8-19.2-44.8-44.8v-192c0-25.6 19.2-44.8 44.8-44.8h614.4c25.6 0 44.8 19.2 44.8 44.8v192z"></path></svg></button><div class="arrow end"></div></div><div class="vp-toc-wrapper"><ul class="vp-toc-list"><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level2" href="#quick-start">Quick Start</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level2" href="#sync-task-management">Sync Task Management</a></li><li><ul class="vp-toc-list"><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#create-a-sync-task">Create a sync task</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#start-task">START TASK</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#stop-task">STOP TASK</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#delete-task">DELETE TASK</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#show-task">SHOW TASK</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#task-running-status-migration">Task Running Status Migration</a></li><!----><!--]--></ul></li><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level2" href="#system-pre-built-data-sync-plugin">System Pre-built Data Sync Plugin</a></li><li><ul class="vp-toc-list"><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#view-pre-built-plugin">View pre-built plugin</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#pre-built-source-plugin">Pre-built Source Plugin</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#pre-built-processor-plugin">Pre-built Processor Plugin</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#pre-built-sink-plugin">Pre-built Sink plugin</a></li><!----><!--]--></ul></li><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level2" href="#authority-management">Authority Management</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level2" href="#configure-parameters">Configure Parameters</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level2" href="#functionality-features">Functionality Features</a></li><li><ul class="vp-toc-list"><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#at-least-one-semantic-guarantee-at-least-once">At least one semantic guarantee at-least-once</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#source-data-writing-with-pipe-processing-and-asynchronous-decoupling-of-data-transmission">Source: Data Writing with Pipe Processing and Asynchronous Decoupling of Data Transmission</a></li><!----><!--]--><!--[--><li class="vp-toc-item"><a class="route-link vp-toc-link level3" href="#source-high-availability-of-pipe-service-in-a-highly-available-cluster-deployment">Source: High Availability of Pipe Service in a Highly Available Cluster Deployment</a></li><!----><!--]--></ul></li><!--]--></ul><div class="vp-toc-marker" style="top:-1.7rem;"></div></div><!--[--><!----><!--]--></aside></div><!--[--><!----><!--]--><div class="theme-hope-content"><h1 id="iotdb-data-sync" tabindex="-1"><a class="header-anchor" href="#iotdb-data-sync"><span>IoTDB Data Sync</span></a></h1><p><strong>The IoTDB data sync transfers data from IoTDB to another data platform, and <!---->.</strong></p><p><strong>A Pipe consists of three subtasks (plugins):</strong></p><ul><li>Source</li><li>Process</li><li>Sink</li></ul><p><strong>Pipe allows users to customize the processing logic of these three subtasks, just like handling data using UDF (User-Defined Functions)</strong>. Within a Pipe, the aforementioned subtasks are executed and implemented by three types of plugins. Data flows through these three plugins sequentially: Pipe Source is used to extract data, Pipe Processor is used to process data, and Pipe Sink is used to send data to an external system.</p><p><strong>The model of a Pipe task is as follows:</strong></p><figure><img src="https://alioss.timecho.com/docs/img/1706698537700.jpg" alt="Task model diagram" tabindex="0" loading="lazy"><figcaption>Task model diagram</figcaption></figure><p>It describes a data sync task, which essentially describes the attributes of the Pipe Source, Pipe Processor, and Pipe Sink plugins. Users can declaratively configure the specific attributes of the three subtasks through SQL statements. By combining different attributes, flexible data ETL (Extract, Transform, Load) capabilities can be achieved.</p><p>By utilizing the data sync functionality, a complete data pipeline can be built to fulfill various requirements such as edge-to-cloud sync, remote disaster recovery, and read-write workload distribution across multiple databases.</p><h2 id="quick-start" tabindex="-1"><a class="header-anchor" href="#quick-start"><span>Quick Start</span></a></h2><p><strong>🎯 Goal: Achieve full data sync of IoTDB A -> IoTDB B</strong></p><ul><li><p>Start two IoTDBs,A（datanode -> 127.0.0.1:6667） B（datanode -> 127.0.0.1:6668）</p></li><li><p>create a Pipe from A -> B, and execute on A</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">create</span> pipe a2b |
| <span class="token keyword">with</span> sink <span class="token punctuation">(</span> |
| <span class="token string">'sink'</span><span class="token operator">=</span><span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.ip'</span><span class="token operator">=</span><span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.port'</span><span class="token operator">=</span><span class="token string">'6668'</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div></li><li><p>start a Pipe from A -> B, and execute on A</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">start</span> pipe a2b |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div></li><li><p>Write data to A</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">INSERT</span> <span class="token keyword">INTO</span> root<span class="token punctuation">.</span>db<span class="token punctuation">.</span>d<span class="token punctuation">(</span><span class="token keyword">time</span><span class="token punctuation">,</span> m<span class="token punctuation">)</span> <span class="token keyword">values</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div></li><li><p>Checking data synchronised from A at B</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SELECT</span> <span class="token operator">*</span><span class="token operator">*</span> <span class="token keyword">FROM</span> root |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div></li></ul><blockquote><p>❗️<strong>Note: The current IoTDB -> IoTDB implementation of data sync does not support DDL sync</strong></p><p>That is: ttl, trigger, alias, template, view, create/delete sequence, create/delete database, etc. are not supported.</p><p><strong>IoTDB -> IoTDB data sync requires the target IoTDB:</strong></p><ul><li>Enable automatic metadata creation: manual configuration of encoding and compression of data types to be consistent with the sender is required</li><li>Do not enable automatic metadata creation: manually create metadata that is consistent with the source</li></ul></blockquote><h2 id="sync-task-management" tabindex="-1"><a class="header-anchor" href="#sync-task-management"><span>Sync Task Management</span></a></h2><h3 id="create-a-sync-task" tabindex="-1"><a class="header-anchor" href="#create-a-sync-task"><span>Create a sync task</span></a></h3><p>A data sync task can be created using the <code>CREATE PIPE</code> statement, a sample SQL statement is shown below:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> <span class="token comment">-- PipeId is the name that uniquely identifies the sync task</span> |
| <span class="token keyword">WITH</span> SOURCE <span class="token punctuation">(</span> |
| <span class="token comment">-- Default IoTDB Data Extraction Plugin</span> |
| <span class="token string">'source'</span> <span class="token operator">=</span> <span class="token string">'iotdb-source'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Path prefix, only data that can match the path prefix will be extracted for subsequent processing and delivery</span> |
| <span class="token string">'source.pattern'</span> <span class="token operator">=</span> <span class="token string">'root.timecho'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Describes the time range of the data being extracted, indicating the earliest possible time</span> |
| <span class="token string">'source.historical.start-time'</span> <span class="token operator">=</span> <span class="token string">'2011.12.03T10:15:30+01:00'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Describes the time range of the extracted data, indicating the latest time</span> |
| <span class="token string">'source.historical.end-time'</span> <span class="token operator">=</span> <span class="token string">'2022.12.03T10:15:30+01:00'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| <span class="token keyword">WITH</span> PROCESSOR <span class="token punctuation">(</span> |
| <span class="token comment">-- Default data processing plugin, means no processing</span> |
| <span class="token string">'processor'</span> <span class="token operator">=</span> <span class="token string">'do-nothing-processor'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token comment">-- IoTDB data sending plugin with target IoTDB</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Data service for one of the DataNode nodes on the target IoTDB ip</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Data service port of one of the DataNode nodes of the target IoTDB</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'6667'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p><strong>To create a sync task it is necessary to configure the PipeId and the parameters of the three plugin sections:</strong></p><table><thead><tr><th>configuration item</th><th>description</th><th>Required or not</th><th>default implementation</th><th>Default implementation description</th><th>Whether to allow custom implementations</th></tr></thead><tbody><tr><td>pipeId</td><td>Globally uniquely identifies the name of a sync task</td><td><!----></td><td>-</td><td>-</td><td>-</td></tr><tr><td>source</td><td>pipe Source plugin, for extracting synchronized data at the bottom of the database</td><td>Optional</td><td>iotdb-source</td><td>Integrate all historical data of the database and subsequent realtime data into the sync task</td><td>no</td></tr><tr><td>processor</td><td>Pipe Processor plugin, for processing data</td><td>Optional</td><td>do-nothing-processor</td><td>no processing of incoming data</td><td><!----></td></tr><tr><td>sink</td><td>Pipe Sink plugin，for sending data</td><td><!----></td><td>-</td><td>-</td><td><!----></td></tr></tbody></table><p>In the example, the iotdb-source, do-nothing-processor, and iotdb-thrift-sink plugins are used to build the data sync task. IoTDB has other built-in data sync plugins, <strong>see the section "System Pre-built Data Sync Plugin"</strong>.<br><strong>An example of a minimalist CREATE PIPE statement is as follows:</strong></p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> <span class="token comment">-- PipeId is a name that uniquely identifies the task.</span> |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token comment">-- IoTDB data sending plugin with target IoTDB</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Data service for one of the DataNode nodes on the target IoTDB ip</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Data service port of one of the DataNode nodes of the target IoTDB</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'6667'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>The expressed semantics are: synchronize the full amount of historical data and subsequent arrivals of realtime data from this database instance to the IoTDB instance with target 127.0.0.1:6667.</p><p><strong>Note:</strong></p><ul><li><p>SOURCE and PROCESSOR are optional, if no configuration parameters are filled in, the system will use the corresponding default implementation.</p></li><li><p>The SINK is a mandatory configuration that needs to be declared in the CREATE PIPE statement for configuring purposes.</p></li><li><p>The SINK exhibits self-reusability. For different tasks, if their SINK possesses identical KV properties (where the value corresponds to every key), <strong>the system will ultimately create only one instance of the SINK</strong> to achieve resource reuse for connections.</p><ul><li>For example, there are the following pipe1, pipe2 task declarations:</li></ul><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE pipe1 |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'localhost'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'9999'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| |
| <span class="token keyword">CREATE</span> PIPE pipe2 |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'9999'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'localhost'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><ul><li>Since they have identical SINK declarations (<strong>even if the order of some properties is different</strong>), the framework will automatically reuse the SINK declared by them. Hence, the SINK instances for pipe1 and pipe2 will be the same.</li></ul></li><li><p>Please note that we should avoid constructing application scenarios that involve data cycle sync (as it can result in an infinite loop):</p><ul><li>IoTDB A -> IoTDB B -> IoTDB A</li><li>IoTDB A -> IoTDB A</li></ul></li></ul><h3 id="start-task" tabindex="-1"><a class="header-anchor" href="#start-task"><span>START TASK</span></a></h3><p>After the successful execution of the CREATE PIPE statement, task-related instances will be created. However, the overall task's running status will be set to STOPPED(V1.3.0), meaning the task will not immediately process data. In version 1.3.1 and later, the status of the task will be set to RUNNING after CREATE.</p><p>You can use the START PIPE statement to begin processing data for a task:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">START</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="stop-task" tabindex="-1"><a class="header-anchor" href="#stop-task"><span>STOP TASK</span></a></h3><p>the STOP PIPE statement can be used to halt the data processing:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code>STOP PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="delete-task" tabindex="-1"><a class="header-anchor" href="#delete-task"><span>DELETE TASK</span></a></h3><p>If a task is in the RUNNING state, you can use the DROP PIPE statement to stop the data processing and delete the entire task:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">DROP</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>Before deleting a task, there is no need to execute the STOP operation.</p><h3 id="show-task" tabindex="-1"><a class="header-anchor" href="#show-task"><span>SHOW TASK</span></a></h3><p>You can use the SHOW PIPES statement to view all tasks:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPES |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>The query results are as follows:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| <span class="token operator">|</span> ID<span class="token operator">|</span> CreationTime <span class="token operator">|</span> State<span class="token operator">|</span>PipeSource<span class="token operator">|</span>PipeProcessor<span class="token operator">|</span>PipeSink<span class="token operator">|</span>ExceptionMessage<span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| <span class="token operator">|</span>iotdb<span class="token operator">-</span>kafka<span class="token operator">|</span><span class="token number">2022</span><span class="token operator">-</span><span class="token number">03</span><span class="token operator">-</span><span class="token number">30</span>T20:<span class="token number">58</span>:<span class="token number">30.689</span><span class="token operator">|</span>RUNNING<span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> {}<span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| <span class="token operator">|</span>iotdb<span class="token operator">-</span>iotdb<span class="token operator">|</span><span class="token number">2022</span><span class="token operator">-</span><span class="token number">03</span><span class="token operator">-</span><span class="token number">31</span>T12:<span class="token number">55</span>:<span class="token number">28.129</span><span class="token operator">|</span>STOPPED<span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> TException: <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>You can use <PipeId> to specify the status of a particular synchronization task:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>Additionally, the WHERE clause can be used to determine if the Pipe Sink used by a specific <PipeId> is being reused.</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPES |
| <span class="token keyword">WHERE</span> SINK USED <span class="token keyword">BY</span> <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="task-running-status-migration" tabindex="-1"><a class="header-anchor" href="#task-running-status-migration"><span>Task Running Status Migration</span></a></h3><p>The task running status can transition through several states during the lifecycle of a data synchronization pipe:</p><ul><li><strong>STOPPED：</strong> The pipe is in a stopped state. It has the following causes: <ul><li>After the successful creation of a pipe, its initial state is set to STOPPED (V1.3.0)</li><li>The user manually pauses a pipe that is in normal running state, transitioning its status from RUNNING to STOPPED</li><li>If a pipe encounters an unrecoverable error during execution, its status automatically changes from RUNNING to STOPPED.</li></ul></li><li><strong>RUNNING：</strong> The pipe is actively processing data. It has the following cause: <ul><li>After the successful creation of a pipe, its initial state is set to RUNNING (V1.3.1+)</li></ul></li><li><strong>DROPPED：</strong> The pipe is permanently deleted</li></ul><p>The following diagram illustrates the different states and their transitions:</p><figure><img src="https://alioss.timecho.com/docs/img/状态迁移图.png" alt="state migration diagram" tabindex="0" loading="lazy"><figcaption>state migration diagram</figcaption></figure><h2 id="system-pre-built-data-sync-plugin" tabindex="-1"><a class="header-anchor" href="#system-pre-built-data-sync-plugin"><span>System Pre-built Data Sync Plugin</span></a></h2><p>📌 Notes: for version 1.3.1 or later, any parameters other than "source", "processor", "sink" themselves need not be with the prefixes. For instance:</p><div class="language-Sql line-numbers-mode" data-ext="Sql" data-title="Sql"><pre class="language-Sql"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-thrift-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6668' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>can be written as</p><div class="language-Sql line-numbers-mode" data-ext="Sql" data-title="Sql"><pre class="language-Sql"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-air-gap-sink', |
| 'ip'='127.0.0.1', |
| 'port'='6668' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="view-pre-built-plugin" tabindex="-1"><a class="header-anchor" href="#view-pre-built-plugin"><span>View pre-built plugin</span></a></h3><p>User can view the plugins in the system on demand. The statement for viewing plugins is shown below.</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPEPLUGINS |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="pre-built-source-plugin" tabindex="-1"><a class="header-anchor" href="#pre-built-source-plugin"><span>Pre-built Source Plugin</span></a></h3><h4 id="iotdb-source" tabindex="-1"><a class="header-anchor" href="#iotdb-source"><span>iotdb-source</span></a></h4><p>Function: Extract historical or realtime data inside IoTDB into pipe.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>source</td><td>iotdb-source</td><td>String: iotdb-source</td><td>required</td></tr><tr><td>source.pattern</td><td>path prefix for filtering time series</td><td>String: any time series prefix</td><td>optional: root</td></tr><tr><td>source.history.start-time</td><td>start of synchronizing historical data event time，including start-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MIN_VALUE</td></tr><tr><td>source.history.end-time</td><td>end of synchronizing historical data event time，including end-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MAX_VALUE</td></tr><tr><td>start-time(V1.3.1+)</td><td>start of synchronizing all data event time，including start-time. Will disable "history.start-time" "history.end-time" if configured</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MIN_VALUE</td></tr><tr><td>end-time(V1.3.1+)</td><td>end of synchronizing all data event time，including end-time. Will disable "history.start-time" "history.end-time" if configured</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MAX_VALUE</td></tr></tbody></table><blockquote><p>🚫 <strong>source.pattern Parameter Description</strong></p><ul><li><p>Pattern should use backquotes to modify illegal characters or illegal path nodes, for example, if you want to filter root.`a@b` or root.`123`, you should set the pattern to root.`a@b` or root.`123`（Refer specifically to <a href="https://iotdb.apache.org/zh/Download/#_1-0-%E7%89%88%E6%9C%AC%E4%B8%8D%E5%85%BC%E5%AE%B9%E7%9A%84%E8%AF%AD%E6%B3%95%E8%AF%A6%E7%BB%86%E8%AF%B4%E6%98%8E" target="_blank" rel="noopener noreferrer">Timing of single and double quotes and backquotes<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span></a>）</p></li><li><p>In the underlying implementation, when pattern is detected as root (default value) or a database name, synchronization efficiency is higher, and any other format will reduce performance.</p></li><li><p>The path prefix does not need to form a complete path. For example, when creating a pipe with the parameter 'source.pattern'='root.aligned.1':</p><ul><li>root.aligned.1TS</li><li>root.aligned.1TS.`1`</li><li>root.aligned.100TS</li></ul><p>the data will be synchronized;</p><ul><li>root.aligned.`1`</li><li>root.aligned.`123`</li></ul><p>the data will not be synchronized.</p></li></ul></blockquote><blockquote><p>❗️<strong>start-time, end-time parameter description of source</strong></p><ul><li>start-time, end-time should be in ISO format, such as 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00. Version 1.3.1+ supports timeStamp format like 1706704494000.</li></ul></blockquote><blockquote><p>✅ <strong>a piece of data from production to IoTDB contains two key concepts of time</strong></p><ul><li><strong>event time：</strong> the time when the data is actually produced (or the generation time assigned to the data by the data production system, which is a time item in the data point), also called the event time.</li><li><strong>arrival time：</strong> the time the data arrived in the IoTDB system.</li></ul><p>The out-of-order data we often refer to refers to data whose <strong>event time</strong> is far behind the current system time (or the maximum <strong>event time</strong> that has been dropped) when the data arrives. On the other hand, whether it is out-of-order data or sequential data, as long as they arrive newly in the system, their <strong>arrival time</strong> will increase with the order in which the data arrives at IoTDB.</p></blockquote><blockquote><p>💎 <strong>the work of iotdb-source can be split into two stages</strong></p><ol><li>Historical data extraction: All data with <strong>arrival time</strong> < <strong>current system time</strong> when creating the pipe is called historical data</li><li>Realtime data extraction: All data with <strong>arrival time</strong> >= <strong>current system time</strong> when the pipe is created is called realtime data</li></ol><p>The historical data transmission phase and the realtime data transmission phase are executed serially. Only when the historical data transmission phase is completed, the realtime data transmission phase is executed.**</p></blockquote><h3 id="pre-built-processor-plugin" tabindex="-1"><a class="header-anchor" href="#pre-built-processor-plugin"><span>Pre-built Processor Plugin</span></a></h3><h4 id="do-nothing-processor" tabindex="-1"><a class="header-anchor" href="#do-nothing-processor"><span>do-nothing-processor</span></a></h4><p>Function: Do nothing with the events passed in by the source.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>processor</td><td>do-nothing-processor</td><td>String: do-nothing-processor</td><td>required</td></tr></tbody></table><h3 id="pre-built-sink-plugin" tabindex="-1"><a class="header-anchor" href="#pre-built-sink-plugin"><span>Pre-built Sink plugin</span></a></h3><h4 id="iotdb-thrift-sync-sink" tabindex="-1"><a class="header-anchor" href="#iotdb-thrift-sync-sink"><span>iotdb-thrift-sync-sink</span></a></h4><p>Function: Primarily used for data transfer between IoTDB instances (v1.2.0+). Data is transmitted using the Thrift RPC framework and a single-threaded blocking IO model. It guarantees that the receiving end applies the data in the same order as the sending end receives the write requests.</p><p>Limitation: Both the source and target IoTDB versions need to be v1.2.0+.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>sink</td><td>iotdb-thrift-sync-sink</td><td>String: iotdb-thrift-sync-sink</td><td>required</td></tr><tr><td>sink.ip</td><td>the data service IP of one of the DataNode nodes in the target IoTDB</td><td>String</td><td>optional: and sink.node-urls fill in either one</td></tr><tr><td>sink.port</td><td>the data service port of one of the DataNode nodes in the target IoTDB</td><td>Integer</td><td>optional: and sink.node-urls fill in either one</td></tr><tr><td>sink.node-urls</td><td>the URL of the data service port of any multiple DataNode nodes in the target IoTDB</td><td>String。eg：'127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>optional: and sink.ip:sink.port fill in either one</td></tr></tbody></table><blockquote><p>📌 Please ensure that the receiving end has already created all the time series present in the sending end or has enabled automatic metadata creation. Otherwise, it may result in the failure of the pipe operation.</p></blockquote><h4 id="iotdb-thrift-async-sink-alias-iotdb-thrift-sink" tabindex="-1"><a class="header-anchor" href="#iotdb-thrift-async-sink-alias-iotdb-thrift-sink"><span>iotdb-thrift-async-sink(alias:iotdb-thrift-sink)</span></a></h4><p>Function: Primarily used for data transfer between IoTDB instances (v1.2.0+).<br> Data is transmitted using the Thrift RPC framework, employing a multi-threaded async non-blocking IO model, resulting in high transfer performance. It is particularly suitable for distributed scenarios on the target end.<br> It does not guarantee that the receiving end applies the data in the same order as the sending end receives the write requests, but it guarantees data integrity (at-least-once).</p><p>Limitation: Both the source and target IoTDB versions need to be v1.2.0+.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>sink</td><td>iotdb-thrift-sink or iotdb-thrift-async-sink</td><td>String: iotdb-thrift-sink or iotdb-thrift-async-sink</td><td>required</td></tr><tr><td>sink.ip</td><td>the data service IP of one of the DataNode nodes in the target IoTDB</td><td>String</td><td>optional: and sink.node-urls fill in either one</td></tr><tr><td>sink.port</td><td>the data service port of one of the DataNode nodes in the target IoTDB</td><td>Integer</td><td>optional: and sink.node-urls fill in either one</td></tr><tr><td>sink.node-urls</td><td>the URL of the data service port of any multiple DataNode nodes in the target IoTDB</td><td>String. eg：'127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>optional: and sink.ip:sink.port fill in either one</td></tr></tbody></table><blockquote><p>📌 Please ensure that the receiving end has already created all the time series present in the sending end or has enabled automatic metadata creation. Otherwise, it may result in the failure of the pipe operation.</p></blockquote><h4 id="iotdb-legacy-pipe-sink" tabindex="-1"><a class="header-anchor" href="#iotdb-legacy-pipe-sink"><span>iotdb-legacy-pipe-sink</span></a></h4><p>Function: Mainly used to transfer data from IoTDB (v1.2.0+) to versions lower than v1.2.0 of IoTDB, using the data synchronization (Sync) protocol before version v1.2.0.<br> Data is transmitted using the Thrift RPC framework. It employs a single-threaded sync blocking IO model, resulting in weak transfer performance.</p><p>Limitation: The source IoTDB version needs to be v1.2.0+. The target IoTDB version can be either v1.2.0+, v1.1.x (lower versions of IoTDB are theoretically supported but untested).</p><p>Note: In theory, any version prior to v1.2.0 of IoTDB can serve as the data synchronization (Sync) receiver for v1.2.0+.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>sink</td><td>iotdb-legacy-pipe-sink</td><td>string: iotdb-legacy-pipe-sink</td><td>required</td></tr><tr><td>sink.ip</td><td>data service of one DataNode node of the target IoTDB ip</td><td>string</td><td>required</td></tr><tr><td>sink.port</td><td>the data service port of one of the DataNode nodes in the target IoTDB</td><td>integer</td><td>required</td></tr><tr><td>sink.user</td><td>the user name of the target IoTDB. Note that the user needs to support data writing and TsFile Load permissions.</td><td>string</td><td>optional: root</td></tr><tr><td>sink.password</td><td>the password of the target IoTDB. Note that the user needs to support data writing and TsFile Load permissions.</td><td>string</td><td>optional: root</td></tr><tr><td>sink.version</td><td>the version of the target IoTDB, used to disguise its actual version and bypass the version consistency check of the target.</td><td>string</td><td>optional: 1.1</td></tr></tbody></table><blockquote><p>📌 Make sure that the receiver has created all the time series on the sender side, or that automatic metadata creation is turned on, otherwise the pipe run will fail.</p></blockquote><h4 id="do-nothing-sink" tabindex="-1"><a class="header-anchor" href="#do-nothing-sink"><span>do-nothing-sink</span></a></h4><p>Function: Does nothing with the events passed in by the processor.</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>sink</td><td>do-nothing-sink</td><td>String: do-nothing-sink</td><td>required</td></tr></tbody></table><h2 id="authority-management" tabindex="-1"><a class="header-anchor" href="#authority-management"><span>Authority Management</span></a></h2><table><thead><tr><th>Authority Name</th><th>Description</th></tr></thead><tbody><tr><td>USE_PIPE</td><td>Register task,path-independent</td></tr><tr><td>USE_PIPE</td><td>Start task,path-independent</td></tr><tr><td>USE_PIPE</td><td>Stop task,path-independent</td></tr><tr><td>USE_PIPE</td><td>Uninstall task,path-independent</td></tr><tr><td>USE_PIPE</td><td>Query task,path-independent</td></tr></tbody></table><h2 id="configure-parameters" tabindex="-1"><a class="header-anchor" href="#configure-parameters"><span>Configure Parameters</span></a></h2><p>In iotdb-common.properties ：</p><p>V1.3.0:</p><div class="language-Properties line-numbers-mode" data-ext="Properties" data-title="Properties"><pre class="language-Properties"><code>#################### |
| ### Pipe Configuration |
| #################### |
| |
| # Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_connector_timeout_ms=900000 |
| |
| # The maximum number of selectors that can be used in the async connector. |
| # pipe_async_connector_selector_number=1 |
| |
| # The core number of clients that can be used in the async connector. |
| # pipe_async_connector_core_client_number=8 |
| |
| # The maximum number of clients that can be used in the async connector. |
| # pipe_async_connector_max_client_number=16 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>V1.3.1+:</p><div class="language-Properties line-numbers-mode" data-ext="Properties" data-title="Properties"><pre class="language-Properties"><code>#################### |
| ### Pipe Configuration |
| #################### |
| |
| # Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_sink_timeout_ms=900000 |
| |
| # The maximum number of selectors that can be used in the sink. |
| # Recommend to set this value to less than or equal to pipe_sink_max_client_number. |
| # pipe_sink_selector_number=4 |
| |
| # The maximum number of clients that can be used in the sink. |
| # pipe_sink_max_client_number=16 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="functionality-features" tabindex="-1"><a class="header-anchor" href="#functionality-features"><span>Functionality Features</span></a></h2><h3 id="at-least-one-semantic-guarantee-at-least-once" tabindex="-1"><a class="header-anchor" href="#at-least-one-semantic-guarantee-at-least-once"><span>At least one semantic guarantee <strong>at-least-once</strong></span></a></h3><p>The data synchronization feature provides an at-least-once delivery semantic when transferring data to external systems. In most scenarios, the synchronization feature guarantees exactly-once delivery, ensuring that all data is synchronized exactly once.</p><p>However, in the following scenarios, it is possible for some data to be synchronized multiple times <strong>(due to resumable transmission)</strong>:</p><ul><li>Temporary network failures: If a data transmission request fails, the system will retry sending it until reaching the maximum retry attempts.</li><li>Abnormal implementation of the Pipe plugin logic: If an error is thrown during the plugin's execution, the system will retry sending the data until reaching the maximum retry attempts.</li><li>Data partition switching due to node failures or restarts: After the partition change is completed, the affected data will be retransmitted.</li><li>Cluster unavailability: Once the cluster becomes available again, the affected data will be retransmitted.</li></ul><h3 id="source-data-writing-with-pipe-processing-and-asynchronous-decoupling-of-data-transmission" tabindex="-1"><a class="header-anchor" href="#source-data-writing-with-pipe-processing-and-asynchronous-decoupling-of-data-transmission"><span>Source: Data Writing with Pipe Processing and Asynchronous Decoupling of Data Transmission</span></a></h3><p>In the data synchronization feature, data transfer adopts an asynchronous replication mode.</p><p>Data synchronization is completely decoupled from the writing operation, eliminating any impact on the critical path of writing. This mechanism allows the framework to maintain the writing speed of a time-series database while ensuring continuous data synchronization.</p><h3 id="source-high-availability-of-pipe-service-in-a-highly-available-cluster-deployment" tabindex="-1"><a class="header-anchor" href="#source-high-availability-of-pipe-service-in-a-highly-available-cluster-deployment"><span>Source: High Availability of Pipe Service in a Highly Available Cluster Deployment</span></a></h3><p>When the sender end IoTDB is deployed in a high availability cluster mode, the data synchronization service will also be highly available. The data synchronization framework monitors the data synchronization progress of each data node and periodically takes lightweight distributed consistent snapshots to preserve the synchronization state.</p><ul><li>In the event of a failure of a data node in the sender cluster, the data synchronization framework can leverage the consistent snapshot and the data stored in replicas to quickly recover and resume synchronization, thus achieving high availability of the data synchronization service.</li><li>In the event of a complete failure and restart of the sender cluster, the data synchronization framework can also use snapshots to recover the synchronization service.</li></ul></div><!--[--><!----><!--]--><footer class="vp-page-meta"><div class="vp-meta-item edit-link"><a href="https://github.com/apache/iotdb-docs/edit/main/src/UserGuide/latest/User-Manual/Data-Sync.md" rel="noopener noreferrer" target="_blank" aria-label="Found Error? Edit this page on GitHub" class="nav-link vp-meta-label"><!--[--><svg xmlns="http://www.w3.org/2000/svg" class="icon edit-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="edit icon"><path d="M430.818 653.65a60.46 60.46 0 0 1-50.96-93.281l71.69-114.012 7.773-10.365L816.038 80.138A60.46 60.46 0 0 1 859.225 62a60.46 60.46 0 0 1 43.186 18.138l43.186 43.186a60.46 60.46 0 0 1 0 86.373L588.879 565.55l-8.637 8.637-117.466 68.234a60.46 60.46 0 0 1-31.958 11.229z"></path><path d="M728.802 962H252.891A190.883 190.883 0 0 1 62.008 771.98V296.934a190.883 190.883 0 0 1 190.883-192.61h267.754a60.46 60.46 0 0 1 0 120.92H252.891a69.962 69.962 0 0 0-69.098 69.099V771.98a69.962 69.962 0 0 0 69.098 69.098h475.911A69.962 69.962 0 0 0 797.9 771.98V503.363a60.46 60.46 0 1 1 120.922 0V771.98A190.883 190.883 0 0 1 728.802 962z"></path></svg><!--]-->Found Error? Edit this page on GitHub<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></div><div class="vp-meta-item git-info"><div class="update-time"><span class="vp-meta-label">Last update: </span><!----></div><!----></div></footer><nav class="vp-page-nav"><a class="route-link nav-link prev" href="/UserGuide/latest/User-Manual/Streaming.html" aria-label="Streaming"><div class="hint"><span class="arrow start"></span>Prev</div><div class="link"><!---->Streaming</div></a><a class="route-link nav-link next" href="/UserGuide/latest/User-Manual/Database-Programming.html" aria-label="Database Programming"><div class="hint">Next<span class="arrow end"></span></div><div class="link">Database Programming<!----></div></a></nav><!----><!--[--><!----><!--]--><!--]--></main><!--]--><footer style="padding-bottom:2rem;"><span id="doc-version" style="display:none;">latest</span><p style="text-align:center;color:#909399;font-size:12px;margin:0 30px;">Copyright © 2024 The Apache Software Foundation.<br> Apache and the Apache feather logo are trademarks of The Apache Software Foundation</p><p style="text-align:center;margin-top:10px;color:#909399;font-size:12px;margin:0 30px;"><strong>Have a question?</strong> Connect with us on QQ, WeChat, or Slack. <a href="https://github.com/apache/iotdb/issues/1995">Join the community</a> now.</p></footer></div><!--]--><!--]--><!--[--><!----><!--]--><!--]--></div> |
| <script type="module" src="/assets/app-DrPcRZG6.js" defer></script> |
| </body> |
| </html> |
