| <!doctype html> |
| <html lang="zh-CN" data-theme="light"> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width,initial-scale=1" /> |
| <meta name="generator" content="VuePress 2.0.0-rc.0" /> |
| <meta name="theme" content="VuePress Theme Hope 2.0.0-rc.2" /> |
| <style> |
| html { |
| background: var(--bg-color, #fff); |
| } |
| |
| html[data-theme="dark"] { |
| background: var(--bg-color, #1d1e1f); |
| } |
| |
| body { |
| background: var(--bg-color); |
| } |
| </style> |
| <script> |
| const userMode = localStorage.getItem("vuepress-theme-hope-scheme"); |
| const systemDarkMode = |
| window.matchMedia && |
| window.matchMedia("(prefers-color-scheme: dark)").matches; |
| |
| if (userMode === "dark" || (userMode !== "light" && systemDarkMode)) { |
| document.documentElement.setAttribute("data-theme", "dark"); |
| } |
| </script> |
| <meta property="og:url" content="https://iotdb.apache.org/zh/UserGuide/V1.2.x/User-Manual/Stage_Data-Sync_timecho.html"><meta property="og:site_name" content="IoTDB Website"><meta property="og:title" content="IoTDB 数据同步"><meta property="og:description" content="IoTDB 数据同步功能可以将 IoTDB 的数据传输到另一个数据平台，我们将一个数据同步任务称为 Pipe。 一个 Pipe 包含三个子任务（插件）： 抽取（Extract）; 处理（Process）; 发送（Connect）; Pipe 允许用户自定义三个子任务的处理逻辑，通过类似 UDF 的方式处理数据。 在一个 Pipe 中，上述的子任务分别由..."><meta property="og:type" content="article"><meta property="og:locale" content="zh-CN"><meta property="og:updated_time" content="2024-03-29T10:50:02.000Z"><meta property="article:modified_time" content="2024-03-29T10:50:02.000Z"><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","headline":"IoTDB 数据同步","image":[""],"dateModified":"2024-03-29T10:50:02.000Z","author":[]}</script><link rel="icon" href="/favicon.ico"><meta name="Description" content="Apache IoTDB: Time Series Database for IoT"><meta name="Keywords" content="TSDB, time series, time series database, IoTDB, IoT database, IoT data management,时序数据库, 时间序列管理, IoTDB, 物联网数据库, 实时数据库, 物联网数据管理, 物联网数据"><meta name="baidu-site-verification" content="wfKETzB3OT"><meta name="google-site-verification" content="mZWAoRY0yj_HAr-s47zHCGHzx5Ju-RVm5wDbPnwQYFo"><script type="text/javascript"> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| _paq.push(["setDoNotTrack", true]); |
| _paq.push(["disableCookies"]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '56']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script><title>IoTDB 数据同步 | IoTDB Website</title><meta name="description" content="IoTDB 数据同步功能可以将 IoTDB 的数据传输到另一个数据平台，我们将一个数据同步任务称为 Pipe。 一个 Pipe 包含三个子任务（插件）： 抽取（Extract）; 处理（Process）; 发送（Connect）; Pipe 允许用户自定义三个子任务的处理逻辑，通过类似 UDF 的方式处理数据。 在一个 Pipe 中，上述的子任务分别由..."> |
| <link rel="preload" href="/assets/style-vuIfcoxv.css" as="style"><link rel="stylesheet" href="/assets/style-vuIfcoxv.css"> |
| <link rel="modulepreload" href="/assets/app-z4N-28Th.js"><link rel="modulepreload" href="/assets/Stage_Data-Sync_timecho.html-D4-rmz3G.js"><link rel="modulepreload" href="/assets/Stage_Data-Sync_timecho.html-2iN5q4QG.js"> |
| |
| </head> |
| <body> |
| <div id="app"><!--[--><!--[--><!--[--><span tabindex="-1"></span><a href="#main-content" class="vp-skip-link sr-only">跳至主要內容</a><!--]--><!--[--><div class="theme-container has-toc"><!--[--><header id="navbar" class="vp-navbar hide-icon"><div class="vp-navbar-start"><button type="button" class="vp-toggle-sidebar-button" title="Toggle Sidebar"><span class="icon"></span></button><!--[--><!----><!--]--><!--[--><a class="vp-link vp-brand vp-brand" href="/zh/"><img class="vp-nav-logo" src="/logo.png" alt="IoTDB Website"><!----><span class="vp-site-name hide-in-pad">IoTDB Website</span></a><!--]--><!--[--><!----><!--]--></div><div class="vp-navbar-center"><!--[--><!----><!--]--><!--[--><!--]--><!--[--><!----><!--]--></div><div class="vp-navbar-end"><!--[--><!----><!--]--><!--[--><div id="docsearch-container"></div><nav class="vp-nav-links"><div class="nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="文档"><span class="title"><!---->文档</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a aria-label="v1.3.x" class="vp-link nav-link nav-link" href="/zh/UserGuide/latest/QuickStart/QuickStart.html"><!---->v1.3.x<!----></a></li><li class="dropdown-item"><a aria-label="v1.2.x" class="vp-link nav-link nav-link" href="/zh/UserGuide/V1.2.x/QuickStart/QuickStart.html"><!---->v1.2.x<!----></a></li><li class="dropdown-item"><a aria-label="v1.1.x" class="vp-link nav-link nav-link" href="/zh/UserGuide/V1.1.x/QuickStart/QuickStart.html"><!---->v1.1.x<!----></a></li><li class="dropdown-item"><a aria-label="v1.0.x" class="vp-link nav-link nav-link" href="/zh/UserGuide/V1.0.x/QuickStart/QuickStart.html"><!---->v1.0.x<!----></a></li><li class="dropdown-item"><a aria-label="v0.13.x" class="vp-link nav-link nav-link" href="/zh/UserGuide/V0.13.x/QuickStart/QuickStart.html"><!---->v0.13.x<!----></a></li></ul></button></div></div><div class="nav-item hide-in-mobile"><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=177051872" rel="noopener noreferrer" target="_blank" aria-label="系统设计" class="nav-link"><!---->系统设计<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></div><div class="nav-item hide-in-mobile"><a aria-label="下载" class="vp-link nav-link nav-link" href="/zh/Download/"><!---->下载<!----></a></div><div class="nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="社区"><span class="title"><!---->社区</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a aria-label="关于社区" class="vp-link nav-link nav-link" href="/zh/Community/About.html"><!---->关于社区<!----></a></li><li class="dropdown-item"><a aria-label="交流与反馈" class="vp-link nav-link nav-link" href="/zh/Community/Feedback.html"><!---->交流与反馈<!----></a></li><li class="dropdown-item"><a aria-label="活动与报告" class="vp-link nav-link nav-link" href="/zh/Community/Materials.html"><!---->活动与报告<!----></a></li></ul></button></div></div><div class="nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="开发"><span class="title"><!---->开发</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a aria-label="成为开发者" class="vp-link nav-link nav-link" href="/zh/Development/Community-Project-Committers.html"><!---->成为开发者<!----></a></li><li class="dropdown-item"><a aria-label="开发指南" class="vp-link nav-link nav-link" href="/zh/Development/Development-Guide.html"><!---->开发指南<!----></a></li><li class="dropdown-item"><a aria-label="Power by" class="vp-link nav-link nav-link" href="/zh/Development/Powered-By.html"><!---->Power by<!----></a></li></ul></button></div></div><div class="nav-item hide-in-mobile"><div class="dropdown-wrapper"><button type="button" class="dropdown-title" aria-label="ASF"><span class="title"><!---->ASF</span><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a href="https://www.apache.org/" rel="noopener noreferrer" target="_blank" aria-label="基金会" class="nav-link"><!---->基金会<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/licenses/" rel="noopener noreferrer" target="_blank" aria-label="许可证" class="nav-link"><!---->许可证<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/security/" rel="noopener noreferrer" target="_blank" aria-label="安全" class="nav-link"><!---->安全<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/foundation/sponsorship.html" rel="noopener noreferrer" target="_blank" aria-label="赞助" class="nav-link"><!---->赞助<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/foundation/thanks.html" rel="noopener noreferrer" target="_blank" aria-label="致谢" class="nav-link"><!---->致谢<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://www.apache.org/events/current-event" rel="noopener noreferrer" target="_blank" aria-label="活动" class="nav-link"><!---->活动<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li><li class="dropdown-item"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" rel="noopener noreferrer" target="_blank" aria-label="隐私" class="nav-link"><!---->隐私<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></li></ul></button></div></div></nav><div class="nav-item"><div class="dropdown-wrapper i18n-dropdown"><button type="button" class="dropdown-title" aria-label="选择语言"><!--[--><svg xmlns="http://www.w3.org/2000/svg" class="icon i18n-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="i18n icon" style="width:1rem;height:1rem;vertical-align:middle;"><path d="M379.392 460.8 494.08 575.488l-42.496 102.4L307.2 532.48 138.24 701.44l-71.68-72.704L234.496 460.8l-45.056-45.056c-27.136-27.136-51.2-66.56-66.56-108.544h112.64c7.68 14.336 16.896 27.136 26.112 35.84l45.568 46.08 45.056-45.056C382.976 312.32 409.6 247.808 409.6 204.8H0V102.4h256V0h102.4v102.4h256v102.4H512c0 70.144-37.888 161.28-87.04 210.944L378.88 460.8zM576 870.4 512 1024H409.6l256-614.4H768l256 614.4H921.6l-64-153.6H576zM618.496 768h196.608L716.8 532.48 618.496 768z"></path></svg><!--]--><span class="arrow"></span><ul class="nav-dropdown"><li class="dropdown-item"><a aria-label="English" class="vp-link nav-link nav-link" href="/"><!---->English<!----></a></li><li class="dropdown-item"><a aria-label="简体中文" class="vp-link nav-link active nav-link active" href="/zh/UserGuide/V1.2.x/User-Manual/Stage_Data-Sync_timecho.html"><!---->简体中文<!----></a></li></ul></button></div></div><div class="nav-item hide-in-mobile"><button type="button" id="appearance-switch"><svg xmlns="http://www.w3.org/2000/svg" class="icon auto-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="auto icon" style="display:block;"><path d="M512 992C246.92 992 32 777.08 32 512S246.92 32 512 32s480 214.92 480 480-214.92 480-480 480zm0-840c-198.78 0-360 161.22-360 360 0 198.84 161.22 360 360 360s360-161.16 360-360c0-198.78-161.22-360-360-360zm0 660V212c165.72 0 300 134.34 300 300 0 165.72-134.28 300-300 300z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" class="icon dark-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="dark icon" style="display:none;"><path d="M524.8 938.667h-4.267a439.893 439.893 0 0 1-313.173-134.4 446.293 446.293 0 0 1-11.093-597.334A432.213 432.213 0 0 1 366.933 90.027a42.667 42.667 0 0 1 45.227 9.386 42.667 42.667 0 0 1 10.24 42.667 358.4 358.4 0 0 0 82.773 375.893 361.387 361.387 0 0 0 376.747 82.774 42.667 42.667 0 0 1 54.187 55.04 433.493 433.493 0 0 1-99.84 154.88 438.613 438.613 0 0 1-311.467 128z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" class="icon light-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="light icon" style="display:none;"><path d="M952 552h-80a40 40 0 0 1 0-80h80a40 40 0 0 1 0 80zM801.88 280.08a41 41 0 0 1-57.96-57.96l57.96-58a41.04 41.04 0 0 1 58 58l-58 57.96zM512 752a240 240 0 1 1 0-480 240 240 0 0 1 0 480zm0-560a40 40 0 0 1-40-40V72a40 40 0 0 1 80 0v80a40 40 0 0 1-40 40zm-289.88 88.08-58-57.96a41.04 41.04 0 0 1 58-58l57.96 58a41 41 0 0 1-57.96 57.96zM192 512a40 40 0 0 1-40 40H72a40 40 0 0 1 0-80h80a40 40 0 0 1 40 40zm30.12 231.92a41 41 0 0 1 57.96 57.96l-57.96 58a41.04 41.04 0 0 1-58-58l58-57.96zM512 832a40 40 0 0 1 40 40v80a40 40 0 0 1-80 0v-80a40 40 0 0 1 40-40zm289.88-88.08 58 57.96a41.04 41.04 0 0 1-58 58l-57.96-58a41 41 0 0 1 57.96-57.96z"></path></svg></button></div><div class="nav-item vp-repo"><a class="vp-repo-link" href="https://github.com/apache/iotdb" target="_blank" rel="noopener noreferrer" aria-label="GitHub"><svg xmlns="http://www.w3.org/2000/svg" class="icon github-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="github icon" style="width:1.25rem;height:1.25rem;vertical-align:middle;"><path d="M511.957 21.333C241.024 21.333 21.333 240.981 21.333 512c0 216.832 140.544 400.725 335.574 465.664 24.49 4.395 32.256-10.07 32.256-23.083 0-11.69.256-44.245 0-85.205-136.448 29.61-164.736-64.64-164.736-64.64-22.315-56.704-54.4-71.765-54.4-71.765-44.587-30.464 3.285-29.824 3.285-29.824 49.195 3.413 75.179 50.517 75.179 50.517 43.776 75.008 114.816 53.333 142.762 40.79 4.523-31.66 17.152-53.377 31.19-65.537-108.971-12.458-223.488-54.485-223.488-242.602 0-53.547 19.114-97.323 50.517-131.67-5.035-12.33-21.93-62.293 4.779-129.834 0 0 41.258-13.184 134.912 50.346a469.803 469.803 0 0 1 122.88-16.554c41.642.213 83.626 5.632 122.88 16.554 93.653-63.488 134.784-50.346 134.784-50.346 26.752 67.541 9.898 117.504 4.864 129.834 31.402 34.347 50.474 78.123 50.474 131.67 0 188.586-114.73 230.016-224.042 242.09 17.578 15.232 33.578 44.672 33.578 90.454v135.85c0 13.142 7.936 27.606 32.854 22.87C862.25 912.597 1002.667 728.747 1002.667 512c0-271.019-219.648-490.667-490.71-490.667z"></path></svg></a></div><!--]--><!--[--><!----><!--]--><button type="button" class="vp-toggle-navbar-button" aria-label="Toggle Navbar" aria-expanded="false" aria-controls="nav-screen"><span><span class="vp-top"></span><span class="vp-middle"></span><span class="vp-bottom"></span></span></button></div></header><!----><!--]--><!----><div class="toggle-sidebar-wrapper"><span class="arrow start"></span></div><aside id="sidebar" class="vp-sidebar"><!--[--><!----><!--]--><ul class="vp-sidebar-links"><li><section class="vp-sidebar-group"><p class="vp-sidebar-heading"><!----><span class="vp-sidebar-title">IoTDB用户手册 (V1.2.x)</span><!----></p><ul class="vp-sidebar-links"></ul></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">关于IoTDB</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">快速上手</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">基础概念</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">部署与运维</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">使用手册</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">工具体系</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">应用编程接口</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">系统集成</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">SQL手册</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">FAQ</span><span class="vp-arrow end"></span></button><!----></section></li><li><section class="vp-sidebar-group"><button class="vp-sidebar-heading clickable" type="button"><!----><span class="vp-sidebar-title">参考</span><span class="vp-arrow end"></span></button><!----></section></li></ul><!--[--><!----><!--]--></aside><!--[--><main id="main-content" class="vp-page"><!--[--><!--[--><!----><!--]--><!----><nav class="vp-breadcrumb disable"></nav><div class="vp-page-title"><h1><!---->IoTDB 数据同步</h1><div class="page-info"><!----><!----><span class="page-date-info" aria-label="写作日期"><svg xmlns="http://www.w3.org/2000/svg" class="icon calendar-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="calendar icon"><path d="M716.4 110.137c0-18.753-14.72-33.473-33.472-33.473-18.753 0-33.473 14.72-33.473 33.473v33.473h66.993v-33.473zm-334.87 0c0-18.753-14.72-33.473-33.473-33.473s-33.52 14.72-33.52 33.473v33.473h66.993v-33.473zm468.81 33.52H716.4v100.465c0 18.753-14.72 33.473-33.472 33.473a33.145 33.145 0 01-33.473-33.473V143.657H381.53v100.465c0 18.753-14.72 33.473-33.473 33.473a33.145 33.145 0 01-33.473-33.473V143.657H180.6A134.314 134.314 0 0046.66 277.595v535.756A134.314 134.314 0 00180.6 947.289h669.74a134.36 134.36 0 00133.94-133.938V277.595a134.314 134.314 0 00-133.94-133.938zm33.473 267.877H147.126a33.145 33.145 0 01-33.473-33.473c0-18.752 14.72-33.473 33.473-33.473h736.687c18.752 0 33.472 14.72 33.472 33.473a33.145 33.145 0 01-33.472 33.473z"></path></svg><span><!----></span><meta property="datePublished" content="2024-03-29T10:50:02.000Z"></span><!----><span class="page-reading-time-info" aria-label="阅读时间"><svg xmlns="http://www.w3.org/2000/svg" class="icon timer-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="timer icon"><path d="M799.387 122.15c4.402-2.978 7.38-7.897 7.38-13.463v-1.165c0-8.933-7.38-16.312-16.312-16.312H256.33c-8.933 0-16.311 7.38-16.311 16.312v1.165c0 5.825 2.977 10.874 7.637 13.592 4.143 194.44 97.22 354.963 220.201 392.763-122.204 37.542-214.893 196.511-220.2 389.397-4.661 5.049-7.638 11.651-7.638 19.03v5.825h566.49v-5.825c0-7.379-2.849-13.981-7.509-18.9-5.049-193.016-97.867-351.985-220.2-389.527 123.24-37.67 216.446-198.453 220.588-392.892zM531.16 450.445v352.632c117.674 1.553 211.787 40.778 211.787 88.676H304.097c0-48.286 95.149-87.382 213.728-88.676V450.445c-93.077-3.107-167.901-81.297-167.901-177.093 0-8.803 6.99-15.793 15.793-15.793 8.803 0 15.794 6.99 15.794 15.793 0 80.261 63.69 145.635 142.01 145.635s142.011-65.374 142.011-145.635c0-8.803 6.99-15.793 15.794-15.793s15.793 6.99 15.793 15.793c0 95.019-73.789 172.82-165.96 177.093z"></path></svg><span>大约 19 分钟</span><meta property="timeRequired" content="PT19M"></span><!----><!----></div><hr></div><div class="toc-place-holder"><aside id="toc"><!--[--><!----><!--]--><div class="toc-header">此页内容<button type="button" class="print-button" title="打印"><svg xmlns="http://www.w3.org/2000/svg" class="icon print-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="print icon"><path d="M819.2 364.8h-44.8V128c0-17.067-14.933-32-32-32H281.6c-17.067 0-32 14.933-32 32v236.8h-44.8C145.067 364.8 96 413.867 96 473.6v192c0 59.733 49.067 108.8 108.8 108.8h44.8V896c0 17.067 14.933 32 32 32h460.8c17.067 0 32-14.933 32-32V774.4h44.8c59.733 0 108.8-49.067 108.8-108.8v-192c0-59.733-49.067-108.8-108.8-108.8zM313.6 160h396.8v204.8H313.6V160zm396.8 704H313.6V620.8h396.8V864zM864 665.6c0 25.6-19.2 44.8-44.8 44.8h-44.8V588.8c0-17.067-14.933-32-32-32H281.6c-17.067 0-32 14.933-32 32v121.6h-44.8c-25.6 0-44.8-19.2-44.8-44.8v-192c0-25.6 19.2-44.8 44.8-44.8h614.4c25.6 0 44.8 19.2 44.8 44.8v192z"></path></svg></button></div><div class="toc-wrapper"><ul class="toc-list"><!--[--><li class="toc-item"><a class="vp-link toc-link level2 toc-link level2" href="#快速开始">快速开始</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level2 toc-link level2" href="#同步任务管理">同步任务管理</a></li><li><ul class="toc-list"><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#创建同步任务">创建同步任务</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#启动任务">启动任务</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#停止任务">停止任务</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#删除任务">删除任务</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#展示任务">展示任务</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#任务运行状态迁移">任务运行状态迁移</a></li><!----><!--]--></ul></li><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level2 toc-link level2" href="#系统预置数据同步插件">系统预置数据同步插件</a></li><li><ul class="toc-list"><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#查看预置插件">查看预置插件</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#预置-extractor-插件">预置 extractor 插件</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#预置-processor-插件">预置 processor 插件</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#预置-connector-插件">预置 connector 插件</a></li><!----><!--]--></ul></li><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level2 toc-link level2" href="#权限管理">权限管理</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level2 toc-link level2" href="#配置参数">配置参数</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level2 toc-link level2" href="#功能特性">功能特性</a></li><li><ul class="toc-list"><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#最少一次语义保证-at-least-once">最少一次语义保证 at-least-once</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#源端-数据写入与-pipe-处理、发送数据异步解耦">源端：数据写入与 Pipe 处理、发送数据异步解耦</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#源端-可自适应数据写入负载的数据传输策略">源端：可自适应数据写入负载的数据传输策略</a></li><!----><!--]--><!--[--><li class="toc-item"><a class="vp-link toc-link level3 toc-link level3" href="#源端-高可用集群部署时-pipe-服务高可用">源端：高可用集群部署时，Pipe 服务高可用</a></li><!----><!--]--></ul></li><!--]--></ul><div class="toc-marker" style="top:-1.7rem;"></div></div><!--[--><!----><!--]--></aside></div><!--[--><!----><!--]--><div class="theme-hope-content"><!-- |
| |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| |
| --><h1 id="iotdb-数据同步" tabindex="-1"><a class="header-anchor" href="#iotdb-数据同步" aria-hidden="true">#</a> IoTDB 数据同步</h1><p><strong>IoTDB 数据同步功能可以将 IoTDB 的数据传输到另一个数据平台，我们将<!---->。</strong></p><p><strong>一个 Pipe 包含三个子任务（插件）：</strong></p><ul><li>抽取（Extract）</li><li>处理（Process）</li><li>发送（Connect）</li></ul><p><strong>Pipe 允许用户自定义三个子任务的处理逻辑，通过类似 UDF 的方式处理数据。</strong> 在一个 Pipe 中，上述的子任务分别由三种插件执行实现，数据会依次经过这三个插件进行处理：Pipe Extractor 用于抽取数据，Pipe Processor 用于处理数据，Pipe Connector 用于发送数据，最终数据将被发至外部系统。</p><p><strong>Pipe 任务的模型如下：</strong></p><figure><img src="https://alioss.timecho.com/docs/img/流处理引擎.jpeg" alt="任务模型图" tabindex="0" loading="lazy"><figcaption>任务模型图</figcaption></figure><p>描述一个数据同步任务，本质就是描述 Pipe Extractor、Pipe Processor 和 Pipe Connector 插件的属性。用户可以通过 SQL 语句声明式地配置三个子任务的具体属性，通过组合不同的属性，实现灵活的数据 ETL 能力。</p><p>利用数据同步功能，可以搭建完整的数据链路来满足端<em>边云同步、异地灾备、读写负载分库</em>等需求。</p><h2 id="快速开始" tabindex="-1"><a class="header-anchor" href="#快速开始" aria-hidden="true">#</a> 快速开始</h2><p><strong>🎯 目标：实现 IoTDB A -> IoTDB B 的全量数据同步</strong></p><ul><li><p>启动两个 IoTDB，A（datanode -> 127.0.0.1:6667） B（datanode -> 127.0.0.1:6668）</p></li><li><p>创建 A -> B 的 Pipe，在 A 上执行</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">create</span> pipe a2b |
| <span class="token keyword">with</span> connector <span class="token punctuation">(</span> |
| <span class="token string">'connector'</span><span class="token operator">=</span><span class="token string">'iotdb-thrift-connector'</span><span class="token punctuation">,</span> |
| <span class="token string">'connector.ip'</span><span class="token operator">=</span><span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token string">'connector.port'</span><span class="token operator">=</span><span class="token string">'6668'</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div></li><li><p>启动 A -> B 的 Pipe，在 A 上执行</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">start</span> pipe a2b |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div></li><li><p>向 A 写入数据</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">INSERT</span> <span class="token keyword">INTO</span> root<span class="token punctuation">.</span>db<span class="token punctuation">.</span>d<span class="token punctuation">(</span><span class="token keyword">time</span><span class="token punctuation">,</span> m<span class="token punctuation">)</span> <span class="token keyword">values</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div></li><li><p>在 B 检查由 A 同步过来的数据</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">SELECT</span> <span class="token operator">*</span><span class="token operator">*</span> <span class="token keyword">FROM</span> root |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div></li></ul><blockquote><p>❗️<strong>注：目前的 IoTDB -> IoTDB 的数据同步实现并不支持 DDL 同步</strong></p><p>即：不支持 ttl，trigger，别名，模板，视图，创建/删除序列，创建/删除存储组等操作</p><p><strong>IoTDB -> IoTDB 的数据同步要求目标端 IoTDB：</strong></p><ul><li>开启自动创建元数据：需要人工配置数据类型的编码和压缩与发送端保持一致</li><li>不开启自动创建元数据：手工创建与源端一致的元数据</li></ul></blockquote><h2 id="同步任务管理" tabindex="-1"><a class="header-anchor" href="#同步任务管理" aria-hidden="true">#</a> 同步任务管理</h2><h3 id="创建同步任务" tabindex="-1"><a class="header-anchor" href="#创建同步任务" aria-hidden="true">#</a> 创建同步任务</h3><p>可以使用 <code>CREATE PIPE</code> 语句来创建一条数据同步任务，示例 SQL 语句如下所示：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> <span class="token comment">-- PipeId 是能够唯一标定同步任务任务的名字</span> |
| <span class="token keyword">WITH</span> EXTRACTOR <span class="token punctuation">(</span> |
| <span class="token comment">-- 默认的 IoTDB 数据抽取插件</span> |
| <span class="token string">'extractor'</span> <span class="token operator">=</span> <span class="token string">'iotdb-extractor'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 路径前缀，只有能够匹配该路径前缀的数据才会被抽取，用作后续的处理和发送</span> |
| <span class="token string">'extractor.pattern'</span> <span class="token operator">=</span> <span class="token string">'root.timecho'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 是否抽取历史数据</span> |
| <span class="token string">'extractor.history.enable'</span> <span class="token operator">=</span> <span class="token string">'true'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 描述被抽取的历史数据的时间范围，表示最早时间</span> |
| <span class="token string">'extractor.history.start-time'</span> <span class="token operator">=</span> <span class="token string">'2011.12.03T10:15:30+01:00'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 描述被抽取的历史数据的时间范围，表示最晚时间</span> |
| <span class="token string">'extractor.history.end-time'</span> <span class="token operator">=</span> <span class="token string">'2022.12.03T10:15:30+01:00'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 是否抽取实时数据</span> |
| <span class="token string">'extractor.realtime.enable'</span> <span class="token operator">=</span> <span class="token string">'true'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 描述实时数据的抽取方式</span> |
| <span class="token string">'extractor.realtime.mode'</span> <span class="token operator">=</span> <span class="token string">'hybrid'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| <span class="token keyword">WITH</span> PROCESSOR <span class="token punctuation">(</span> |
| <span class="token comment">-- 默认的数据处理插件，即不做任何处理</span> |
| <span class="token string">'processor'</span> <span class="token operator">=</span> <span class="token string">'do-nothing-processor'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| <span class="token keyword">WITH</span> CONNECTOR <span class="token punctuation">(</span> |
| <span class="token comment">-- IoTDB 数据发送插件，目标端为 IoTDB</span> |
| <span class="token string">'connector'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-connector'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 目标端 IoTDB 其中一个 DataNode 节点的数据服务 ip</span> |
| <span class="token string">'connector.ip'</span> <span class="token operator">=</span> <span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 目标端 IoTDB 其中一个 DataNode 节点的数据服务 port</span> |
| <span class="token string">'connector.port'</span> <span class="token operator">=</span> <span class="token string">'6667'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p><strong>创建同步任务时需要配置 PipeId 以及三个插件部分的参数：</strong></p><table><thead><tr><th>配置项</th><th>说明</th><th>是否必填</th><th>默认实现</th><th>默认实现说明</th><th>是否允许自定义实现</th></tr></thead><tbody><tr><td>PipeId</td><td>全局唯一标定一个同步任务的名称</td><td><!----></td><td>-</td><td>-</td><td>-</td></tr><tr><td>extractor</td><td>Pipe Extractor 插件，负责在数据库底层抽取同步数据</td><td>选填</td><td>iotdb-extractor</td><td>将数据库的全量历史数据和后续到达的实时数据接入同步任务</td><td>否</td></tr><tr><td>processor</td><td>Pipe Processor 插件，负责处理数据</td><td>选填</td><td>do-nothing-processor</td><td>对传入的数据不做任何处理</td><td><!----></td></tr><tr><td>connector</td><td>Pipe Connector 插件，负责发送数据</td><td><!----></td><td>-</td><td>-</td><td><!----></td></tr></tbody></table><p>示例中，使用了 iotdb-extractor、do-nothing-processor 和 iotdb-thrift-connector 插件构建数据同步任务。IoTDB 还内置了其他的数据同步插件，<strong>请查看“系统预置数据同步插件”一节</strong>。</p><p><strong>一个最简的 CREATE PIPE 语句示例如下：</strong></p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> <span class="token comment">-- PipeId 是能够唯一标定任务任务的名字</span> |
| <span class="token keyword">WITH</span> CONNECTOR <span class="token punctuation">(</span> |
| <span class="token comment">-- IoTDB 数据发送插件，目标端为 IoTDB</span> |
| <span class="token string">'connector'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-connector'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 目标端 IoTDB 其中一个 DataNode 节点的数据服务 ip</span> |
| <span class="token string">'connector.ip'</span> <span class="token operator">=</span> <span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- 目标端 IoTDB 其中一个 DataNode 节点的数据服务 port</span> |
| <span class="token string">'connector.port'</span> <span class="token operator">=</span> <span class="token string">'6667'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>其表达的语义是：将本数据库实例中的全量历史数据和后续到达的实时数据，同步到目标为 127.0.0.1:6667 的 IoTDB 实例上。</p><p><strong>注意：</strong></p><ul><li><p>EXTRACTOR 和 PROCESSOR 为选填配置，若不填写配置参数，系统则会采用相应的默认实现</p></li><li><p>CONNECTOR 为必填配置，需要在 CREATE PIPE 语句中声明式配置</p></li><li><p>CONNECTOR 具备自复用能力。对于不同的任务，如果他们的 CONNECTOR 具备完全相同 KV 属性的（所有属性的 key 对应的 value 都相同），<strong>那么系统最终只会创建一个 CONNECTOR 实例</strong>，以实现对连接资源的复用。</p><ul><li>例如，有下面 pipe1, pipe2 两个任务的声明：</li></ul><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE pipe1 |
| <span class="token keyword">WITH</span> CONNECTOR <span class="token punctuation">(</span> |
| <span class="token string">'connector'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-connector'</span><span class="token punctuation">,</span> |
| <span class="token string">'connector.thrift.host'</span> <span class="token operator">=</span> <span class="token string">'localhost'</span><span class="token punctuation">,</span> |
| <span class="token string">'connector.thrift.port'</span> <span class="token operator">=</span> <span class="token string">'9999'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| |
| <span class="token keyword">CREATE</span> PIPE pipe2 |
| <span class="token keyword">WITH</span> CONNECTOR <span class="token punctuation">(</span> |
| <span class="token string">'connector'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-connector'</span><span class="token punctuation">,</span> |
| <span class="token string">'connector.thrift.port'</span> <span class="token operator">=</span> <span class="token string">'9999'</span><span class="token punctuation">,</span> |
| <span class="token string">'connector.thrift.host'</span> <span class="token operator">=</span> <span class="token string">'localhost'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><ul><li>因为它们对 CONNECTOR 的声明完全相同（<strong>即使某些属性声明时的顺序不同</strong>），所以框架会自动对它们声明的 CONNECTOR 进行复用，最终 pipe1, pipe2 的CONNECTOR 将会是同一个实例。</li></ul></li><li><p>在 extractor 为默认的 iotdb-extractor，且 extractor.forwarding-pipe-requests 为默认值 true 时，请不要构建出包含数据循环同步的应用场景（会导致无限循环）：</p><ul><li>IoTDB A -> IoTDB B -> IoTDB A</li><li>IoTDB A -> IoTDB A</li></ul></li></ul><h3 id="启动任务" tabindex="-1"><a class="header-anchor" href="#启动任务" aria-hidden="true">#</a> 启动任务</h3><p>CREATE PIPE 语句成功执行后，任务相关实例会被创建，但整个任务的运行状态会被置为 STOPPED，即任务不会立刻处理数据。</p><p>可以使用 START PIPE 语句使任务开始处理数据：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">START</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="停止任务" tabindex="-1"><a class="header-anchor" href="#停止任务" aria-hidden="true">#</a> 停止任务</h3><p>使用 STOP PIPE 语句使任务停止处理数据：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code>STOP PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="删除任务" tabindex="-1"><a class="header-anchor" href="#删除任务" aria-hidden="true">#</a> 删除任务</h3><p>使用 DROP PIPE 语句使任务停止处理数据（当任务状态为 RUNNING 时），然后删除整个任务同步任务：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">DROP</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>用户在删除任务前，不需要执行 STOP 操作。</p><h3 id="展示任务" tabindex="-1"><a class="header-anchor" href="#展示任务" aria-hidden="true">#</a> 展示任务</h3><p>使用 SHOW PIPES 语句查看所有任务：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPES |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>查询结果如下：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+-------------+-------------+-------------+----------------+</span> |
| <span class="token operator">|</span> ID<span class="token operator">|</span> CreationTime <span class="token operator">|</span> State<span class="token operator">|</span>PipeExtractor<span class="token operator">|</span>PipeProcessor<span class="token operator">|</span>PipeConnector<span class="token operator">|</span>ExceptionMessage<span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+-------------+-------------+-------------+----------------+</span> |
| <span class="token operator">|</span>iotdb<span class="token operator">-</span>kafka<span class="token operator">|</span><span class="token number">2022</span><span class="token operator">-</span><span class="token number">03</span><span class="token operator">-</span><span class="token number">30</span>T20:<span class="token number">58</span>:<span class="token number">30.689</span><span class="token operator">|</span>RUNNING<span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> None<span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+-------------+-------------+-------------+----------------+</span> |
| <span class="token operator">|</span>iotdb<span class="token operator">-</span>iotdb<span class="token operator">|</span><span class="token number">2022</span><span class="token operator">-</span><span class="token number">03</span><span class="token operator">-</span><span class="token number">31</span>T12:<span class="token number">55</span>:<span class="token number">28.129</span><span class="token operator">|</span>STOPPED<span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> TException: <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+-------------+-------------+-------------+----------------+</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>可以使用 <code><PipeId></code> 指定想看的某个同步任务状态：</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>您也可以通过 where 子句，判断某个 <PipeId> 使用的 Pipe Connector 被复用的情况。</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPES |
| <span class="token keyword">WHERE</span> CONNECTOR USED <span class="token keyword">BY</span> <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="任务运行状态迁移" tabindex="-1"><a class="header-anchor" href="#任务运行状态迁移" aria-hidden="true">#</a> 任务运行状态迁移</h3><p>一个数据同步 pipe 在其被管理的生命周期中会经过多种状态：</p><ul><li><strong>STOPPED：</strong> pipe 处于停止运行状态。当管道处于该状态时，有如下几种可能： <ul><li>当一个 pipe 被成功创建之后，其初始状态为暂停状态</li><li>用户手动将一个处于正常运行状态的 pipe 暂停，其状态会被动从 RUNNING 变为 STOPPED</li><li>当一个 pipe 运行过程中出现无法恢复的错误时，其状态会自动从 RUNNING 变为 STOPPED</li></ul></li><li><strong>RUNNING：</strong> pipe 正在正常工作</li><li><strong>DROPPED：</strong> pipe 任务被永久删除</li></ul><p>下图表明了所有状态以及状态的迁移：</p><figure><img src="https://alioss.timecho.com/docs/img/状态迁移图.png" alt="状态迁移图" tabindex="0" loading="lazy"><figcaption>状态迁移图</figcaption></figure><h2 id="系统预置数据同步插件" tabindex="-1"><a class="header-anchor" href="#系统预置数据同步插件" aria-hidden="true">#</a> 系统预置数据同步插件</h2><h3 id="查看预置插件" tabindex="-1"><a class="header-anchor" href="#查看预置插件" aria-hidden="true">#</a> 查看预置插件</h3><p>用户可以按需查看系统中的插件。查看插件的语句如图所示。</p><div class="language-sql line-numbers-mode" data-ext="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPEPLUGINS |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="预置-extractor-插件" tabindex="-1"><a class="header-anchor" href="#预置-extractor-插件" aria-hidden="true">#</a> 预置 extractor 插件</h3><h4 id="iotdb-extractor" tabindex="-1"><a class="header-anchor" href="#iotdb-extractor" aria-hidden="true">#</a> iotdb-extractor</h4><p>作用：抽取 IoTDB 内部的历史或实时数据进入 pipe。</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>extractor</td><td>iotdb-extractor</td><td>String: iotdb-extractor</td><td>required</td></tr><tr><td>extractor.pattern</td><td>用于筛选时间序列的路径前缀</td><td>String: 任意的时间序列前缀</td><td>optional: root</td></tr><tr><td>extractor.history.enable</td><td>是否同步历史数据</td><td>Boolean: true, false</td><td>optional: true</td></tr><tr><td>extractor.history.start-time</td><td>同步历史数据的开始 event time，包含 start-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MIN_VALUE</td></tr><tr><td>extractor.history.end-time</td><td>同步历史数据的结束 event time，包含 end-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MAX_VALUE</td></tr><tr><td>extractor.realtime.enable</td><td>是否同步实时数据</td><td>Boolean: true, false</td><td>optional: true</td></tr><tr><td>extractor.realtime.mode</td><td>实时数据的抽取模式</td><td>String: hybrid, log, file</td><td>optional: hybrid</td></tr><tr><td>extractor.forwarding-pipe-requests</td><td>是否转发由其他 Pipe （通常是数据同步）写入的数据</td><td>Boolean: true, false</td><td>optional: true</td></tr></tbody></table><blockquote><p>🚫 <strong>extractor.pattern 参数说明</strong></p><ul><li><p>Pattern 需用反引号修饰不合法字符或者是不合法路径节点，例如如果希望筛选 root.`a@b` 或者 root.`123`，应设置 pattern 为 root.`a@b` 或者 root.`123`（具体参考 <a href="https://iotdb.apache.org/zh/Download/#_1-0-%E7%89%88%E6%9C%AC%E4%B8%8D%E5%85%BC%E5%AE%B9%E7%9A%84%E8%AF%AD%E6%B3%95%E8%AF%A6%E7%BB%86%E8%AF%B4%E6%98%8E" target="_blank" rel="noopener noreferrer">单双引号和反引号的使用时机<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span></a>）</p></li><li><p>在底层实现中，当检测到 pattern 为 root（默认值）时，同步效率较高，其他任意格式都将降低性能</p></li><li><p>路径前缀不需要能够构成完整的路径。例如，当创建一个包含参数为 'extractor.pattern'='root.aligned.1' 的 pipe 时：</p><ul><li>root.aligned.1TS</li><li>root.aligned.1TS.`1`</li><li>root.aligned.100TS</li></ul><p>的数据会被同步；</p><ul><li>root.aligned.`1`</li><li>root.aligned.`123`</li></ul><p>的数据不会被同步。</p></li><li><p>root.__system 的数据不会被 pipe 抽取，即不会被同步到目标端。用户虽然可以在 extractor.pattern 中包含任意前缀，包括带有（或覆盖） root.__system 的前缀，但是 root.__system 下的数据总是会被 pipe 忽略的</p></li></ul></blockquote><blockquote><p>❗️<strong>extractor.history 的 start-time，end-time 参数说明</strong></p><ul><li>start-time，end-time 应为 ISO 格式，例如 2011-12-03T10:15:30 或 2011-12-03T10:15:30+01:00</li></ul></blockquote><blockquote><p>✅ <strong>一条数据从生产到落库 IoTDB，包含两个关键的时间概念</strong></p><ul><li><strong>event time：</strong> 数据实际生产时的时间（或者数据生产系统给数据赋予的生成时间，是数据点中的时间项），也称为事件时间。</li><li><strong>arrival time：</strong> 数据到达 IoTDB 系统内的时间。</li></ul><p>我们常说的乱序数据，指的是数据到达时，其 <strong>event time</strong> 远落后于当前系统时间（或者已经落库的最大 <strong>event time</strong>）的数据。另一方面，不论是乱序数据还是顺序数据，只要它们是新到达系统的，那它们的 <strong>arrival time</strong> 都是会随着数据到达 IoTDB 的顺序递增的。</p></blockquote><blockquote><p>💎 <strong>iotdb-extractor 的工作可以拆分成两个阶段</strong></p><ol><li>历史数据抽取：所有 <strong>arrival time</strong> < 创建 pipe 时<strong>当前系统时间</strong>的数据称为历史数据</li><li>实时数据抽取：所有 <strong>arrival time</strong> >= 创建 pipe 时<strong>当前系统时间</strong>的数据称为实时数据</li></ol><p>历史数据传输阶段和实时数据传输阶段，<strong>两阶段串行执行，只有当历史数据传输阶段完成后，才执行实时数据传输阶段。</strong></p><p>用户可以指定 iotdb-extractor 进行：</p><ul><li>历史数据抽取（<code>'extractor.history.enable' = 'true'</code>, <code>'extractor.realtime.enable' = 'false'</code> ）</li><li>实时数据抽取（<code>'extractor.history.enable' = 'false'</code>, <code>'extractor.realtime.enable' = 'true'</code> ）</li><li>全量数据抽取（<code>'extractor.history.enable' = 'true'</code>, <code>'extractor.realtime.enable' = 'true'</code> ）</li><li>禁止同时设置 <code>extractor.history.enable</code> 和 <code>extractor.realtime.enable</code> 为 <code>false</code></li></ul></blockquote><blockquote><p>📌 <strong>extractor.realtime.mode：数据抽取的模式</strong></p><ul><li>log：该模式下，任务仅使用操作日志进行数据处理、发送</li><li>file：该模式下，任务仅使用数据文件进行数据处理、发送</li><li>hybrid：该模式，考虑了按操作日志逐条目发送数据时延迟低但吞吐低的特点，以及按数据文件批量发送时发送吞吐高但延迟高的特点，能够在不同的写入负载下自动切换适合的数据抽取方式，首先采取基于操作日志的数据抽取方式以保证低发送延迟，当产生数据积压时自动切换成基于数据文件的数据抽取方式以保证高发送吞吐，积压消除时自动切换回基于操作日志的数据抽取方式，避免了采用单一数据抽取算法难以平衡数据发送延迟或吞吐的问题。</li></ul></blockquote><blockquote><p>🍕 <strong>extractor.forwarding-pipe-requests：是否允许转发从另一 pipe 传输而来的数据</strong></p><ul><li>如果要使用 pipe 构建 A -> B -> C 的数据同步，那么 B -> C 的 pipe 需要将该参数为 true 后，A -> B 中 A 通过 pipe 写入 B 的数据才能被正确转发到 C</li><li>如果要使用 pipe 构建 A <-> B 的双向数据同步（双活），那么 A -> B 和 B -> A 的 pipe 都需要将该参数设置为 false，否则将会造成数据无休止的集群间循环转发</li></ul></blockquote><h3 id="预置-processor-插件" tabindex="-1"><a class="header-anchor" href="#预置-processor-插件" aria-hidden="true">#</a> 预置 processor 插件</h3><h4 id="do-nothing-processor" tabindex="-1"><a class="header-anchor" href="#do-nothing-processor" aria-hidden="true">#</a> do-nothing-processor</h4><p>作用：不对 extractor 传入的事件做任何的处理。</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>processor</td><td>do-nothing-processor</td><td>String: do-nothing-processor</td><td>required</td></tr></tbody></table><h3 id="预置-connector-插件" tabindex="-1"><a class="header-anchor" href="#预置-connector-插件" aria-hidden="true">#</a> 预置 connector 插件</h3><h4 id="iotdb-thrift-sync-connector-别名-iotdb-thrift-connector" tabindex="-1"><a class="header-anchor" href="#iotdb-thrift-sync-connector-别名-iotdb-thrift-connector" aria-hidden="true">#</a> iotdb-thrift-sync-connector（别名：iotdb-thrift-connector）</h4><p>作用：主要用于 IoTDB（v1.2.0+）与 IoTDB（v1.2.0+）之间的数据传输。<br> 使用 Thrift RPC 框架传输数据，单线程 blocking IO 模型。<br> 保证接收端 apply 数据的顺序与发送端接受写入请求的顺序一致。</p><p>限制：源端 IoTDB 与 目标端 IoTDB 版本都需要在 v1.2.0+。</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>connector</td><td>iotdb-thrift-connector 或 iotdb-thrift-sync-connector</td><td>String: iotdb-thrift-connector 或 iotdb-thrift-sync-connector</td><td>required</td></tr><tr><td>connector.ip</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 ip</td><td>String</td><td>optional: 与 connector.node-urls 任选其一填写</td></tr><tr><td>connector.port</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 port</td><td>Integer</td><td>optional: 与 connector.node-urls 任选其一填写</td></tr><tr><td>connector.node-urls</td><td>目标端 IoTDB 任意多个 DataNode 节点的数据服务端口的 url</td><td>String。例：'127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>optional: 与 connector.ip:connector.port 任选其一填写</td></tr><tr><td>connector.batch.enable</td><td>是否开启日志攒批发送模式，用于提高传输吞吐，降低 IOPS</td><td>Boolean: true, false</td><td>optional: true</td></tr><tr><td>connector.batch.max-delay-seconds</td><td>在开启日志攒批发送模式时生效，表示一批数据在发送前的最长等待时间（单位：s）</td><td>Integer</td><td>optional: 1</td></tr><tr><td>connector.batch.size-bytes</td><td>在开启日志攒批发送模式时生效，表示一批数据最大的攒批大小（单位：byte）</td><td>Long</td><td>optional: 16 * 1024 * 1024 (16MiB)</td></tr></tbody></table><blockquote><p>📌 请确保接收端已经创建了发送端的所有时间序列，或是开启了自动创建元数据，否则将会导致 pipe 运行失败。</p></blockquote><h4 id="iotdb-thrift-async-connector" tabindex="-1"><a class="header-anchor" href="#iotdb-thrift-async-connector" aria-hidden="true">#</a> iotdb-thrift-async-connector</h4><p>作用：主要用于 IoTDB（v1.2.0+）与 IoTDB（v1.2.0+）之间的数据传输。<br> 使用 Thrift RPC 框架传输数据，多线程 async non-blocking IO 模型，传输性能高，尤其适用于目标端为分布式时的场景。<br> 不保证接收端 apply 数据的顺序与发送端接受写入请求的顺序一致，但是保证数据发送的完整性（at-least-once）。</p><p>限制：源端 IoTDB 与 目标端 IoTDB 版本都需要在 v1.2.0+。</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>connector</td><td>iotdb-thrift-async-connector</td><td>String: iotdb-thrift-async-connector</td><td>required</td></tr><tr><td>connector.ip</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 ip</td><td>String</td><td>optional: 与 connector.node-urls 任选其一填写</td></tr><tr><td>connector.port</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 port</td><td>Integer</td><td>optional: 与 connector.node-urls 任选其一填写</td></tr><tr><td>connector.node-urls</td><td>目标端 IoTDB 任意多个 DataNode 节点的数据服务端口的 url</td><td>String。例：'127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>optional: 与 connector.ip:connector.port 任选其一填写</td></tr><tr><td>connector.batch.enable</td><td>是否开启日志攒批发送模式，用于提高传输吞吐，降低 IOPS</td><td>Boolean: true, false</td><td>optional: true</td></tr><tr><td>connector.batch.max-delay-seconds</td><td>在开启日志攒批发送模式时生效，表示一批数据在发送前的最长等待时间（单位：s）</td><td>Integer</td><td>optional: 1</td></tr><tr><td>connector.batch.size-bytes</td><td>在开启日志攒批发送模式时生效，表示一批数据最大的攒批大小（单位：byte）</td><td>Long</td><td>optional: 16 * 1024 * 1024 (16MiB)</td></tr></tbody></table><blockquote><p>📌 请确保接收端已经创建了发送端的所有时间序列，或是开启了自动创建元数据，否则将会导致 pipe 运行失败。</p></blockquote><h4 id="iotdb-legacy-pipe-connector" tabindex="-1"><a class="header-anchor" href="#iotdb-legacy-pipe-connector" aria-hidden="true">#</a> iotdb-legacy-pipe-connector</h4><p>作用：主要用于 IoTDB（v1.2.0+）向更低版本的 IoTDB 传输数据，使用 v1.2.0 版本前的数据同步（Sync）协议。<br> 使用 Thrift RPC 框架传输数据。单线程 sync blocking IO 模型，传输性能较弱。</p><p>限制：源端 IoTDB 版本需要在 v1.2.0+，目标端 IoTDB 版本可以是 v1.2.0+、v1.1.x（更低版本的 IoTDB 理论上也支持，但是未经测试）。</p><p>注意：理论上 v1.2.0+ IoTDB 可作为 v1.2.0 版本前的任意版本的数据同步（Sync）接收端。</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>connector</td><td>iotdb-legacy-pipe-connector</td><td>String: iotdb-legacy-pipe-connector</td><td>required</td></tr><tr><td>connector.ip</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 ip</td><td>String</td><td>required</td></tr><tr><td>connector.port</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 port</td><td>Integer</td><td>required</td></tr><tr><td>connector.user</td><td>目标端 IoTDB 的用户名，注意该用户需要支持数据写入、TsFile Load 的权限</td><td>String</td><td>optional: root</td></tr><tr><td>connector.password</td><td>目标端 IoTDB 的密码，注意该用户需要支持数据写入、TsFile Load 的权限</td><td>String</td><td>optional: root</td></tr><tr><td>connector.version</td><td>目标端 IoTDB 的版本，用于伪装自身实际版本，绕过目标端的版本一致性检查</td><td>String</td><td>optional: 1.1</td></tr></tbody></table><blockquote><p>📌 请确保接收端已经创建了发送端的所有时间序列，或是开启了自动创建元数据，否则将会导致 pipe 运行失败。</p></blockquote><h4 id="iotdb-air-gap-connector" tabindex="-1"><a class="header-anchor" href="#iotdb-air-gap-connector" aria-hidden="true">#</a> iotdb-air-gap-connector</h4><p>作用：用于 IoTDB（v1.2.2+）向 IoTDB（v1.2.2+）跨单向数据网闸的数据同步。支持的网闸型号包括南瑞 Syskeeper 2000 等。<br> 该 Connector 使用 Java 自带的 Socket 实现数据传输，单线程 blocking IO 模型，其性能与 iotdb-thrift-sync-connector 相当。<br> 保证接收端 apply 数据的顺序与发送端接受写入请求的顺序一致。</p><p>场景：例如，在电力系统的规范中</p><blockquote><p>1．I/II 区与 III 区之间的应用程序禁止采用 SQL 命令访问数据库和基于 B/S 方式的双向数据传输</p><p>2．I/II 区与 III 区之间的数据通信，传输的启动端由内网发起，反向的应答报文不容许携带数据，应用层的应答报文最多为 1 个字节，并且 1 个字节为全 0 或者全 1 两种状态</p></blockquote><p>限制：</p><ol><li>源端 IoTDB 与 目标端 IoTDB 版本都需要在 v1.2.2+。</li><li>单向数据网闸需要允许 TCP 请求跨越，且每一个请求可返回一个全 1 或全 0 的 byte。</li><li>目标端 IoTDB 需要在 iotdb-common.properties 内，配置<br> a. pipe_air_gap_receiver_enabled=true<br> b. pipe_air_gap_receiver_port 配置 receiver 的接收端口</li></ol><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>connector</td><td>iotdb-air-gap-connector</td><td>String: iotdb-air-gap-connector</td><td>required</td></tr><tr><td>connector.ip</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 ip</td><td>String</td><td>optional: 与 connector.node-urls 任选其一填写</td></tr><tr><td>connector.port</td><td>目标端 IoTDB 其中一个 DataNode 节点的数据服务 port</td><td>Integer</td><td>optional: 与 connector.node-urls 任选其一填写</td></tr><tr><td>connector.node-urls</td><td>目标端 IoTDB 任意多个 DataNode 节点的数据服务端口的 url</td><td>String。例：'127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>optional: 与 connector.ip:connector.port 任选其一填写</td></tr><tr><td>connector.air-gap.handshake-timeout-ms</td><td>发送端与接收端在首次尝试建立连接时握手请求的超时时长，单位：毫秒</td><td>Integer</td><td>optional: 5000</td></tr></tbody></table><blockquote><p>📌 请确保接收端已经创建了发送端的所有时间序列，或是开启了自动创建元数据，否则将会导致 pipe 运行失败。</p></blockquote><h4 id="do-nothing-connector" tabindex="-1"><a class="header-anchor" href="#do-nothing-connector" aria-hidden="true">#</a> do-nothing-connector</h4><p>作用：不对 processor 传入的事件做任何的处理。</p><table><thead><tr><th>key</th><th>value</th><th>value 取值范围</th><th>required or optional with default</th></tr></thead><tbody><tr><td>connector</td><td>do-nothing-connector</td><td>String: do-nothing-connector</td><td>required</td></tr></tbody></table><h2 id="权限管理" tabindex="-1"><a class="header-anchor" href="#权限管理" aria-hidden="true">#</a> 权限管理</h2><table><thead><tr><th>权限名称</th><th>描述</th></tr></thead><tbody><tr><td>CREATE_PIPE</td><td>注册任务。路径无关。</td></tr><tr><td>START_PIPE</td><td>开启任务。路径无关。</td></tr><tr><td>STOP_PIPE</td><td>停止任务。路径无关。</td></tr><tr><td>DROP_PIPE</td><td>卸载任务。路径无关。</td></tr><tr><td>SHOW_PIPES</td><td>查询任务。路径无关。</td></tr></tbody></table><h2 id="配置参数" tabindex="-1"><a class="header-anchor" href="#配置参数" aria-hidden="true">#</a> 配置参数</h2><p>在 iotdb-common.properties 中：</p><div class="language-Properties line-numbers-mode" data-ext="Properties"><pre class="language-Properties"><code>#################### |
| ### Pipe Configuration |
| #################### |
| |
| # Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_connector_timeout_ms=900000 |
| |
| # The maximum number of selectors that can be used in the async connector. |
| # pipe_async_connector_selector_number=1 |
| |
| # The core number of clients that can be used in the async connector. |
| # pipe_async_connector_core_client_number=8 |
| |
| # The maximum number of clients that can be used in the async connector. |
| # pipe_async_connector_max_client_number=16 |
| |
| # Whether to enable receiving pipe data through air gap. |
| # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. |
| # pipe_air_gap_receiver_enabled=false |
| |
| # The port for the server to receive pipe data through air gap. |
| # pipe_air_gap_receiver_port=9780 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="功能特性" tabindex="-1"><a class="header-anchor" href="#功能特性" aria-hidden="true">#</a> 功能特性</h2><h3 id="最少一次语义保证-at-least-once" tabindex="-1"><a class="header-anchor" href="#最少一次语义保证-at-least-once" aria-hidden="true">#</a> 最少一次语义保证 <strong>at-least-once</strong></h3><p>数据同步功能向外部系统传输数据时，提供 at-least-once 的传输语义。在大部分场景下，同步功能可提供 exactly-once 保证，即所有数据被恰好同步一次。</p><p>但是在以下场景中，可能存在部分数据被同步多次 <strong>（断点续传）</strong> 的情况：</p><ul><li>临时的网络故障：某次数据传输请求失败后，系统会进行重试发送，直至到达最大尝试次数</li><li>Pipe 插件逻辑实现异常：插件运行中抛出错误，系统会进行重试发送，直至到达最大尝试次数</li><li>数据节点宕机、重启等导致的数据分区切主：分区变更完成后，受影响的数据会被重新传输</li><li>集群不可用：集群可用后，受影响的数据会重新传输</li></ul><h3 id="源端-数据写入与-pipe-处理、发送数据异步解耦" tabindex="-1"><a class="header-anchor" href="#源端-数据写入与-pipe-处理、发送数据异步解耦" aria-hidden="true">#</a> 源端：数据写入与 Pipe 处理、发送数据异步解耦</h3><p>数据同步功能中，数据传输采用的是异步复制模式。</p><p>数据同步与写入操作完全脱钩，不存在对写入关键路径的影响。该机制允许框架在保证持续数据同步的前提下，保持时序数据库的写入速度。</p><h3 id="源端-可自适应数据写入负载的数据传输策略" tabindex="-1"><a class="header-anchor" href="#源端-可自适应数据写入负载的数据传输策略" aria-hidden="true">#</a> 源端：可自适应数据写入负载的数据传输策略</h3><p>支持根据写入负载，动态调整数据传输方式，同步默认使用 TsFile 文件与操作流动态混合传输（<code>'extractor.realtime.mode'='hybrid'</code>）。</p><p>在数据写入负载高时，优先选择 TsFile 传输的方式。TsFile 压缩比高，节省网络带宽。</p><p>在数据写入负载低时，优先选择操作流同步传输的方式。操作流传输实时性高。</p><h3 id="源端-高可用集群部署时-pipe-服务高可用" tabindex="-1"><a class="header-anchor" href="#源端-高可用集群部署时-pipe-服务高可用" aria-hidden="true">#</a> 源端：高可用集群部署时，Pipe 服务高可用</h3><p>当发送端 IoTDB 为高可用集群部署模式时，数据同步服务也将是高可用的。 数据同步框架将监控每个数据节点的数据同步进度，并定期做轻量级的分布式一致性快照以保存同步状态。</p><ul><li>当发送端集群某数据节点宕机时，数据同步框架可以利用一致性快照以及保存在副本上的数据快速恢复同步，以此实现数据同步服务的高可用。</li><li>当发送端集群整体宕机并重启时，数据同步框架也能使用快照恢复同步服务。</li></ul></div><!--[--><!----><!--]--><footer class="page-meta"><div class="meta-item edit-link"><a href="https://github.com/apache/iotdb-docs/edit/main/src/zh/UserGuide/V1.2.x/User-Manual/Stage_Data-Sync_timecho.md" rel="noopener noreferrer" target="_blank" aria-label="发现错误？在 GitHub 上编辑此页" class="nav-link label"><!--[--><svg xmlns="http://www.w3.org/2000/svg" class="icon edit-icon" viewBox="0 0 1024 1024" fill="currentColor" aria-label="edit icon"><path d="M430.818 653.65a60.46 60.46 0 0 1-50.96-93.281l71.69-114.012 7.773-10.365L816.038 80.138A60.46 60.46 0 0 1 859.225 62a60.46 60.46 0 0 1 43.186 18.138l43.186 43.186a60.46 60.46 0 0 1 0 86.373L588.879 565.55l-8.637 8.637-117.466 68.234a60.46 60.46 0 0 1-31.958 11.229z"></path><path d="M728.802 962H252.891A190.883 190.883 0 0 1 62.008 771.98V296.934a190.883 190.883 0 0 1 190.883-192.61h267.754a60.46 60.46 0 0 1 0 120.92H252.891a69.962 69.962 0 0 0-69.098 69.099V771.98a69.962 69.962 0 0 0 69.098 69.098h475.911A69.962 69.962 0 0 0 797.9 771.98V503.363a60.46 60.46 0 1 1 120.922 0V771.98A190.883 190.883 0 0 1 728.802 962z"></path></svg><!--]-->发现错误？在 GitHub 上编辑此页<span><svg class="external-link-icon" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path><polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg><span class="external-link-icon-sr-only">open in new window</span></span><!----></a></div><div class="meta-item git-info"><div class="update-time"><span class="label">上次编辑于: </span><!----></div><div class="contributors"><span class="label">贡献者: </span><!--[--><!--[--><span class="contributor" title="email: 105700158+wanghui42@users.noreply.github.com">wanghui42</span><!--]--><!--]--></div></div></footer><!----><!----><!--[--><!----><!--]--><!--]--></main><!--]--><footer style="padding-bottom:2rem;"><span id="doc-version" style="display:none;">rel/1.2</span><p style="text-align:center;color:#909399;font-size:12px;margin:0 30px;">Copyright © 2024 The Apache Software Foundation.<br> Apache and the Apache feather logo are trademarks of The Apache Software Foundation</p><p style="text-align:center;margin-top:10px;color:#909399;font-size:12px;margin:0 30px;"><strong>Have a question?</strong> Connect with us on QQ, WeChat, or Slack. <a href="https://github.com/apache/iotdb/issues/1995">Join the community</a> now.</p></footer></div><!--]--><!--]--><!----><!--]--></div> |
| <script type="module" src="/assets/app-z4N-28Th.js" defer></script> |
| </body> |
| </html> |
