| import{_ as e,o as t,c as i,e as a}from"./app-Bp5kEZWW.js";const n={},s=a(`<h1 id="data-synchronisation" tabindex="-1"><a class="header-anchor" href="#data-synchronisation"><span>Data synchronisation</span></a></h1><p>Data synchronisation is a typical requirement of industrial IoT. Through the data synchronisation mechanism, data sharing between IoTDBs can be achieved, and a complete data link can be built to meet the needs of intranet and extranet data interoperability, end-to-end cloud synchronisation, data migration, data backup, and so on.</p><h2 id="introduction" tabindex="-1"><a class="header-anchor" href="#introduction"><span>Introduction</span></a></h2><h3 id="synchronisation-task-overview" tabindex="-1"><a class="header-anchor" href="#synchronisation-task-overview"><span>Synchronisation Task Overview</span></a></h3><p>A data synchronisation task consists of 2 phases:</p><ul><li>Source phase: This part is used to extract data from the source IoTDB, which is defined in the source part of the SQL statement.</li><li>Sink phase: This part is used to send data to the target IoTDB and is defined in the sink part of the SQL statement.</li></ul><p>Flexible data synchronisation capabilities can be achieved by declaratively configuring the specifics of the 2 sections through SQL statements.</p><h3 id="synchronisation-task-create" tabindex="-1"><a class="header-anchor" href="#synchronisation-task-create"><span>Synchronisation Task - Create</span></a></h3><p>Use the <code>CREATE PIPE</code> statement to create a data synchronisation task, the following attributes <code>PipeId</code> and <code>sink</code> are mandatory, <code>source</code> and <code>processor</code> are optional, when entering the SQL note that the order of the <code>SOURCE </code> and <code>SINK</code> plugins are not interchangeable.</p><p>The SQL example is as follows:</p><div class="language-SQL line-numbers-mode" data-ext="SQL" data-title="SQL"><pre class="language-SQL"><code>CREATE PIPE <PipeId> -- PipeId is the name that uniquely identifies the task. |
| -- Data Extraction Plugin, Required Plugin |
| WITH SOURCE ( |
| [<parameter> = <value>,], [<value>,] |
| -- Data connection plugin, required |
| WITH SINK ( |
| [<parameter> = <value>,], -- data connection plugin, required. |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><blockquote><p>π Note: To use the data synchronisation feature, make sure that automatic metadata creation is enabled on the receiving side</p></blockquote><h3 id="synchronisation-tasks-management" tabindex="-1"><a class="header-anchor" href="#synchronisation-tasks-management"><span>Synchronisation Tasks - Management</span></a></h3><p>The Data Synchronisation task has three states; RUNNING, STOPPED and DROPPED.The task state transitions are shown below:</p><figure><img src="https://alioss.timecho.com/docs/img/ηΆζθΏη§»εΎ.png" alt="State Migration Diagram" tabindex="0" loading="lazy"><figcaption>State Migration Diagram</figcaption></figure><p>A data synchronisation task passes through multiple states during its lifecycle:</p><ul><li>RUNNING: Running state. <ul><li>Explanation 1: The initial state of the task is the running state(V1.3.1+).</li></ul></li><li>STOPPED: Stopped state. <ul><li>Description 1: The initial state of the task is the stopped state(V1.3.0). A SQL statement is required to start the task.</li><li>Description 2: You can manually stop a running task with a SQL statement, and the state will change from RUNNING to STOPPED.</li><li>Description 3: When a task has an unrecoverable error, its status will automatically change from RUNNING to STOPPED.</li></ul></li><li>DROPPED: deleted state.</li></ul><p>We provide the following SQL statements to manage the status of synchronisation tasks.</p><h4 id="starting-a-task" tabindex="-1"><a class="header-anchor" href="#starting-a-task"><span>Starting a Task</span></a></h4><p>After creation, the task will not be processed immediately, you need to start the task. Use the <code>START PIPE</code> statement to start the task so that it can begin processing data:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>START PIPE<PipeId> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h4 id="stop-the-task" tabindex="-1"><a class="header-anchor" href="#stop-the-task"><span>Stop the task</span></a></h4><p>Stop processing data:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>STOP PIPE <PipeId> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h4 id="delete-a-task" tabindex="-1"><a class="header-anchor" href="#delete-a-task"><span>Delete a task</span></a></h4><p>Deletes the specified task:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>DROP PIPE <PipeId> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>Deleting a task does not require you to stop synchronising the task first.</p><h4 id="viewing-tasks" tabindex="-1"><a class="header-anchor" href="#viewing-tasks"><span>Viewing Tasks</span></a></h4><p>View all tasks:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>SHOW PIPES |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>To view a specified task:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>SHOW PIPE <PipeId>. |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="plugin" tabindex="-1"><a class="header-anchor" href="#plugin"><span>Plugin</span></a></h3><p>In order to make the overall architecture more flexible to match different synchronisation scenarios, IoTDB supports plugin assembly in the above synchronisation task framework. Some common plugins are pre-built for you to use directly, and you can also customise sink plugins and load them into the IoTDB system for use.</p><table><thead><tr><th>Modules</th><th>Plugins</th><th>Pre-configured Plugins</th><th>Customised Plugins</th></tr></thead><tbody><tr><td>Extract (Source)</td><td>Source Plugin</td><td>iotdb-source</td><td>Not Supported</td></tr><tr><td>Send (Sink)</td><td>Sink plugin</td><td>iotdb-thrift-sink, iotdb-air-gap-sink</td><td>Support</td></tr></tbody></table><h4 id="preconfigured-plugins" tabindex="-1"><a class="header-anchor" href="#preconfigured-plugins"><span>Preconfigured Plugins</span></a></h4><p>The preset plugins are listed below:</p><table><thead><tr><th>Plugin Name</th><th>Type</th><th>Introduction</th><th>Available Versions</th></tr></thead><tbody><tr><td>iotdb-source</td><td>source plugin</td><td>Default source plugin for extracting IoTDB historical or real-time data</td><td>1.2.x</td></tr><tr><td>iotdb-thrift-sink</td><td>sink plugin</td><td>Used for data transfer between IoTDB (v1.2.0 and above) and IoTDB (v1.2.0 and above). Uses the Thrift RPC framework to transfer data, multi-threaded async non-blocking IO model, high transfer performance, especially for scenarios where the target is distributed</td><td>1.2.x</td></tr><tr><td>iotdb-air-gap-sink</td><td>sink plugin</td><td>Used for data synchronization from IoTDB (v1.2.2+) to IoTDB (v1.2.2+) across unidirectional data gates. Supported gate models include Nanrui Syskeeper 2000, etc.</td><td>1.2.2+</td></tr><tr><td>iotdb-thrift-ssl-sink</td><td>sink plugin</td><td>Used for data synchronization from IoTDB (v1.3.1+) to IoTDB (v1.2.0+). Uses the Thrift RPC framework to transfer data, single-thread blocking IO model.</td><td>1.3.1+</td></tr></tbody></table><p>Detailed parameters for each plugin can be found in the <a href="#sink-parameters">Parameter Description</a> section of this document.</p><h4 id="view-plugins" tabindex="-1"><a class="header-anchor" href="#view-plugins"><span>View Plugins</span></a></h4><p>To view the plugins in the system (including custom and built-in plugins) you can use the following statement:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>SHOW PIPEPLUGINS |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>The following results are returned:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>IoTDB> show pipeplugins |
| +------------------------------+----------+---------------------------------------------------------------------------------+---------+ |
| | PluginName|PluginType| ClassName|PluginJar| |
| +------------------------------+--------------------------------------------------------------------------------------------+---------+ |
| | DO-NOTHING-PROCESSOR| Builtin| org.apache.iotdb.commons.pipe.plugin.builtin.processor.DoNothingProcessor| | |
| | DO-NOTHING-SINK| Builtin| org.apache.iotdb.commons.pipe.plugin.builtin.sink.DoNothingSink| | |
| | IOTDB-AIR-GAP-SINK| Builtin| org.apache.iotdb.commons.pipe.plugin.builtin.sink.IoTDBAirGapSink| | |
| | IOTDB-SOURCE| Builtin| org.apache.iotdb.commons.pipe.plugin.builtin.source.IoTDBSOURCE| | |
| | IOTDB-THRIFT-SINK| Builtin| org.apache.iotdb.commons.pipe.plugin.builtin.sink.IoTDBThriftSink| | |
| |IOTDB-THRIFT-SSL-SINK(V1.3.1+)| Builtin|org.apache.iotdb.commons.pipe.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink| | |
| +------------------------------+----------+---------------------------------------------------------------------------------+---------+ |
| |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="use-examples" tabindex="-1"><a class="header-anchor" href="#use-examples"><span>Use examples</span></a></h2><h3 id="full-data-synchronisation" tabindex="-1"><a class="header-anchor" href="#full-data-synchronisation"><span>Full data synchronisation</span></a></h3><p>This example is used to demonstrate the synchronisation of all data from one IoTDB to another IoTDB with the data link as shown below:</p><figure><img src="https://alioss.timecho.com/upload/pipe1.jpg" alt="" tabindex="0" loading="lazy"><figcaption></figcaption></figure><p>In this example, we can create a synchronisation task named A2B to synchronise the full amount of data from IoTDB A to IoTDB B. Here we need to use the iotdb-thrift-sink plugin (built-in plugin) which uses sink, and we need to specify the address of the receiving end, in this example, we have specified 'sink.ip' and 'sink.port', and we can also specify 'sink.port'. This example specifies 'sink.ip' and 'sink.port', and also 'sink.node-urls', as in the following example statement:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-thrift-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6668' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="synchronising-historical-data" tabindex="-1"><a class="header-anchor" href="#synchronising-historical-data"><span>Synchronising historical data</span></a></h3><p>This example is used to demonstrate the synchronisation of data from a certain historical time range (8:00pm 23 August 2023 to 8:00pm 23 October 2023) to another IoTDB, the data link is shown below:</p><figure><img src="https://alioss.timecho.com/upload/pipe2.jpg" alt="" tabindex="0" loading="lazy"><figcaption></figcaption></figure><p>In this example we can create a synchronisation task called A2B. First of all, we need to define the range of data to be transferred in source, since the data to be transferred is historical data (historical data refers to the data that existed before the creation of the synchronisation task), we need to configure the source.realtime.enable parameter to false; at the same time, we need to configure the start-time and end-time of the data and the mode of the transfer. At the same time, you need to configure the start-time and end-time of the data and the mode of transmission, and it is recommended that the mode be set to hybrid mode (hybrid mode is a mixed transmission mode, which adopts the real-time transmission mode when there is no backlog of data, and adopts the batch transmission mode when there is a backlog of data, and automatically switches according to the internal situation of the system).</p><p>The detailed statements are as follows:</p><div class="language-SQL line-numbers-mode" data-ext="SQL" data-title="SQL"><pre class="language-SQL"><code>create pipe A2B |
| WITH SOURCE ( |
| 'source'= 'iotdb-source', |
| 'source.realtime.mode'='hybrid', |
| 'source.history.start-time' = '2023.08.23T08:00:00+00:00', |
| 'source.history.end-time' = '2023.10.23T08:00:00+00:00') |
| with SINK ( |
| 'sink'='iotdb-thrift-async-sink', |
| 'sink.node-urls'='xxxx:6668', |
| 'sink.batch.enable'='false') |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="bidirectional-data-transfer" tabindex="-1"><a class="header-anchor" href="#bidirectional-data-transfer"><span>Bidirectional data transfer</span></a></h3><p>This example is used to demonstrate a scenario where two IoTDBs are dual-active with each other, with the data link shown below:</p><figure><img src="https://alioss.timecho.com/upload/pipe3.jpg" alt="" tabindex="0" loading="lazy"><figcaption></figcaption></figure><p>In this example, in order to avoid an infinite loop of data, the parameter <code>'source.forwarding-pipe-requests</code> needs to be set to <code>false</code> on both A and B to indicate that the data transferred from the other pipe will not be forwarded. Also set <code>'source.history.enable'</code> to <code>false</code> to indicate that historical data is not transferred, i.e., data prior to the creation of the task is not synchronised.</p><p>The detailed statement is as follows:</p><p>Execute the following statements on A IoTDB:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>create pipe AB |
| with source ( |
| 'source.forwarding-pipe-requests' = 'false', |
| with sink ( |
| 'sink'='iotdb-thrift-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6668' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>Execute the following statements on B IoTDB:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>create pipe BA |
| with source ( |
| 'source.forwarding-pipe-requests' = 'false', |
| with sink ( |
| 'sink'='iotdb-thrift-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6667' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="cascading-data-transfer" tabindex="-1"><a class="header-anchor" href="#cascading-data-transfer"><span>Cascading Data Transfer</span></a></h3><p>This example is used to demonstrate a cascading data transfer scenario between multiple IoTDBs, where data is synchronised from cluster A to cluster B and then to cluster C. The data link is shown in the figure below:</p><figure><img src="https://alioss.timecho.com/upload/pipe4.jpg" alt="" tabindex="0" loading="lazy"><figcaption></figcaption></figure><p>In this example, in order to synchronise the data from cluster A to C, the pipe between BC needs to be configured with <code>source.forwarding-pipe-requests</code> to <code>true</code>, the detailed statement is as follows:</p><p>Execute the following statement on A IoTDB to synchronise data from A to B:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>create pipe AB |
| with sink ( |
| 'sink'='iotdb-thrift-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6668' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>Execute the following statement on B IoTDB to synchronise data in B to C:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>create pipe BC |
| with source ( |
| 'source.forwarding-pipe-requests' = 'true', |
| with sink ( |
| 'sink'='iotdb-thrift-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6669' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="transmission-of-data-through-an-air-gap" tabindex="-1"><a class="header-anchor" href="#transmission-of-data-through-an-air-gap"><span>Transmission of data through an air gap</span></a></h3><p>This example is used to demonstrate a scenario where data from one IoTDB is synchronised to another IoTDB via a unidirectional gate, with the data link shown below:</p><figure><img src="https://alioss.timecho.com/docs/img/1706698659207.jpg" alt="" tabindex="0" loading="lazy"><figcaption></figcaption></figure><p>In this example, you need to use the iotdb-air-gap-sink plugin in the sink task (currently supports some models of network gates, please contact the staff of Timecho Technology to confirm the specific model), and after configuring the network gate, execute the following statements on IoTDB A, where ip and port fill in the information of the network gate, and the detailed statements are as follows:</p><div class="language-Go line-numbers-mode" data-ext="Go" data-title="Go"><pre class="language-Go"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-air-gap-sink', |
| 'sink.ip'='10.53.53.53', |
| 'sink.port'='9780' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="transfer-data-using-ssl-protocol" tabindex="-1"><a class="header-anchor" href="#transfer-data-using-ssl-protocol"><span>Transfer data using SSL protocol</span></a></h3><p>This example demonstrates the scenario of configuring IoTDB one-way data synchronization using the SSL protocol, with the data link shown in the following figureοΌ</p><figure><img src="https://alioss.timecho.com/docs/img/1706696772065.jpg" alt="" tabindex="0" loading="lazy"><figcaption></figcaption></figure><p>In this scenario, it is necessary to use IoTDB's iotdb-thrift-ssl-sink plugin. We can create a synchronization task called A2B and configure the password and address of our own certificate. The detailed statement is as follows:</p><div class="language-Sql line-numbers-mode" data-ext="Sql" data-title="Sql"><pre class="language-Sql"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-thrift-ssl-sink', |
| 'sink.ip'='127.0.0.1', |
| 'sink.port'='6669', |
| 'ssl.trust-store-path'='pki/trusted' |
| 'ssl.trust-store-pwd'='root' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="reference-notes" tabindex="-1"><a class="header-anchor" href="#reference-notes"><span>Reference: Notes</span></a></h2><p>The IoTDB configuration file (iotdb-common.properties) can be modified in order to adjust the parameters for data synchronisation, such as the synchronisation data storage directory. The complete configuration is as follows:</p><p>V1.3.0+:</p><div class="language-Properties line-numbers-mode" data-ext="Properties" data-title="Properties"><pre class="language-Properties"><code>#################### |
| ### Pipe Configuration |
| #################### |
| |
| # Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\\\", or if its prefix is "\\\\\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_connector_timeout_ms=900000 |
| |
| # The maximum number of selectors that can be used in the async connector. |
| # pipe_async_connector_selector_number=1 |
| |
| # The core number of clients that can be used in the async connector. |
| # pipe_async_connector_core_client_number=8 |
| |
| # The maximum number of clients that can be used in the async connector. |
| # pipe_async_connector_max_client_number=16 |
| |
| # Whether to enable receiving pipe data through air gap. |
| # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. |
| # pipe_air_gap_receiver_enabled=false |
| |
| # The port for the server to receive pipe data through air gap. |
| # pipe_air_gap_receiver_port=9780 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>V1.3.1+:</p><div class="language-Properties line-numbers-mode" data-ext="Properties" data-title="Properties"><pre class="language-Properties"><code>#################### |
| ### Pipe Configuration |
| #################### |
| |
| # Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\\\", or if its prefix is "\\\\\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_sink_timeout_ms=900000 |
| |
| # Whether to enable receiving pipe data through air gap. |
| # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. |
| # pipe_air_gap_receiver_enabled=false |
| |
| # The port for the server to receive pipe data through air gap. |
| # pipe_air_gap_receiver_port=9780 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="reference-parameter-description" tabindex="-1"><a class="header-anchor" href="#reference-parameter-description"><span>Reference: parameter description</span></a></h2><p>π Notes: for version 1.3.1 or later, any parameters other than "source", "processor", "sink" themselves need not be with the prefixes. For instance:</p><div class="language-Sql line-numbers-mode" data-ext="Sql" data-title="Sql"><pre class="language-Sql"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-air-gap-sink', |
| 'sink.ip'='10.53.53.53', |
| 'sink.port'='9780' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>can be written as</p><div class="language-Sql line-numbers-mode" data-ext="Sql" data-title="Sql"><pre class="language-Sql"><code>create pipe A2B |
| with sink ( |
| 'sink'='iotdb-air-gap-sink', |
| 'ip'='10.53.53.53', |
| 'port'='9780' |
| ) |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="source-parameter" tabindex="-1"><a class="header-anchor" href="#source-parameter"><span>source parameter</span></a></h3><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or not</th><th>default value</th></tr></thead><tbody><tr><td>source</td><td>iotdb-source</td><td>String: iotdb-source</td><td>required</td><td>-</td></tr><tr><td>source.pattern</td><td>Path prefix for filtering time series</td><td>String: any time series prefix</td><td>optional</td><td>root</td></tr><tr><td>source.history.start-time</td><td>Synchronise the start event time of historical data, including start-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional</td><td>Long.MIN_VALUE</td></tr><tr><td>source.history.end-time</td><td>end event time for synchronised history data, contains end-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional</td><td>Long.MAX_VALUE</td></tr><tr><td>start-time(V1.3.1+)</td><td>Synchronise the start event time of all data, including start-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional</td><td>Long.MIN_VALUE</td></tr><tr><td>end-time(V1.3.1+)</td><td>end event time for synchronised all data, contains end-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional</td><td>Long.MAX_VALUE</td></tr><tr><td>source.realtime.mode</td><td>Extraction mode for real-time data</td><td>String: hybrid, stream, batch</td><td>optional</td><td>hybrid</td></tr><tr><td>source.forwarding-pipe-requests</td><td>Whether to forward data written by another Pipe (usually Data Sync)</td><td>Boolean: true, false</td><td>optional</td><td>true</td></tr></tbody></table><blockquote><p>π <strong>Note: Difference between historical and real-time data</strong></p><ul><li><strong>Historical data</strong>: all data with arrival time < current system time when the pipe was created is called historical data</li><li><strong>Real-time data</strong>: All data with arrival time >= current system time when the pipe was created is called real-time data.</li><li><strong>Full data</strong>: full data = historical data + real time data</li></ul></blockquote><blockquote><p>π <strong>Explanation: Difference between data extraction modes hybrid, stream and batch</strong></p><ul><li><strong>hybrid (recommended)</strong>: In this mode, the task will give priority to real-time processing and sending of data, and automatically switch to batch sending mode when data backlog occurs, which is characterised by a balance between timeliness of data synchronisation and throughput</li><li><strong>stream</strong>: In this mode, the task will process and send data in real time, which is characterised by high timeliness and low throughput.</li><li><strong>batch</strong>: In this mode, the task will process and send data in batch (by underlying data file), which is characterised by low latency and high throughput.</li></ul></blockquote><h3 id="sink-parameters" tabindex="-1"><a class="header-anchor" href="#sink-parameters"><span>sink parameters</span></a></h3><h4 id="iotdb-thrift-sink" tabindex="-1"><a class="header-anchor" href="#iotdb-thrift-sink"><span>iotdb-thrift-sink</span></a></h4><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or not</th><th>default value</th></tr></thead><tbody><tr><td>sink</td><td>iotdb-thrift-sink or iotdb-thrift-async-sink</td><td>String: iotdb-thrift-sink or iotdb-thrift-async-sink</td><td>required</td><td></td></tr><tr><td>sink.ip</td><td>Data service IP of a DataNode in the target IoTDB (note that the synchronisation task does not support forwarding to its own service)</td><td>String</td><td>Optional</td><td>Fill in either sink.node-urls</td></tr><tr><td>sink.port</td><td>Data service port of a DataNode in the target IoTDB (note that the synchronisation task does not support forwarding to its own service)</td><td>Integer</td><td>Optional</td><td>Fill in either sink.node-urls</td></tr><tr><td>sink.node-urls</td><td>The url of the data service port of any number of DataNodes on the target IoTDB (note that the synchronisation task does not support forwarding to its own service)</td><td>String. Example: '127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>Optional</td><td>Fill in either sink.ip:sink.port</td></tr><tr><td>sink.batch.enable</td><td>Whether to enable the log saving wholesale delivery mode, which is used to improve transmission throughput and reduce IOPS</td><td>Boolean: true, false</td><td>Optional</td><td>true</td></tr><tr><td>sink.batch.max-delay-seconds</td><td>Effective when the log save and send mode is turned on, indicates the longest time a batch of data waits before being sent (unit: s)</td><td>Integer</td><td>Optional</td><td>1</td></tr><tr><td>sink.batch.size-bytes</td><td>Effective when log saving and delivery mode is enabled, indicates the maximum saving size of a batch of data (unit: byte)</td><td>Long</td><td>Optional</td><td></td></tr></tbody></table><h4 id="iotdb-air-gap-sink" tabindex="-1"><a class="header-anchor" href="#iotdb-air-gap-sink"><span>iotdb-air-gap-sink</span></a></h4><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or not</th><th>default value</th></tr></thead><tbody><tr><td>sink</td><td>iotdb-air-gap-sink</td><td>String: iotdb-air-gap-sink</td><td>required</td><td></td></tr><tr><td>sink.ip</td><td>Data service IP of a DataNode in the target IoTDB</td><td>String</td><td>Optional</td><td>Fill in either sink.node-urls</td></tr><tr><td>sink.port</td><td>Data service port of a DataNode in the target IoTDB</td><td>Integer</td><td>Optional</td><td>Fill in either sink.node-urls</td></tr><tr><td>sink.node-urls</td><td>URL of the data service port of any multiple DataNodes on the target</td><td>String.Example: '127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>Optional</td><td>Fill in either sink.ip:sink.port</td></tr><tr><td>sink.air-gap.handshake-timeout-ms</td><td>The timeout length of the handshake request when the sender and the receiver try to establish a connection for the first time, unit: milliseconds</td><td>Integer</td><td>Optional</td><td>5000</td></tr></tbody></table><h4 id="iotdb-thrift-ssl-sink-v1-3-1" tabindex="-1"><a class="header-anchor" href="#iotdb-thrift-ssl-sink-v1-3-1"><span>iotdb-thrift-ssl-sink(V1.3.1+)</span></a></h4><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or not</th><th>default value</th></tr></thead><tbody><tr><td>sink</td><td>iotdb-thrift-sink or iotdb-thrift-async-sink</td><td>String: iotdb-thrift-sink or iotdb-thrift-sync-sink</td><td>required</td><td></td></tr><tr><td>sink.ip</td><td>Data service IP of a DataNode in the target IoTDB (note that the synchronisation task does not support forwarding to its own service)</td><td>String</td><td>Optional</td><td>Fill in either sink.node-urls</td></tr><tr><td>sink.port</td><td>Data service port of a DataNode in the target IoTDB (note that the synchronisation task does not support forwarding to its own service)</td><td>Integer</td><td>Optional</td><td>Fill in either sink.node-urls</td></tr><tr><td>sink.node-urls</td><td>The url of the data service port of any number of DataNodes on the target IoTDB (note that the synchronisation task does not support forwarding to its own service)</td><td>String. Example: '127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>Optional</td><td>Fill in either sink.ip:sink.port</td></tr><tr><td>sink.batch.enable</td><td>Whether to enable the log saving wholesale delivery mode, which is used to improve transmission throughput and reduce IOPS</td><td>Boolean: true, false</td><td>Optional</td><td>true</td></tr><tr><td>sink.batch.max-delay-seconds</td><td>Effective when the log save and send mode is turned on, indicates the longest time a batch of data waits before being sent (unit: s)</td><td>Integer</td><td>Optional</td><td>1</td></tr><tr><td>sink.batch.size-bytes</td><td>Effective when log saving and delivery mode is enabled, indicates the maximum saving size of a batch of data (unit: byte)</td><td>Long</td><td>Optional</td><td></td></tr><tr><td>ssl.trust-store-path</td><td>The certificate trust store path to connect to the target DataNodes</td><td>String.Example: '127.0.0.1:6667,127.0.0.1:6668,127.0.0.1:6669', '127.0.0.1:6667'</td><td>Optional</td><td>Fill in either sink.ip:sink.port</td></tr><tr><td>ssl.trust-store-pwd</td><td>The certificate trust store password to connect to the target DataNodes</td><td>Integer</td><td>Optional</td><td>5000</td></tr></tbody></table>`,106),d=[s];function r(o,l){return t(),i("div",null,d)}const h=e(n,[["render",r],["__file","Data-Sync_timecho.html.vue"]]),u=JSON.parse('{"path":"/UserGuide/latest/User-Manual/Data-Sync_timecho.html","title":"Data synchronisation","lang":"en-US","frontmatter":{"description":"Data synchronisation Data synchronisation is a typical requirement of industrial IoT. Through the data synchronisation mechanism, data sharing between IoTDBs can be achieved, an...","head":[["link",{"rel":"alternate","hreflang":"zh-cn","href":"https://iotdb.apache.org/zh/UserGuide/latest/User-Manual/Data-Sync_timecho.html"}],["meta",{"property":"og:url","content":"https://iotdb.apache.org/UserGuide/latest/User-Manual/Data-Sync_timecho.html"}],["meta",{"property":"og:site_name","content":"IoTDB Website"}],["meta",{"property":"og:title","content":"Data synchronisation"}],["meta",{"property":"og:description","content":"Data synchronisation Data synchronisation is a typical requirement of industrial IoT. Through the data synchronisation mechanism, data sharing between IoTDBs can be achieved, an..."}],["meta",{"property":"og:type","content":"article"}],["meta",{"property":"og:image","content":"https://alioss.timecho.com/docs/img/%E7%8A%B6%E6%80%81%E8%BF%81%E7%A7%BB%E5%9B%BE.png"}],["meta",{"property":"og:locale","content":"en-US"}],["meta",{"property":"og:locale:alternate","content":"zh-CN"}],["meta",{"property":"og:updated_time","content":"2024-04-08T07:45:44.000Z"}],["meta",{"property":"article:modified_time","content":"2024-04-08T07:45:44.000Z"}],["script",{"type":"application/ld+json"},"{\\"@context\\":\\"https://schema.org\\",\\"@type\\":\\"Article\\",\\"headline\\":\\"Data synchronisation\\",\\"image\\":[\\"https://alioss.timecho.com/docs/img/%E7%8A%B6%E6%80%81%E8%BF%81%E7%A7%BB%E5%9B%BE.png\\",\\"https://alioss.timecho.com/upload/pipe1.jpg\\",\\"https://alioss.timecho.com/upload/pipe2.jpg\\",\\"https://alioss.timecho.com/upload/pipe3.jpg\\",\\"https://alioss.timecho.com/upload/pipe4.jpg\\",\\"https://alioss.timecho.com/docs/img/1706698659207.jpg\\",\\"https://alioss.timecho.com/docs/img/1706696772065.jpg\\"],\\"dateModified\\":\\"2024-04-08T07:45:44.000Z\\",\\"author\\":[]}"]]},"headers":[{"level":2,"title":"Introduction","slug":"introduction","link":"#introduction","children":[{"level":3,"title":"Synchronisation Task Overview","slug":"synchronisation-task-overview","link":"#synchronisation-task-overview","children":[]},{"level":3,"title":"Synchronisation Task - Create","slug":"synchronisation-task-create","link":"#synchronisation-task-create","children":[]},{"level":3,"title":"Synchronisation Tasks - Management","slug":"synchronisation-tasks-management","link":"#synchronisation-tasks-management","children":[]},{"level":3,"title":"Plugin","slug":"plugin","link":"#plugin","children":[]}]},{"level":2,"title":"Use examples","slug":"use-examples","link":"#use-examples","children":[{"level":3,"title":"Full data synchronisation","slug":"full-data-synchronisation","link":"#full-data-synchronisation","children":[]},{"level":3,"title":"Synchronising historical data","slug":"synchronising-historical-data","link":"#synchronising-historical-data","children":[]},{"level":3,"title":"Bidirectional data transfer","slug":"bidirectional-data-transfer","link":"#bidirectional-data-transfer","children":[]},{"level":3,"title":"Cascading Data Transfer","slug":"cascading-data-transfer","link":"#cascading-data-transfer","children":[]},{"level":3,"title":"Transmission of data through an air gap","slug":"transmission-of-data-through-an-air-gap","link":"#transmission-of-data-through-an-air-gap","children":[]},{"level":3,"title":"Transfer data using SSL protocol","slug":"transfer-data-using-ssl-protocol","link":"#transfer-data-using-ssl-protocol","children":[]}]},{"level":2,"title":"Reference: Notes","slug":"reference-notes","link":"#reference-notes","children":[]},{"level":2,"title":"Reference: parameter description","slug":"reference-parameter-description","link":"#reference-parameter-description","children":[{"level":3,"title":"source parameter","slug":"source-parameter","link":"#source-parameter","children":[]},{"level":3,"title":"sink parameters","slug":"sink-parameters","link":"#sink-parameters","children":[]}]}],"git":{"createdTime":1704854860000,"updatedTime":1712562344000,"contributors":[{"name":"Caideyipi","email":"87789683+Caideyipi@users.noreply.github.com","commits":1},{"name":"CritasWang","email":"critas@outlook.com","commits":1}]},"readingTime":{"minutes":10.4,"words":3121},"filePathRelative":"UserGuide/latest/User-Manual/Data-Sync_timecho.md","localizedDate":"January 10, 2024","autoDesc":true}');export{h as comp,u as data}; |
