| import{_ as l,r as p,o as c,c as r,b as n,d as s,a,w as t,e as i}from"./app-Bp5kEZWW.js";const d={},u=n("h1",{id:"iotdb-stream-processing-framework",tabindex:"-1"},[n("a",{class:"header-anchor",href:"#iotdb-stream-processing-framework"},[n("span",null,"IoTDB stream processing framework")])],-1),m=n("p",null,"The IoTDB stream processing framework allows users to implement customized stream processing logic, which can monitor and capture storage engine changes, transform changed data, and push transformed data outward.",-1),k=i(`<ul><li>Source task</li><li>Processor task</li><li>Sink task</li></ul><p>The stream processing framework allows users to customize the processing logic of three subtasks using Java language and process data in a UDF-like manner.<br> In a Pipe, the above three subtasks are executed by three plugins respectively, and the data will be processed by these three plugins in turn:<br> Pipe Source is used to extract data, Pipe Processor is used to process data, Pipe Sink is used to send data, and the final data will be sent to an external system.</p><p><strong>The model of the Pipe task is as follows:</strong></p><figure><img src="https://alioss.timecho.com/docs/img/1706778988482.jpg" alt="pipe.png" tabindex="0" loading="lazy"><figcaption>pipe.png</figcaption></figure><p>Describing a data flow processing task essentially describes the properties of Pipe Source, Pipe Processor and Pipe Sink plugins.<br> Users can declaratively configure the specific attributes of the three subtasks through SQL statements, and achieve flexible data ETL capabilities by combining different attributes.</p><p>Using the stream processing framework, a complete data link can be built to meet the needs of end-side-cloud synchronization, off-site disaster recovery, and read-write load sub-library*.</p><h2 id="custom-stream-processing-plugin-development" tabindex="-1"><a class="header-anchor" href="#custom-stream-processing-plugin-development"><span>Custom stream processing plugin development</span></a></h2><h3 id="programming-development-dependencies" tabindex="-1"><a class="header-anchor" href="#programming-development-dependencies"><span>Programming development dependencies</span></a></h3><p>It is recommended to use maven to build the project and add the following dependencies in <code>pom.xml</code>. Please be careful to select the same dependency version as the IoTDB server version.</p><div class="language-xml line-numbers-mode" data-ext="xml" data-title="xml"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span> |
| <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>org.apache.iotdb<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span> |
| <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>pipe-api<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span> |
| <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.3.1<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span> |
| <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>scope</span><span class="token punctuation">></span></span>provided<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>scope</span><span class="token punctuation">></span></span> |
| <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="event-driven-programming-model" tabindex="-1"><a class="header-anchor" href="#event-driven-programming-model"><span>Event-driven programming model</span></a></h3><p>The user programming interface design of the stream processing plugin refers to the general design concept of the event-driven programming model. Events are data abstractions in the user programming interface, and the programming interface is decoupled from the specific execution method. It only needs to focus on describing the processing method expected by the system after the event (data) reaches the system.</p><p>In the user programming interface of the stream processing plugin, events are an abstraction of database data writing operations. The event is captured by the stand-alone stream processing engine, and is passed to the PipeSource plugin, PipeProcessor plugin, and PipeSink plugin in sequence according to the three-stage stream processing process, and triggers the execution of user logic in the three plugins in turn.</p><p>In order to take into account the low latency of stream processing in low load scenarios on the end side and the high throughput of stream processing in high load scenarios on the end side, the stream processing engine will dynamically select processing objects in the operation logs and data files. Therefore, user programming of stream processing The interface requires users to provide processing logic for the following two types of events: operation log writing event TabletInsertionEvent and data file writing event TsFileInsertionEvent.</p><h4 id="operation-log-writing-event-tabletinsertionevent" tabindex="-1"><a class="header-anchor" href="#operation-log-writing-event-tabletinsertionevent"><span><strong>Operation log writing event (TabletInsertionEvent)</strong></span></a></h4><p>The operation log write event (TabletInsertionEvent) is a high-level data abstraction for user write requests. It provides users with the ability to manipulate the underlying data of write requests by providing a unified operation interface.</p><p>For different database deployment methods, the underlying storage structures corresponding to operation log writing events are different. For stand-alone deployment scenarios, the operation log writing event is an encapsulation of write-ahead log (WAL) entries; for a distributed deployment scenario, the operation log writing event is an encapsulation of a single node consensus protocol operation log entry.</p><p>For write operations generated by different write request interfaces in the database, the data structure of the request structure corresponding to the operation log write event is also different. IoTDB provides numerous writing interfaces such as InsertRecord, InsertRecords, InsertTablet, InsertTablets, etc. Each writing request uses a completely different serialization method, and the generated binary entries are also different.</p><p>The existence of operation log writing events provides users with a unified view of data operations, which shields the implementation differences of the underlying data structure, greatly reduces the user's programming threshold, and improves the ease of use of the function.</p><div class="language-java line-numbers-mode" data-ext="java" data-title="java"><pre class="language-java"><code><span class="token doc-comment comment">/** TabletInsertionEvent is used to define the event of data insertion. */</span> |
| <span class="token keyword">public</span> <span class="token keyword">interface</span> <span class="token class-name">TabletInsertionEvent</span> <span class="token keyword">extends</span> <span class="token class-name">Event</span> <span class="token punctuation">{</span> |
| |
| <span class="token doc-comment comment">/** |
| * The consumer processes the data row by row and collects the results by RowCollector. |
| * |
| * <span class="token keyword">@return</span> <span class="token punctuation">{</span><span class="token keyword">@code</span> <span class="token code-section"><span class="token code language-java"><span class="token class-name">Iterable</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">></span></span></span></span><span class="token punctuation">}</span> a list of new TabletInsertionEvent contains the |
| * results collected by the RowCollector |
| */</span> |
| <span class="token class-name">Iterable</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">></span></span> <span class="token function">processRowByRow</span><span class="token punctuation">(</span><span class="token class-name">BiConsumer</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">Row</span><span class="token punctuation">,</span> <span class="token class-name">RowCollector</span><span class="token punctuation">></span></span> consumer<span class="token punctuation">)</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * The consumer processes the Tablet directly and collects the results by RowCollector. |
| * |
| * <span class="token keyword">@return</span> <span class="token punctuation">{</span><span class="token keyword">@code</span> <span class="token code-section"><span class="token code language-java"><span class="token class-name">Iterable</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">></span></span></span></span><span class="token punctuation">}</span> a list of new TabletInsertionEvent contains the |
| * results collected by the RowCollector |
| */</span> |
| <span class="token class-name">Iterable</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">></span></span> <span class="token function">processTablet</span><span class="token punctuation">(</span><span class="token class-name">BiConsumer</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">Tablet</span><span class="token punctuation">,</span> <span class="token class-name">RowCollector</span><span class="token punctuation">></span></span> consumer<span class="token punctuation">)</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h4 id="data-file-writing-event-tsfileinsertionevent" tabindex="-1"><a class="header-anchor" href="#data-file-writing-event-tsfileinsertionevent"><span><strong>Data file writing event (TsFileInsertionEvent)</strong></span></a></h4><p>The data file writing event (TsFileInsertionEvent) is a high-level abstraction of the database file writing operation. It is a data collection of several operation log writing events (TabletInsertionEvent).</p><p>The storage engine of IoTDB is LSM structured. When data is written, the writing operation will first be placed into a log-structured file, and the written data will be stored in the memory at the same time. When the memory reaches the control upper limit, the disk flushing behavior will be triggered, that is, the data in the memory will be converted into a database file, and the previously prewritten operation log will be deleted. When the data in the memory is converted into the data in the database file, it will undergo two compression processes: encoding compression and general compression. Therefore, the data in the database file takes up less space than the original data in the memory.</p><p>In extreme network conditions, directly transmitting data files is more economical than transmitting data writing operations. It will occupy lower network bandwidth and achieve faster transmission speeds. Of course, there is no free lunch. Computing and processing data in files requires additional file I/O costs compared to directly computing and processing data in memory. However, it is precisely the existence of two structures, disk data files and memory write operations, with their own advantages and disadvantages, that gives the system the opportunity to make dynamic trade-offs and adjustments. It is based on this observation that data files are introduced into the plugin's event model. Write event.</p><p>To sum up, the data file writing event appears in the event stream of the stream processing plugin, and there are two situations:</p><p>(1) Historical data extraction: Before a stream processing task starts, all written data that has been placed on the disk will exist in the form of TsFile. After a stream processing task starts, when collecting historical data, the historical data will be abstracted using TsFileInsertionEvent;</p><p>(2) Real-time data extraction: When a stream processing task is in progress, when the real-time processing speed of operation log write events in the data stream is slower than the write request speed, after a certain progress, the operation log write events that cannot be processed in the future will be persisted. to disk and exists in the form of TsFile. After this data is extracted by the stream processing engine, TsFileInsertionEvent will be used as an abstraction.</p><div class="language-java line-numbers-mode" data-ext="java" data-title="java"><pre class="language-java"><code><span class="token doc-comment comment">/** |
| * TsFileInsertionEvent is used to define the event of writing TsFile. Event data stores in disks, |
| * which is compressed and encoded, and requires IO cost for computational processing. |
| */</span> |
| <span class="token keyword">public</span> <span class="token keyword">interface</span> <span class="token class-name">TsFileInsertionEvent</span> <span class="token keyword">extends</span> <span class="token class-name">Event</span> <span class="token punctuation">{</span> |
| |
| <span class="token doc-comment comment">/** |
| * The method is used to convert the TsFileInsertionEvent into several TabletInsertionEvents. |
| * |
| * <span class="token keyword">@return</span> <span class="token punctuation">{</span><span class="token keyword">@code</span> <span class="token code-section"><span class="token code language-java"><span class="token class-name">Iterable</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">></span></span></span></span><span class="token punctuation">}</span> the list of TabletInsertionEvent |
| */</span> |
| <span class="token class-name">Iterable</span><span class="token generics"><span class="token punctuation"><</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">></span></span> <span class="token function">toTabletInsertionEvents</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="custom-stream-processing-plugin-programming-interface-definition" tabindex="-1"><a class="header-anchor" href="#custom-stream-processing-plugin-programming-interface-definition"><span>Custom stream processing plugin programming interface definition</span></a></h3><p>Based on the custom stream processing plugin programming interface, users can easily write data extraction plugins, data processing plugins and data sending plugins, so that the stream processing function can be flexibly adapted to various industrial scenarios.</p><h4 id="data-extraction-plugin-interface" tabindex="-1"><a class="header-anchor" href="#data-extraction-plugin-interface"><span>Data extraction plugin interface</span></a></h4><p>Data extraction is the first stage of the three stages of stream processing data from data extraction to data sending. The data extraction plugin (PipeSource) is the bridge between the stream processing engine and the storage engine. It monitors the behavior of the storage engine,<br> Capture various data write events.</p><div class="language-java line-numbers-mode" data-ext="java" data-title="java"><pre class="language-java"><code><span class="token doc-comment comment">/** |
| * PipeSource |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>PipeSource is responsible for capturing events from sources. |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>Various data sources can be supported by implementing different PipeSource classes. |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>The lifecycle of a PipeSource is as follows: |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>When a collaboration task is created, the KV pairs of \`WITH Source\` clause in SQL are |
| * parsed and the validation method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will |
| * be called to validate the parameters. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Before the collaboration task starts, the method <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeSourceRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called to |
| * configure the runtime behavior of the PipeSource. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Then the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">start</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called to start the PipeSource. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>While the collaboration task is in progress, the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">supply</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be |
| * called to capture events from sources and then the events will be passed to the |
| * PipeProcessor. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>The method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">close</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called when the collaboration task is |
| * cancelled (the \`DROP PIPE\` command is executed). |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| */</span> |
| <span class="token keyword">public</span> <span class="token keyword">interface</span> <span class="token class-name">PipeSource</span> <span class="token keyword">extends</span> <span class="token class-name">PipePlugin</span> <span class="token punctuation">{</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is mainly used to validate <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeParameters</span></span><span class="token punctuation">}</span> and it is executed before <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeSourceRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">validator</span> the validator used to validate <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeParameters</span></span><span class="token punctuation">}</span> |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> if any parameter is not valid |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span> validator<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is mainly used to customize PipeSource. In this method, the user can do the |
| * following things: |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Use PipeParameters to parse key-value pair attributes entered by the user. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Set the running configurations in PipeSourceRuntimeConfiguration. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>This method is called after the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> |
| * is called. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">parameters</span> used to parse the input parameters entered by the user |
| * <span class="token keyword">@param</span> <span class="token parameter">configuration</span> used to set the required properties of the running PipeSource |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span> parameters<span class="token punctuation">,</span> <span class="token class-name">PipeSourceRuntimeConfiguration</span> configuration<span class="token punctuation">)</span> |
| <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * Start the Source. After this method is called, events should be ready to be supplied by |
| * <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">supply</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span>. This method is called after <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeSourceRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called. |
| * |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">start</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * Supply single event from the Source and the caller will send the event to the processor. |
| * This method is called after <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSource</span><span class="token punctuation">#</span><span class="token function">start</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called. |
| * |
| * <span class="token keyword">@return</span> the event to be supplied. the event may be null if the Source has no more events at |
| * the moment, but the Source is still running for more events. |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token class-name">Event</span> <span class="token function">supply</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h4 id="data-processing-plugin-interface" tabindex="-1"><a class="header-anchor" href="#data-processing-plugin-interface"><span>Data processing plugin interface</span></a></h4><p>Data processing is the second stage of the three stages of stream processing data from data extraction to data sending. The data processing plugin (PipeProcessor) is mainly used to filter and transform the data captured by the data extraction plugin (PipeSource).<br> various events.</p><div class="language-java line-numbers-mode" data-ext="java" data-title="java"><pre class="language-java"><code><span class="token doc-comment comment">/** |
| * PipeProcessor |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>PipeProcessor is used to filter and transform the Event formed by the PipeSource. |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>The lifecycle of a PipeProcessor is as follows: |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>When a collaboration task is created, the KV pairs of \`WITH PROCESSOR\` clause in SQL are |
| * parsed and the validation method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> |
| * will be called to validate the parameters. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Before the collaboration task starts, the method <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeProcessorRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called |
| * to configure the runtime behavior of the PipeProcessor. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>While the collaboration task is in progress: |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>PipeSource captures the events and wraps them into three types of Event instances. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>PipeProcessor processes the event and then passes them to the PipeSource. The |
| * following 3 methods will be called: <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">process</span><span class="token punctuation">(</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">,</span> <span class="token class-name">EventCollector</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span>, <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">process</span><span class="token punctuation">(</span><span class="token class-name">TsFileInsertionEvent</span><span class="token punctuation">,</span> <span class="token class-name">EventCollector</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> and <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">process</span><span class="token punctuation">(</span><span class="token class-name">Event</span><span class="token punctuation">,</span> <span class="token class-name">EventCollector</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span>. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>PipeSink serializes the events into binaries and send them to sinks. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>When the collaboration task is cancelled (the \`DROP PIPE\` command is executed), the <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">close</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> <span class="token punctuation">}</span> method will be called. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| */</span> |
| <span class="token keyword">public</span> <span class="token keyword">interface</span> <span class="token class-name">PipeProcessor</span> <span class="token keyword">extends</span> <span class="token class-name">PipePlugin</span> <span class="token punctuation">{</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is mainly used to validate <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeParameters</span></span><span class="token punctuation">}</span> and it is executed before <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeProcessorRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">validator</span> the validator used to validate <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeParameters</span></span><span class="token punctuation">}</span> |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> if any parameter is not valid |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span> validator<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is mainly used to customize PipeProcessor. In this method, the user can do the |
| * following things: |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Use PipeParameters to parse key-value pair attributes entered by the user. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Set the running configurations in PipeProcessorRuntimeConfiguration. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>This method is called after the method <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeProcessor</span><span class="token punctuation">#</span><span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called and before the beginning of the |
| * events processing. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">parameters</span> used to parse the input parameters entered by the user |
| * <span class="token keyword">@param</span> <span class="token parameter">configuration</span> used to set the required properties of the running PipeProcessor |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span> parameters<span class="token punctuation">,</span> <span class="token class-name">PipeProcessorRuntimeConfiguration</span> configuration<span class="token punctuation">)</span> |
| <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is called to process the TabletInsertionEvent. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">tabletInsertionEvent</span> TabletInsertionEvent to be processed |
| * <span class="token keyword">@param</span> <span class="token parameter">eventCollector</span> used to collect result events after processing |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">process</span><span class="token punctuation">(</span><span class="token class-name">TabletInsertionEvent</span> tabletInsertionEvent<span class="token punctuation">,</span> <span class="token class-name">EventCollector</span> eventCollector<span class="token punctuation">)</span> |
| <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is called to process the TsFileInsertionEvent. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">tsFileInsertionEvent</span> TsFileInsertionEvent to be processed |
| * <span class="token keyword">@param</span> <span class="token parameter">eventCollector</span> used to collect result events after processing |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">default</span> <span class="token keyword">void</span> <span class="token function">process</span><span class="token punctuation">(</span><span class="token class-name">TsFileInsertionEvent</span> tsFileInsertionEvent<span class="token punctuation">,</span> <span class="token class-name">EventCollector</span> eventCollector<span class="token punctuation">)</span> |
| <span class="token keyword">throws</span> <span class="token class-name">Exception</span> <span class="token punctuation">{</span> |
| <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">final</span> <span class="token class-name">TabletInsertionEvent</span> tabletInsertionEvent <span class="token operator">:</span> |
| tsFileInsertionEvent<span class="token punctuation">.</span><span class="token function">toTabletInsertionEvents</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span> |
| <span class="token function">process</span><span class="token punctuation">(</span>tabletInsertionEvent<span class="token punctuation">,</span> eventCollector<span class="token punctuation">)</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| <span class="token punctuation">}</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is called to process the Event. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">event</span> Event to be processed |
| * <span class="token keyword">@param</span> <span class="token parameter">eventCollector</span> used to collect result events after processing |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">process</span><span class="token punctuation">(</span><span class="token class-name">Event</span> event<span class="token punctuation">,</span> <span class="token class-name">EventCollector</span> eventCollector<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h4 id="data-sending-plugin-interface" tabindex="-1"><a class="header-anchor" href="#data-sending-plugin-interface"><span>Data sending plugin interface</span></a></h4><p>Data sending is the third stage of the three stages of stream processing data from data extraction to data sending. The data sending plugin (PipeSink) is mainly used to send data processed by the data processing plugin (PipeProcessor).<br> Various events, it serves as the network implementation layer of the stream processing framework, and the interface should allow access to multiple real-time communication protocols and multiple sinks.</p><div class="language-java line-numbers-mode" data-ext="java" data-title="java"><pre class="language-java"><code><span class="token doc-comment comment">/** |
| * PipeSink |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>PipeSink is responsible for sending events to sinks. |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>Various network protocols can be supported by implementing different PipeSink classes. |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>The lifecycle of a PipeSink is as follows: |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>When a collaboration task is created, the KV pairs of \`WITH SINK\` clause in SQL are |
| * parsed and the validation method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be |
| * called to validate the parameters. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Before the collaboration task starts, the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> |
| * <span class="token class-name">PipeSinkRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called to configure the runtime behavior of the |
| * PipeSink and the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">handshake</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called to create a connection |
| * with sink. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>While the collaboration task is in progress: |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>PipeSource captures the events and wraps them into three types of Event instances. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>PipeProcessor processes the event and then passes them to the PipeSink. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>PipeSink serializes the events into binaries and send them to sinks. The following 3 |
| * methods will be called: <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">transfer</span><span class="token punctuation">(</span><span class="token class-name">TabletInsertionEvent</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span>, <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">transfer</span><span class="token punctuation">(</span><span class="token class-name">TsFileInsertionEvent</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> and <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">transfer</span><span class="token punctuation">(</span><span class="token class-name">Event</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span>. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>When the collaboration task is cancelled (the \`DROP PIPE\` command is executed), the <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">close</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> <span class="token punctuation">}</span> method will be called. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>In addition, the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">heartbeat</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be called periodically to check |
| * whether the connection with sink is still alive. The method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">handshake</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> will be |
| * called to create a new connection with the sink when the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">heartbeat</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> |
| * throws exceptions. |
| */</span> |
| <span class="token keyword">public</span> <span class="token keyword">interface</span> <span class="token class-name">PipeSink</span> <span class="token keyword">extends</span> <span class="token class-name">PipePlugin</span> <span class="token punctuation">{</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is mainly used to validate <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeParameters</span></span><span class="token punctuation">}</span> and it is executed before <span class="token punctuation">{</span><span class="token keyword">@link</span> |
| * <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeSinkRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">validator</span> the validator used to validate <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeParameters</span></span><span class="token punctuation">}</span> |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> if any parameter is not valid |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span> validator<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is mainly used to customize PipeSink. In this method, the user can do the following |
| * things: |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>ul</span><span class="token punctuation">></span></span> |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Use PipeParameters to parse key-value pair attributes entered by the user. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>li</span><span class="token punctuation">></span></span>Set the running configurations in PipeSinkRuntimeConfiguration. |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>ul</span><span class="token punctuation">></span></span> |
| * |
| * <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>p</span><span class="token punctuation">></span></span>This method is called after the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">validate</span><span class="token punctuation">(</span><span class="token class-name">PipeParameterValidator</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is |
| * called and before the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">handshake</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">parameters</span> used to parse the input parameters entered by the user |
| * <span class="token keyword">@param</span> <span class="token parameter">configuration</span> used to set the required properties of the running PipeSink |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span> parameters<span class="token punctuation">,</span> <span class="token class-name">PipeSinkRuntimeConfiguration</span> configuration<span class="token punctuation">)</span> |
| <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is used to create a connection with sink. This method will be called after the |
| * method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">customize</span><span class="token punctuation">(</span><span class="token class-name">PipeParameters</span><span class="token punctuation">,</span> <span class="token class-name">PipeSinkRuntimeConfiguration</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> is called or |
| * will be called when the method <span class="token punctuation">{</span><span class="token keyword">@link</span> <span class="token reference"><span class="token class-name">PipeSink</span><span class="token punctuation">#</span><span class="token function">heartbeat</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span><span class="token punctuation">}</span> throws exceptions. |
| * |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> if the connection is failed to be created |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">handshake</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method will be called periodically to check whether the connection with sink is still |
| * alive. |
| * |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> if the connection dies |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">heartbeat</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is used to transfer the TabletInsertionEvent. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">tabletInsertionEvent</span> TabletInsertionEvent to be transferred |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">PipeConnectionException</span></span> if the connection is broken |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">transfer</span><span class="token punctuation">(</span><span class="token class-name">TabletInsertionEvent</span> tabletInsertionEvent<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is used to transfer the TsFileInsertionEvent. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">tsFileInsertionEvent</span> TsFileInsertionEvent to be transferred |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">PipeConnectionException</span></span> if the connection is broken |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">default</span> <span class="token keyword">void</span> <span class="token function">transfer</span><span class="token punctuation">(</span><span class="token class-name">TsFileInsertionEvent</span> tsFileInsertionEvent<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span> <span class="token punctuation">{</span> |
| <span class="token keyword">try</span> <span class="token punctuation">{</span> |
| <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">final</span> <span class="token class-name">TabletInsertionEvent</span> tabletInsertionEvent <span class="token operator">:</span> |
| tsFileInsertionEvent<span class="token punctuation">.</span><span class="token function">toTabletInsertionEvents</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span> |
| <span class="token function">transfer</span><span class="token punctuation">(</span>tabletInsertionEvent<span class="token punctuation">)</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| <span class="token punctuation">}</span> <span class="token keyword">finally</span> <span class="token punctuation">{</span> |
| tsFileInsertionEvent<span class="token punctuation">.</span><span class="token function">close</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| <span class="token punctuation">}</span> |
| |
| <span class="token doc-comment comment">/** |
| * This method is used to transfer the generic events, including HeartbeatEvent. |
| * |
| * <span class="token keyword">@param</span> <span class="token parameter">event</span> Event to be transferred |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">PipeConnectionException</span></span> if the connection is broken |
| * <span class="token keyword">@throws</span> <span class="token reference"><span class="token class-name">Exception</span></span> the user can throw errors if necessary |
| */</span> |
| <span class="token keyword">void</span> <span class="token function">transfer</span><span class="token punctuation">(</span><span class="token class-name">Event</span> event<span class="token punctuation">)</span> <span class="token keyword">throws</span> <span class="token class-name">Exception</span><span class="token punctuation">;</span> |
| <span class="token punctuation">}</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="custom-stream-processing-plugin-management" tabindex="-1"><a class="header-anchor" href="#custom-stream-processing-plugin-management"><span>Custom stream processing plugin management</span></a></h2><p>In order to ensure the flexibility and ease of use of user-defined plugins in actual production, the system also needs to provide the ability to dynamically and uniformly manage plugins.<br> The stream processing plugin management statements introduced in this chapter provide an entry point for dynamic unified management of plugins.</p><h3 id="load-plugin-statement" tabindex="-1"><a class="header-anchor" href="#load-plugin-statement"><span>Load plugin statement</span></a></h3><p>In IoTDB, if you want to dynamically load a user-defined plugin in the system, you first need to implement a specific plugin class based on PipeSource, PipeProcessor or PipeSink.<br> Then the plugin class needs to be compiled and packaged into a jar executable file, and finally the plugin is loaded into IoTDB using the management statement for loading the plugin.</p><p>The syntax of the management statement for loading the plugin is shown in the figure.</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPEPLUGIN <span class="token operator"><</span>alias<span class="token operator">></span> |
| <span class="token keyword">AS</span> <span class="token operator"><</span><span class="token keyword">full</span> class name<span class="token operator">></span> |
| <span class="token keyword">USING</span> <span class="token operator"><</span>URI <span class="token keyword">of</span> JAR package<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>Example: If you implement a data processing plugin named edu.tsinghua.iotdb.pipe.ExampleProcessor, and the packaged jar package is pipe-plugin.jar, you want to use this plugin in the stream processing engine, and mark the plugin as example. There are two ways to use the plugin package, one is to upload to the URI server, and the other is to upload to the local directory of the cluster.</p><p>Method 1: Upload to the URI server</p>`,47),v={href:"https://example.com:8080/iotdb/pipe-plugin.jar",target:"_blank",rel:"noopener noreferrer"},h=i(`<p>SQL：</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SQL</span> <span class="token keyword">CREATE</span> PIPEPLUGIN example |
| <span class="token keyword">AS</span> <span class="token string">'edu.tsinghua.iotdb.pipe.ExampleProcessor'</span> |
| <span class="token keyword">USING</span> URI <span class="token string">'<https://example.com:8080/iotdb/pipe-plugin.jar>'</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>Method 2: Upload the data to the local directory of the cluster</p><p>Preparation: To register in this way, you need to place the JAR package in any path on the machine where the DataNode node is located, and we recommend that you place the JAR package in the /ext/pipe directory of the IoTDB installation path (the installation package is already in the installation package, so you do not need to create a new one). For example: iotdb-1.x.x-bin/ext/pipe/pipe-plugin.jar. <strong>(Note: If you are using a cluster, you will need to place the JAR package under the same path as the machine where each DataNode node is located)</strong></p><p>SQL：</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SQL</span> <span class="token keyword">CREATE</span> PIPEPLUGIN example |
| <span class="token keyword">AS</span> <span class="token string">'edu.tsinghua.iotdb.pipe.ExampleProcessor'</span> |
| <span class="token keyword">USING</span> URI <span class="token string">'<file:/IoTDB installation path/iotdb-1.x.x-bin/ext/pipe/pipe-plugin.jar>'</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="delete-plugin-statement" tabindex="-1"><a class="header-anchor" href="#delete-plugin-statement"><span>Delete plugin statement</span></a></h3><p>When the user no longer wants to use a plugin and needs to uninstall the plugin from the system, he can use the delete plugin statement as shown in the figure.</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">DROP</span> PIPEPLUGIN <span class="token operator"><</span>alias<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="view-plugin-statements" tabindex="-1"><a class="header-anchor" href="#view-plugin-statements"><span>View plugin statements</span></a></h3><p>Users can also view plugins in the system on demand. View the statement of the plugin as shown in the figure.</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPEPLUGINS |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h2 id="system-preset-stream-processing-plugin" tabindex="-1"><a class="header-anchor" href="#system-preset-stream-processing-plugin"><span>System preset stream processing plugin</span></a></h2><h3 id="pre-built-source-plugin" tabindex="-1"><a class="header-anchor" href="#pre-built-source-plugin"><span>Pre-built Source Plugin</span></a></h3><h4 id="iotdb-source" tabindex="-1"><a class="header-anchor" href="#iotdb-source"><span>iotdb-source</span></a></h4><p>Function: Extract historical or realtime data inside IoTDB into pipe.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>source</td><td>iotdb-source</td><td>String: iotdb-source</td><td>required</td></tr><tr><td>source.pattern</td><td>path prefix for filtering time series</td><td>String: any time series prefix</td><td>optional: root</td></tr><tr><td>source.history.start-time</td><td>start of synchronizing historical data event time，including start-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MIN_VALUE</td></tr><tr><td>source.history.end-time</td><td>end of synchronizing historical data event time，including end-time</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MAX_VALUE</td></tr><tr><td>source.forwarding-pipe-requests</td><td>Whether to forward data written by another Pipe (usually Data Sync)</td><td>Boolean: true, false</td><td>optional:true</td></tr><tr><td>start-time(V1.3.1+)</td><td>start of synchronizing all data event time，including start-time. Will disable "history.start-time" "history.end-time" if configured</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MIN_VALUE</td></tr><tr><td>end-time(V1.3.1+)</td><td>end of synchronizing all data event time，including end-time. Will disable "history.start-time" "history.end-time" if configured</td><td>Long: [Long.MIN_VALUE, Long.MAX_VALUE]</td><td>optional: Long.MAX_VALUE</td></tr><tr><td>source.realtime.mode</td><td>Extraction mode for real-time data</td><td>String: hybrid, stream, batch</td><td>optional:hybrid</td></tr><tr><td>source.forwarding-pipe-requests</td><td>Whether to forward data written by another Pipe (usually Data Sync)</td><td>Boolean: true, false</td><td>optional:true</td></tr></tbody></table>`,17),g=n("p",null,[s("🚫 "),n("strong",null,"source.pattern Parameter Description")],-1),b={href:"https://iotdb.apache.org/Download/",target:"_blank",rel:"noopener noreferrer"},f=i("<li><p>In the underlying implementation, when pattern is detected as root (default value) or a database name, synchronization efficiency is higher, and any other format will reduce performance.</p></li><li><p>The path prefix does not need to form a complete path. For example, when creating a pipe with the parameter 'source.pattern'='root.aligned.1':</p><ul><li>root.aligned.1TS</li><li>root.aligned.1TS.`1`</li><li>root.aligned.100TS</li></ul><p>the data will be synchronized;</p><ul><li>root.aligned.`1`</li><li>root.aligned.`123`</li></ul><p>the data will not be synchronized.</p></li>",2),w=i(`<blockquote><p>❗️<strong>start-time, end-time parameter description of source</strong></p><ul><li>start-time, end-time should be in ISO format, such as 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00. However, version 1.3.1+ supports timeStamp format like 1706704494000.</li></ul></blockquote><blockquote><p>✅ <strong>A piece of data from production to IoTDB contains two key concepts of time</strong></p><ul><li><strong>event time:</strong> The time when the data is actually produced (or the generation time assigned to the data by the data production system, which is the time item in the data point), also called event time.</li><li><strong>arrival time:</strong> The time when data arrives in the IoTDB system.</li></ul><p>The out-of-order data we often refer to refers to data whose <strong>event time</strong> is far behind the current system time (or the maximum <strong>event time</strong> that has been dropped) when the data arrives. On the other hand, whether it is out-of-order data or sequential data, as long as they arrive newly in the system, their <strong>arrival time</strong> will increase with the order in which the data arrives at IoTDB.</p></blockquote><blockquote><p>💎 <strong>The work of iotdb-source can be split into two stages</strong></p><ol><li>Historical data extraction: All data with <strong>arrival time</strong> < <strong>current system time</strong> when creating the pipe is called historical data</li><li>Realtime data extraction: All data with <strong>arrival time</strong> >= <strong>current system time</strong> when the pipe is created is called realtime data</li></ol><p>The historical data transmission phase and the realtime data transmission phase are executed serially. Only when the historical data transmission phase is completed, the realtime data transmission phase is executed.**</p></blockquote><blockquote><p>📌 <strong>source.realtime.mode: Data extraction mode</strong></p><ul><li>log: In this mode, the task only uses the operation log for data processing and sending</li><li>file: In this mode, the task only uses data files for data processing and sending.</li><li>hybrid: This mode takes into account the characteristics of low latency but low throughput when sending data one by one in the operation log, and the characteristics of high throughput but high latency when sending in batches of data files. It can automatically operate under different write loads. Switch the appropriate data extraction method. First, adopt the data extraction method based on operation logs to ensure low sending delay. When a data backlog occurs, it will automatically switch to the data extraction method based on data files to ensure high sending throughput. When the backlog is eliminated, it will automatically switch back to the data extraction method based on data files. The data extraction method of the operation log avoids the problem of difficulty in balancing data sending delay or throughput using a single data extraction algorithm.</li></ul></blockquote><blockquote><p>🍕 <strong>source.forwarding-pipe-requests: Whether to allow forwarding data transmitted from another pipe</strong></p><ul><li>If you want to use pipe to build data synchronization of A -> B -> C, then the pipe of B -> C needs to set this parameter to true, so that the data written by A to B through the pipe in A -> B can be forwarded correctly. to C</li><li>If you want to use pipe to build two-way data synchronization (dual-active) of A <-> B, then the pipes of A -> B and B -> A need to set this parameter to false, otherwise the data will be endless. inter-cluster round-robin forwarding</li></ul></blockquote><h3 id="preset-processor-plugin" tabindex="-1"><a class="header-anchor" href="#preset-processor-plugin"><span>Preset processor plugin</span></a></h3><h4 id="do-nothing-processor" tabindex="-1"><a class="header-anchor" href="#do-nothing-processor"><span>do-nothing-processor</span></a></h4><p>Function: No processing is done on the events passed in by the source.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>processor</td><td>do-nothing-processor</td><td>String: do-nothing-processor</td><td>required</td></tr></tbody></table><h3 id="preset-sink-plugin" tabindex="-1"><a class="header-anchor" href="#preset-sink-plugin"><span>Preset sink plugin</span></a></h3><h4 id="do-nothing-sink" tabindex="-1"><a class="header-anchor" href="#do-nothing-sink"><span>do-nothing-sink</span></a></h4><p>Function: No processing is done on the events passed in by the processor.</p><table><thead><tr><th>key</th><th>value</th><th>value range</th><th>required or optional with default</th></tr></thead><tbody><tr><td>sink</td><td>do-nothing-sink</td><td>String: do-nothing-sink</td><td>required</td></tr></tbody></table><h2 id="stream-processing-task-management" tabindex="-1"><a class="header-anchor" href="#stream-processing-task-management"><span>Stream processing task management</span></a></h2><h3 id="create-a-stream-processing-task" tabindex="-1"><a class="header-anchor" href="#create-a-stream-processing-task"><span>Create a stream processing task</span></a></h3><p>Use the <code>CREATE PIPE</code> statement to create a stream processing task. Taking the creation of a data synchronization stream processing task as an example, the sample SQL statement is as follows:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> <span class="token comment">-- PipeId is the name that uniquely identifies the sync task</span> |
| <span class="token keyword">WITH</span> SOURCE <span class="token punctuation">(</span> |
| <span class="token comment">-- Default IoTDB Data Extraction Plugin</span> |
| <span class="token string">'source'</span> <span class="token operator">=</span> <span class="token string">'iotdb-source'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Path prefix, only data that can match the path prefix will be extracted for subsequent processing and delivery</span> |
| <span class="token string">'source.pattern'</span> <span class="token operator">=</span> <span class="token string">'root.timecho'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Whether to extract historical data</span> |
| <span class="token string">'source.history.enable'</span> <span class="token operator">=</span> <span class="token string">'true'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Describes the time range of the historical data being extracted, indicating the earliest possible time</span> |
| <span class="token string">'source.history.start-time'</span> <span class="token operator">=</span> <span class="token string">'2011.12.03T10:15:30+01:00'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Describes the time range of the extracted historical data, indicating the latest time</span> |
| <span class="token string">'source.history.end-time'</span> <span class="token operator">=</span> <span class="token string">'2022.12.03T10:15:30+01:00'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Whether to extract realtime data</span> |
| <span class="token string">'source.realtime.enable'</span> <span class="token operator">=</span> <span class="token string">'true'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| <span class="token keyword">WITH</span> PROCESSOR <span class="token punctuation">(</span> |
| <span class="token comment">-- Default data processing plugin, means no processing</span> |
| <span class="token string">'processor'</span> <span class="token operator">=</span> <span class="token string">'do-nothing-processor'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token comment">-- IoTDB data sending plugin with target IoTDB</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Data service for one of the DataNode nodes on the target IoTDB ip</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- Data service port of one of the DataNode nodes of the target IoTDB</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'6667'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p><strong>When creating a stream processing task, you need to configure the PipeId and the parameters of the three plugin parts:</strong></p>`,18),y=n("thead",null,[n("tr",null,[n("th",null,"Configuration"),n("th",null,"Description"),n("th",null,"Required or not"),n("th",null,"Default implementation"),n("th",null,"Default implementation description"),n("th",null,"Default implementation description")])],-1),P=n("td",null,"PipeId",-1),I=n("td",null,"A globally unique name that identifies a stream processing",-1),x=n("td",null,"-",-1),T=n("td",null,"-",-1),E=n("td",null,"-",-1),S=n("tr",null,[n("td",null,"source"),n("td",null,"Pipe Source plugin, responsible for extracting stream processing data at the bottom of the database"),n("td",null,"Optional"),n("td",null,"iotdb-source"),n("td",null,"Integrate the full historical data of the database and subsequent real-time data arriving into the stream processing task"),n("td",null,"No")],-1),_=n("td",null,"processor",-1),q=n("td",null,"Pipe Processor plugin, responsible for processing data",-1),D=n("td",null,"Optional",-1),R=n("td",null,"do-nothing-processor",-1),C=n("td",null,"Does not do any processing on the incoming data",-1),U=n("td",null,"sink",-1),B=n("td",null,"Pipe Sink plugin, responsible for sending data",-1),A=n("td",null,"-",-1),N=n("td",null,"-",-1),W=i(`<p>In the example, the iotdb-source, do-nothing-processor and iotdb-thrift-sink plugins are used to build the data flow processing task. IoTDB also has other built-in stream processing plugins, <strong>please check the "System Preset Stream Processing plugin" section</strong>.</p><p><strong>A simplest example of the CREATE PIPE statement is as follows:</strong></p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> <span class="token comment">-- PipeId is a name that uniquely identifies the stream processing task</span> |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token comment">-- IoTDB data sending plugin, the target is IoTDB</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token comment">--The data service IP of one of the DataNode nodes in the target IoTDB</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> |
| <span class="token comment">-- The data service port of one of the DataNode nodes in the target IoTDB</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'6667'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>The semantics expressed are: synchronize all historical data in this database instance and subsequent real-time data arriving to the IoTDB instance with the target 127.0.0.1:6667.</p><p><strong>Notice:</strong></p><ul><li><p>SOURCE and PROCESSOR are optional configurations. If you do not fill in the configuration parameters, the system will use the corresponding default implementation.</p></li><li><p>SINK is a required configuration and needs to be configured declaratively in the CREATE PIPE statement</p></li><li><p>SINK has self-reuse capability. For different stream processing tasks, if their SINKs have the same KV attributes (the keys corresponding to the values of all attributes are the same), then the system will only create one SINK instance in the end to realize the duplication of connection resources.</p><ul><li>For example, there are the following declarations of two stream processing tasks, pipe1 and pipe2:</li></ul><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">CREATE</span> PIPE pipe1 |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'localhost'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'9999'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| |
| <span class="token keyword">CREATE</span> PIPE pipe2 |
| <span class="token keyword">WITH</span> SINK <span class="token punctuation">(</span> |
| <span class="token string">'sink'</span> <span class="token operator">=</span> <span class="token string">'iotdb-thrift-sink'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.port'</span> <span class="token operator">=</span> <span class="token string">'9999'</span><span class="token punctuation">,</span> |
| <span class="token string">'sink.ip'</span> <span class="token operator">=</span> <span class="token string">'localhost'</span><span class="token punctuation">,</span> |
| <span class="token punctuation">)</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div></li><li><p>Because their declarations of SINK are exactly the same (<strong>even if the order of declaration of some attributes is different</strong>), the framework will automatically reuse the SINKs they declared, and ultimately the SINKs of pipe1 and pipe2 will be the same instance. .</p></li><li><p>When the source is the default iotdb-source, and source.forwarding-pipe-requests is the default value true, please do not build an application scenario that includes data cycle synchronization (it will cause an infinite loop):</p><ul><li>IoTDB A -> IoTDB B -> IoTDB A</li><li>IoTDB A -> IoTDB A</li></ul></li></ul><h3 id="start-the-stream-processing-task" tabindex="-1"><a class="header-anchor" href="#start-the-stream-processing-task"><span>Start the stream processing task</span></a></h3><p>After the CREATE PIPE statement is successfully executed, the stream processing task-related instance will be created, but the running status of the entire stream processing task will be set to STOPPED(V1.3.0), that is, the stream processing task will not process data immediately. In version 1.3.1 and later, the status of the task will be set to RUNNING after CREATE.</p><p>You can use the START PIPE statement to cause a stream processing task to start processing data:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">START</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="stop-the-stream-processing-task" tabindex="-1"><a class="header-anchor" href="#stop-the-stream-processing-task"><span>Stop the stream processing task</span></a></h3><p>Use the STOP PIPE statement to stop the stream processing task from processing data:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code>STOP PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><h3 id="delete-stream-processing-tasks" tabindex="-1"><a class="header-anchor" href="#delete-stream-processing-tasks"><span>Delete stream processing tasks</span></a></h3><p>Use the DROP PIPE statement to stop the stream processing task from processing data (when the stream processing task status is RUNNING), and then delete the entire stream processing task:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">DROP</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>Users do not need to perform a STOP operation before deleting the stream processing task.</p><h3 id="display-stream-processing-tasks" tabindex="-1"><a class="header-anchor" href="#display-stream-processing-tasks"><span>Display stream processing tasks</span></a></h3><p>Use the SHOW PIPES statement to view all stream processing tasks:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPES |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>The query results are as follows:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| <span class="token operator">|</span> ID<span class="token operator">|</span> CreationTime<span class="token operator">|</span> State<span class="token operator">|</span>PipeSource<span class="token operator">|</span>PipeProcessor<span class="token operator">|</span>PipeSink<span class="token operator">|</span>ExceptionMessage<span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| <span class="token operator">|</span>iotdb<span class="token operator">-</span>kafka<span class="token operator">|</span><span class="token number">2022</span><span class="token operator">-</span><span class="token number">03</span><span class="token operator">-</span><span class="token number">30</span>T20:<span class="token number">58</span>:<span class="token number">30.689</span><span class="token operator">|</span>RUNNING<span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> {}<span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| <span class="token operator">|</span>iotdb<span class="token operator">-</span>iotdb<span class="token operator">|</span><span class="token number">2022</span><span class="token operator">-</span><span class="token number">03</span><span class="token operator">-</span><span class="token number">31</span>T12:<span class="token number">55</span>:<span class="token number">28.129</span><span class="token operator">|</span>STOPPED<span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> TException: <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token operator">|</span> |
| <span class="token operator">+</span><span class="token comment">-----------+-----------------------+-------+----------+-------------+--------+----------------+</span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>You can use <code><PipeId></code> to specify the status of a stream processing task you want to see:</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPE <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div></div></div><p>You can also use the where clause to determine whether the Pipe Sink used by a certain <PipeId> is reused.</p><div class="language-sql line-numbers-mode" data-ext="sql" data-title="sql"><pre class="language-sql"><code><span class="token keyword">SHOW</span> PIPES |
| <span class="token keyword">WHERE</span> SINK USED <span class="token keyword">BY</span> <span class="token operator"><</span>PipeId<span class="token operator">></span> |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="stream-processing-task-running-status-migration" tabindex="-1"><a class="header-anchor" href="#stream-processing-task-running-status-migration"><span>Stream processing task running status migration</span></a></h3><p>A stream processing pipe will pass through various states during its managed life cycle:</p><ul><li><strong>RUNNING:</strong> pipe is working properly <ul><li>When a pipe is successfully created, its initial state is RUNNING.(V1.3.1+)</li></ul></li><li><strong>STOPPED:</strong> The pipe is stopped. When the pipeline is in this state, there are several possibilities: <ul><li>When a pipe is successfully created, its initial state is STOPPED.(V1.3.0)</li><li>The user manually pauses a pipe that is in normal running status, and its status will passively change from RUNNING to STOPPED.</li><li>When an unrecoverable error occurs during the running of a pipe, its status will automatically change from RUNNING to STOPPED</li></ul></li><li><strong>DROPPED:</strong> The pipe task was permanently deleted</li></ul><p>The following diagram shows all states and state transitions:</p><figure><img src="https://alioss.timecho.com/docs/img/状态迁移图.png" alt="State migration diagram" tabindex="0" loading="lazy"><figcaption>State migration diagram</figcaption></figure><h2 id="authority-management" tabindex="-1"><a class="header-anchor" href="#authority-management"><span>authority management</span></a></h2><h3 id="stream-processing-tasks" tabindex="-1"><a class="header-anchor" href="#stream-processing-tasks"><span>Stream processing tasks</span></a></h3><table><thead><tr><th>Permission name</th><th>Description</th></tr></thead><tbody><tr><td>USE_PIPE</td><td>Register a stream processing task. The path is irrelevant.</td></tr><tr><td>USE_PIPE</td><td>Start the stream processing task. The path is irrelevant.</td></tr><tr><td>USE_PIPE</td><td>Stop the stream processing task. The path is irrelevant.</td></tr><tr><td>USE_PIPE</td><td>Offload stream processing tasks. The path is irrelevant.</td></tr><tr><td>USE_PIPE</td><td>Query stream processing tasks. The path is irrelevant.</td></tr></tbody></table><h3 id="stream-processing-task-plugin" tabindex="-1"><a class="header-anchor" href="#stream-processing-task-plugin"><span>Stream processing task plugin</span></a></h3><table><thead><tr><th>Permission name</th><th>Description</th></tr></thead><tbody><tr><td>USE_PIPE</td><td>Register stream processing task plugin. The path is irrelevant.</td></tr><tr><td>USE_PIPE</td><td>Uninstall the stream processing task plugin. The path is irrelevant.</td></tr><tr><td>USE_PIPE</td><td>Query stream processing task plugin. The path is irrelevant.</td></tr></tbody></table><h2 id="configuration-parameters" tabindex="-1"><a class="header-anchor" href="#configuration-parameters"><span>Configuration parameters</span></a></h2><p>In iotdb-common.properties：</p><p>V1.3.0+:</p><div class="language-Properties line-numbers-mode" data-ext="Properties" data-title="Properties"><pre class="language-Properties"><code>#################### |
| ### Pipe Configuration |
| #################### |
| |
| # Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\\\", or if its prefix is "\\\\\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_connector_timeout_ms=900000 |
| |
| # The maximum number of selectors that can be used in the async connector. |
| # pipe_async_connector_selector_number=1 |
| |
| # The core number of clients that can be used in the async connector. |
| # pipe_async_connector_core_client_number=8 |
| |
| # The maximum number of clients that can be used in the async connector. |
| # pipe_async_connector_max_client_number=16 |
| |
| # Whether to enable receiving pipe data through air gap. |
| # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. |
| # pipe_air_gap_receiver_enabled=false |
| |
| # The port for the server to receive pipe data through air gap. |
| # pipe_air_gap_receiver_port=9780 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>V1.3.1+:</p><div class="language-Properties line-numbers-mode" data-ext="Properties" data-title="Properties"><pre class="language-Properties"><code># Uncomment the following field to configure the pipe lib directory. |
| # For Windows platform |
| # If its prefix is a drive specifier followed by "\\\\", or if its prefix is "\\\\\\\\", then the path is |
| # absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext\\\\pipe |
| # For Linux platform |
| # If its prefix is "/", then the path is absolute. Otherwise, it is relative. |
| # pipe_lib_dir=ext/pipe |
| |
| # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. |
| # The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). |
| # pipe_subtask_executor_max_thread_num=5 |
| |
| # The connection timeout (in milliseconds) for the thrift client. |
| # pipe_sink_timeout_ms=900000 |
| |
| # The maximum number of selectors that can be used in the sink. |
| # Recommend to set this value to less than or equal to pipe_sink_max_client_number. |
| # pipe_sink_selector_number=4 |
| |
| # The maximum number of clients that can be used in the sink. |
| # pipe_sink_max_client_number=16 |
| |
| # Whether to enable receiving pipe data through air gap. |
| # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. |
| # pipe_air_gap_receiver_enabled=false |
| |
| # The port for the server to receive pipe data through air gap. |
| # pipe_air_gap_receiver_port=9780 |
| </code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div>`,42);function L(F,O){const e=p("font"),o=p("ExternalLinkIcon");return c(),r("div",null,[u,m,n("p",null,[s("We call "),a(e,{color:"RED"},{default:t(()=>[s("a data flow processing task a Pipe")]),_:1}),s(". A stream processing task (Pipe) contains three subtasks:")]),k,n("p",null,[s("Preparation: To register in this way, you need to upload the JAR package to the URI server in advance and ensure that the IoTDB instance that executes the registration statement can access the URI server. For example "),n("a",v,[s("https://example.com:8080/iotdb/pipe-plugin.jar"),a(o)]),s(" .")]),h,n("blockquote",null,[g,n("ul",null,[n("li",null,[n("p",null,[s("Pattern should use backquotes to modify illegal characters or illegal path nodes, for example, if you want to filter root.`a@b` or root.`123`, you should set the pattern to root.`a@b` or root.`123`（Refer specifically to "),n("a",b,[s("Timing of single and double quotes and backquotes"),a(o)]),s("）")])]),f])]),w,n("table",null,[y,n("tbody",null,[n("tr",null,[P,I,n("td",null,[a(e,{color:"red"},{default:t(()=>[s("Required")]),_:1})]),x,T,E]),S,n("tr",null,[_,q,D,R,C,n("td",null,[a(e,{color:"red"},{default:t(()=>[s("Yes")]),_:1})])]),n("tr",null,[U,B,n("td",null,[a(e,{color:"red"},{default:t(()=>[s("Required")]),_:1})]),A,N,n("td",null,[a(e,{color:"red"},{default:t(()=>[s("Yes")]),_:1})])])])]),W])}const z=l(d,[["render",L],["__file","Streaming_timecho.html.vue"]]),j=JSON.parse('{"path":"/UserGuide/latest/User-Manual/Streaming_timecho.html","title":"IoTDB stream processing framework","lang":"en-US","frontmatter":{"description":"IoTDB stream processing framework The IoTDB stream processing framework allows users to implement customized stream processing logic, which can monitor and capture storage engin...","head":[["link",{"rel":"alternate","hreflang":"zh-cn","href":"https://iotdb.apache.org/zh/UserGuide/latest/User-Manual/Streaming_timecho.html"}],["meta",{"property":"og:url","content":"https://iotdb.apache.org/UserGuide/latest/User-Manual/Streaming_timecho.html"}],["meta",{"property":"og:site_name","content":"IoTDB Website"}],["meta",{"property":"og:title","content":"IoTDB stream processing framework"}],["meta",{"property":"og:description","content":"IoTDB stream processing framework The IoTDB stream processing framework allows users to implement customized stream processing logic, which can monitor and capture storage engin..."}],["meta",{"property":"og:type","content":"article"}],["meta",{"property":"og:image","content":"https://alioss.timecho.com/docs/img/1706778988482.jpg"}],["meta",{"property":"og:locale","content":"en-US"}],["meta",{"property":"og:locale:alternate","content":"zh-CN"}],["meta",{"property":"og:updated_time","content":"2024-04-08T07:45:44.000Z"}],["meta",{"property":"article:modified_time","content":"2024-04-08T07:45:44.000Z"}],["script",{"type":"application/ld+json"},"{\\"@context\\":\\"https://schema.org\\",\\"@type\\":\\"Article\\",\\"headline\\":\\"IoTDB stream processing framework\\",\\"image\\":[\\"https://alioss.timecho.com/docs/img/1706778988482.jpg\\",\\"https://alioss.timecho.com/docs/img/%E7%8A%B6%E6%80%81%E8%BF%81%E7%A7%BB%E5%9B%BE.png\\"],\\"dateModified\\":\\"2024-04-08T07:45:44.000Z\\",\\"author\\":[]}"]]},"headers":[{"level":2,"title":"Custom stream processing plugin development","slug":"custom-stream-processing-plugin-development","link":"#custom-stream-processing-plugin-development","children":[{"level":3,"title":"Programming development dependencies","slug":"programming-development-dependencies","link":"#programming-development-dependencies","children":[]},{"level":3,"title":"Event-driven programming model","slug":"event-driven-programming-model","link":"#event-driven-programming-model","children":[]},{"level":3,"title":"Custom stream processing plugin programming interface definition","slug":"custom-stream-processing-plugin-programming-interface-definition","link":"#custom-stream-processing-plugin-programming-interface-definition","children":[]}]},{"level":2,"title":"Custom stream processing plugin management","slug":"custom-stream-processing-plugin-management","link":"#custom-stream-processing-plugin-management","children":[{"level":3,"title":"Load plugin statement","slug":"load-plugin-statement","link":"#load-plugin-statement","children":[]},{"level":3,"title":"Delete plugin statement","slug":"delete-plugin-statement","link":"#delete-plugin-statement","children":[]},{"level":3,"title":"View plugin statements","slug":"view-plugin-statements","link":"#view-plugin-statements","children":[]}]},{"level":2,"title":"System preset stream processing plugin","slug":"system-preset-stream-processing-plugin","link":"#system-preset-stream-processing-plugin","children":[{"level":3,"title":"Pre-built Source Plugin","slug":"pre-built-source-plugin","link":"#pre-built-source-plugin","children":[]},{"level":3,"title":"Preset processor plugin","slug":"preset-processor-plugin","link":"#preset-processor-plugin","children":[]},{"level":3,"title":"Preset sink plugin","slug":"preset-sink-plugin","link":"#preset-sink-plugin","children":[]}]},{"level":2,"title":"Stream processing task management","slug":"stream-processing-task-management","link":"#stream-processing-task-management","children":[{"level":3,"title":"Create a stream processing task","slug":"create-a-stream-processing-task","link":"#create-a-stream-processing-task","children":[]},{"level":3,"title":"Start the stream processing task","slug":"start-the-stream-processing-task","link":"#start-the-stream-processing-task","children":[]},{"level":3,"title":"Stop the stream processing task","slug":"stop-the-stream-processing-task","link":"#stop-the-stream-processing-task","children":[]},{"level":3,"title":"Delete stream processing tasks","slug":"delete-stream-processing-tasks","link":"#delete-stream-processing-tasks","children":[]},{"level":3,"title":"Display stream processing tasks","slug":"display-stream-processing-tasks","link":"#display-stream-processing-tasks","children":[]},{"level":3,"title":"Stream processing task running status migration","slug":"stream-processing-task-running-status-migration","link":"#stream-processing-task-running-status-migration","children":[]}]},{"level":2,"title":"authority management","slug":"authority-management","link":"#authority-management","children":[{"level":3,"title":"Stream processing tasks","slug":"stream-processing-tasks","link":"#stream-processing-tasks","children":[]},{"level":3,"title":"Stream processing task plugin","slug":"stream-processing-task-plugin","link":"#stream-processing-task-plugin","children":[]}]},{"level":2,"title":"Configuration parameters","slug":"configuration-parameters","link":"#configuration-parameters","children":[]}],"git":{"createdTime":1704854860000,"updatedTime":1712562344000,"contributors":[{"name":"Caideyipi","email":"87789683+Caideyipi@users.noreply.github.com","commits":1},{"name":"CritasWang","email":"critas@outlook.com","commits":1},{"name":"Tansgr","email":"101696091+tanxilo@users.noreply.github.com","commits":1},{"name":"Tanxilo","email":"101696091+tanxilo@users.noreply.github.com","commits":1}]},"readingTime":{"minutes":20.21,"words":6062},"filePathRelative":"UserGuide/latest/User-Manual/Streaming_timecho.md","localizedDate":"January 10, 2024","autoDesc":true}');export{z as comp,j as data}; |
