| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache InLong</b></a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/introduction">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/introduction">Next</a></li><li><a class="dropdown__link" href="/docs/introduction">1.11.0</a></li><li><a class="dropdown__link" href="/docs/1.10.0/introduction">1.10.0</a></li><li><a class="dropdown__link" href="/docs/1.9.0/introduction">1.9.0</a></li><li><a class="dropdown__link" href="/docs/1.8.0/introduction">1.8.0</a></li><li><a class="dropdown__link" href="/versions/">All versions</a></li></ul></div><a class="navbar__item navbar__link" href="/downloads">Download</a><a class="navbar__item navbar__link" href="/community/how-to-contribute">Community</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/blog">Blog</a><a class="navbar__item navbar__link" href="/team">Team</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">ASF</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li><li><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Privacy</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/blog/2022/06/16/inlong-sort-etl" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li><li><a href="/zh-CN/blog/2022/06/16/inlong-sort-etl" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="zh-CN">简体中文</a></li></ul></div><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/2023/12/13/release-1.10.0">Release 1.10.0</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/2023/09/25/release-1.9.0">Release 1.9.0</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/2023/07/24/release-1.8.0">Release 1.8.0</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/2023/05/19/release-1.7.0">Release 1.7.0</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/2023/03/23/release-1.6.0">Release 1.6.0</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">Analysis of InLong Sort ETL Solution</h1><div class="container_mt6G margin-vert--md"><time datetime="2022-06-16T00:00:00.000Z" itemprop="datePublished">June 16, 2022</time> · <!-- -->11 min read</div><div class="margin-top--md margin-bottom--sm row"><div class="col col--6 authorCol_Hf19"><div class="avatar margin-bottom--sm"><a href="https://github.com/Oneal65" target="_blank" rel="noopener noreferrer" class="avatar__photo-link"><img class="avatar__photo" src="https://avatars.githubusercontent.com/u/13121552?v=4" alt="Oneal65"></a><div class="avatar__intro" itemprop="author" itemscope="" itemtype="https://schema.org/Person"><div class="avatar__name"><a href="https://github.com/Oneal65" target="_blank" rel="noopener noreferrer" itemprop="url"><span itemprop="name">Oneal65</span></a></div></div></div></div></div></header><div id="post-content" class="markdown" itemprop="articleBody"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1-background">1. Background<a href="#1-background" class="hash-link" aria-label="Direct link to 1. Background" title="Direct link to 1. Background"></a></h2><p>With the increasing number of users and developers of Apache InLong(incubating), the demand for richer usage scenarios and low-cost operation is getting stronger and stronger. Among them, the demand for adding Transform (T) to the whole link of InLong has received the most feedback. After the research and design of @yunqingmoswu, @EMsnap, @gong, @thexiay community developers, the InLong Sort ETL solution based on Flink SQL has been completed. This article will introduce the implementation details of the solution in detail.</p><p>Firstly, based on Apache Flink SQL, there are mainly the following considerations：</p><ul><li>Flink SQL has high scalability and flexibility brought about by its powerful expression ability. Basically, Flink SQL can support most demand scenarios in the community. When the built-in functions of Flink SQL do not meet the requirements, we can also extend them through various UDFs.</li><li>Compared with the implementation of the underlying API of Flink, the development cost of Flink SQL is lower. Only for the first time, the conversion logic of Flink SQL needs to be implemented. In the future, we can focus on the construction of the ability of Flink SQL, such as the extension connector and the UDF.</li><li>In general, Flink SQL will be more robust and run more stable. The reason is that Flink SQL shields a lot of the underlying details of Flink, has strong community support, and has been practiced by a large number of users.</li><li>For users, Flink SQL is also easier to understand, especially for users who have used SQL, the usage is simple and familiar, which helps users to land quickly.</li><li>For the migration of existing real-time tasks, if they are originally SQL-type tasks, especially Flink SQL tasks, the migration cost is extremely low, and in some cases, no changes are even required.</li></ul><p><strong>Note</strong>: For all codes of this scheme, please refer to <a href="https://github.com/apache/incubator-inlong/tree/master/inlong-sort" target="_blank" rel="noopener noreferrer">Apache InLong Sort</a>, which can be downloaded and used in the upcoming version 1.2.0.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2-introduction">2. Introduction<a href="#2-introduction" class="hash-link" aria-label="Direct link to 2. Introduction" title="Direct link to 2. Introduction"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="21-requirements">2.1 Requirements<a href="#21-requirements" class="hash-link" aria-label="Direct link to 2.1 Requirements" title="Direct link to 2.1 Requirements"></a></h3><p>The main requirements of this solution are the completed inlong sort module transform (T) capability, including:</p><table><thead><tr><th align="center">Transform</th><th align="center">Notes</th></tr></thead><tbody><tr><td align="center">Deduplication in the window</td><td align="center">Deduplicate data within a time window</td></tr><tr><td align="center">time window aggregation</td><td align="center">Aggregate data within a time window</td></tr><tr><td align="center">time format conversion</td><td align="center">Converts the value of a field to a string in the target time format</td></tr><tr><td align="center">field segmentation</td><td align="center">Split a field into multiple new fields by a delimiter</td></tr><tr><td align="center">string replacement</td><td align="center">Replace some or all of the contents of a string field</td></tr><tr><td align="center">Data filtering</td><td align="center">Discard or retain data that meets the filter conditions</td></tr><tr><td align="center">Content extraction</td><td align="center">Extract part of a field to create a new field</td></tr><tr><td align="center">Join</td><td align="center">Support two table join</td></tr><tr><td align="center">Value substitution</td><td align="center">Given a matching value, if the field's value is equal to that value, replace it with the target value</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="22-usage-scenarios">2.2 Usage Scenarios<a href="#22-usage-scenarios" class="hash-link" aria-label="Direct link to 2.2 Usage Scenarios" title="Direct link to 2.2 Usage Scenarios"></a></h3><p>Users of big data integration have transform requirements such as data transformation, connection and filtering in many business scenarios.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="23-design-goal">2.3 Design Goal<a href="#23-design-goal" class="hash-link" aria-label="Direct link to 2.3 Design Goal" title="Direct link to 2.3 Design Goal"></a></h3><p>This design needs to achieve the following goals:</p><ul><li>Functionality: Under InLong Sort's existing architecture and data flow model, it covers basic Transform capabilities and has the ability to expand rapidly.</li><li>Compatibility: The new InLong Sort data model is forward compatible to ensure that historical tasks can be configured and run properly.</li><li>Maintainability: The conversion of the InLong Sort data model to Flink SQL only needs to be implemented once. When there are new functional requirements later, this part does not need to be changed, even if there are changes, it can be supported with a small amount of changes.</li><li>Extensibility: When the open source Flink Connector or the built-in Flink SQL function does not meet the requirements, you can customize the Flink Connector and UDF to achieve its function expansion.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="24-basic-concepts">2.4 Basic Concepts<a href="#24-basic-concepts" class="hash-link" aria-label="Direct link to 2.4 Basic Concepts" title="Direct link to 2.4 Basic Concepts"></a></h3><p>The core concept refers to the explanation of terms in the outline design</p><table><thead><tr><th align="center">Name</th><th align="center">Meaning</th></tr></thead><tbody><tr><td align="center">InLong Dashboard</td><td align="center">Inlong front end management interface</td></tr><tr><td align="center">InLong Manager Client</td><td align="center">Wrap the interface in the manager for external user programs to call without going through the front-end inlong dashboard</td></tr><tr><td align="center">InLong Manager Openapi</td><td align="center">Inlong manager and external system call interface</td></tr><tr><td align="center">InLong Manager metaData</td><td align="center">Inlong manager metadata management, including metadata information of group and stream dimensions</td></tr><tr><td align="center">InLong Manager task manager</td><td align="center">Inlong manager manages the data source collection task module, manages agent task distribution, instruction distribution, and heartbeat reporting</td></tr><tr><td align="center">InLong Group</td><td align="center">Data flow group, including multiple data flows, one group represents one data access</td></tr><tr><td align="center">InLong Stream</td><td align="center">Data flow: a data flow has a specific flow direction</td></tr><tr><td align="center">Stream Source</td><td align="center">There are corresponding acquisition end and sink end in the stream. This design only involves the stream source</td></tr><tr><td align="center">Stream Info</td><td align="center">Abstract of data flow in sort, including various sources, transformations, destinations, etc. of the data flow</td></tr><tr><td align="center">Group Info</td><td align="center">Encapsulation of data flow in sort. A group info can contain multiple stream infos</td></tr><tr><td align="center">Node</td><td align="center">Abstraction of data source, data transformation and data destination in data synchronization</td></tr><tr><td align="center">Extract Node</td><td align="center">Source side abstraction of data synchronization</td></tr><tr><td align="center">Load Node</td><td align="center">Destination abstraction of data synchronization</td></tr><tr><td align="center">MySQL Extract Node</td><td align="center">MySQL data source abstraction</td></tr><tr><td align="center">Kafka Load Node</td><td align="center">Kafka data destination abstraction</td></tr><tr><td align="center">Transform Node</td><td align="center">Transformation process abstraction of data synchronization</td></tr><tr><td align="center">Aggregate Transform Node</td><td align="center">Data synchronization aggregation class transformation process abstraction</td></tr><tr><td align="center">Node Relation</td><td align="center">Relationship abstraction of nodes in data synchronization</td></tr><tr><td align="center">Field Relation</td><td align="center">Abstraction of the relationship between upstream and downstream node fields in data synchronization</td></tr><tr><td align="center">Function</td><td align="center">Abstraction of the relationship between upstream and downstream node fields in data synchronization</td></tr><tr><td align="center">Substring Function</td><td align="center">Abstraction of string interception function</td></tr><tr><td align="center">Filter Function</td><td align="center">Abstraction of data filter function</td></tr><tr><td align="center">Function Param</td><td align="center">Input parameter abstraction of function</td></tr><tr><td align="center">Constant Param</td><td align="center">Constant parameters</td></tr><tr><td align="center">Field Info</td><td align="center">Node field</td></tr><tr><td align="center">Meta FieldInfo</td><td align="center">Node meta information field</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="25-domain-model">2.5 Domain Model<a href="#25-domain-model" class="hash-link" aria-label="Direct link to 2.5 Domain Model" title="Direct link to 2.5 Domain Model"></a></h3><p>This design mainly involves the following entities: </p><p>Group, Stream, GroupInfo, StreamInfo, Node, NodeRelation, FieldRelation, Function, FilterFunction, SubstringFunction, FunctionParam, FieldInfo, MetaFieldInfo, MySQLExtractNode, KafkaLoadNode, etc.</p><p>For ease of understanding, this section will model and analyze the relationship between entities. Description of entity correspondence of domain model:</p><ul><li>One group corresponds to one group info</li><li>A group contains one or more streams</li><li>One stream corresponds to one StreamInfo</li><li>A GroupInfo contains one or more StreamInfo</li><li>A StreamInfo contains multiple nodes</li><li>A StreamInfo contains one or more NodeRelations</li><li>A NodeRelation contains one or more FieldRelations</li><li>A NodeRelation contains 0 or more FilterFunctions</li><li>A FieldRelation contains one function or one FieldInfo as the source field and one FieldInfo as the target field</li><li>A function contains one or more FunctionParams</li></ul><p>The above relationship can be represented by UML object relationship diagram as:</p><p><img loading="lazy" alt="sort_UML" src="/assets/images/sort_UML-896d751427509d769add998680df9516.png" width="2576" height="869" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="26-function-use-case-diagram">2.6 Function Use-case Diagram<a href="#26-function-use-case-diagram" class="hash-link" aria-label="Direct link to 2.6 Function Use-case Diagram" title="Direct link to 2.6 Function Use-case Diagram"></a></h3><p><img loading="lazy" alt="sort-usecase" src="/assets/images/sort-usecase-fb8639f9724899ab3afcbf35b8a21902.png" width="606" height="356" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-system-outline-design">3. System Outline Design<a href="#3-system-outline-design" class="hash-link" aria-label="Direct link to 3. System Outline Design" title="Direct link to 3. System Outline Design"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="31-system-architecture-diagram">3.1 System Architecture Diagram<a href="#31-system-architecture-diagram" class="hash-link" aria-label="Direct link to 3.1 System Architecture Diagram" title="Direct link to 3.1 System Architecture Diagram"></a></h3><p><img loading="lazy" alt="architecture" src="/assets/images/architecture-b4c0fb3783a6ed2f2868f534df98e74b.png" width="461" height="741" class="img_ev3q"></p><ul><li>Serialization: Serialization Implementation Module</li><li>Deserialization: Deserialization Implementation Module</li><li>Flink Source: Custom Flink source implementation module</li><li>Flink Sink: Custom Flink sink implementation module</li><li>Transformation: Custom Transform implementation module</li><li>GroupInfo: Corresponding to Inlong group</li><li>StreamInfo: Corresponding to inlong stream</li><li>Node: Abstraction of data source, data conversion and data destination in data synchronization</li><li>FlinkSQLParser: SQL parser</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="32-inlong-sort-internal-operation-flow-chart">3.2 InLong Sort Internal Operation Flow Chart<a href="#32-inlong-sort-internal-operation-flow-chart" class="hash-link" aria-label="Direct link to 3.2 InLong Sort Internal Operation Flow Chart" title="Direct link to 3.2 InLong Sort Internal Operation Flow Chart"></a></h3><p><img loading="lazy" alt="sort-operation-flow" src="/assets/images/sort-operation-flow-77363f12a68a011beba26db9ccc6fedb.png" width="771" height="61" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="33-module-design">3.3 Module Design<a href="#33-module-design" class="hash-link" aria-label="Direct link to 3.3 Module Design" title="Direct link to 3.3 Module Design"></a></h3><p>This design only adds Flink connector and Flink SQL generator to the original system, and modifies the data model module.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="331-module-structure">3.3.1 Module Structure<a href="#331-module-structure" class="hash-link" aria-label="Direct link to 3.3.1 Module Structure" title="Direct link to 3.3.1 Module Structure"></a></h4><p><img loading="lazy" alt="sort-module-structure" src="/assets/images/sort-module-structure-4dd424ae93043cb912dba69c08590b33.png" width="771" height="1011" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="332-module-division">3.3.2 Module Division<a href="#332-module-division" class="hash-link" aria-label="Direct link to 3.3.2 Module Division" title="Direct link to 3.3.2 Module Division"></a></h4><p>Description of important module division:</p><table><thead><tr><th align="center">Name</th><th align="center">Description</th></tr></thead><tbody><tr><td align="center">FlinkSQLParser</td><td align="center">Used to generate Flink SQL core classes, including references to GroupInfo</td></tr><tr><td align="center">GroupInfo</td><td align="center">The internal abstraction of sort for inlong group is used to encapsulate the synchronization related information of the entire inlong group, including the reference to list\<StreamInfo<!-- -->></td></tr><tr><td align="center">StreamInfo</td><td align="center">The internal abstraction of sort to inlong stream is used to encapsulate inlong stream synchronization related information, including references to list\<node<!-- -->><!-- -->, list\<NodeRelation<!-- -->></td></tr><tr><td align="center">Node</td><td align="center">The top-level interface of the synchronization node. Its subclass implementation is mainly used to encapsulate the data of the synchronization data source and the transformation node</td></tr><tr><td align="center">ExtractNode</td><td align="center">Data extract node abstraction, inherited from node</td></tr><tr><td align="center">LoadNode</td><td align="center">Data load node abstraction, inherited from node</td></tr><tr><td align="center">TransformNode</td><td align="center">Data transformation node abstraction, inherited from node</td></tr><tr><td align="center">NodeRelation</td><td align="center">Define relationships between nodes</td></tr><tr><td align="center">FieldRelation</td><td align="center">Define field relationships between nodes</td></tr><tr><td align="center">Function</td><td align="center">Abstract of T-ability execution function</td></tr><tr><td align="center">FilterFunction</td><td align="center">Function abstraction for data filtering, inherited from function</td></tr><tr><td align="center">SubstringFunction</td><td align="center">Used for string interception function abstraction, inherited from function</td></tr><tr><td align="center">FunctionParam</td><td align="center">Abstraction for function parameters</td></tr><tr><td align="center">ConstantParam</td><td align="center">Encapsulation of function constant parameters, inherited from FunctionParam</td></tr><tr><td align="center">FieldInfo</td><td align="center">The encapsulation of node fields can also be used as function input parameters, inherited from FunctionParam</td></tr><tr><td align="center">MetaFieldInfo</td><td align="center">The encapsulation of built-in fields is currently mainly used in the metadata field scenario of canal JSON, which is inherited from FieldInfo</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4-detailed-system-design">4. Detailed System Design<a href="#4-detailed-system-design" class="hash-link" aria-label="Direct link to 4. Detailed System Design" title="Direct link to 4. Detailed System Design"></a></h2><p>The following describes the principle of SQL generation by taking MySQL synchronizing data to Kafka as an example</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="41-node-described-in-sql">4.1 Node Described in SQL<a href="#41-node-described-in-sql" class="hash-link" aria-label="Direct link to 4.1 Node Described in SQL" title="Direct link to 4.1 Node Described in SQL"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="411-extractnode-described-in-sql">4.1.1 ExtractNode Described in SQL<a href="#411-extractnode-described-in-sql" class="hash-link" aria-label="Direct link to 4.1.1 ExtractNode Described in SQL" title="Direct link to 4.1.1 ExtractNode Described in SQL"></a></h4><p>The node configuration is:</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> private Node buildMySQLExtractNode() {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FieldInfo> fields = Arrays.asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("name", new StringFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()));</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> return new MySqlExtractNode("1", "mysql_input", fields,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> null, null, "id",</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Collections.singletonList("tableName"), "localhost", "root", "password",</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> "inlong", null, null,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> null, null);</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The generated SQL is:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">CREATE</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">TABLE</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">mysql_1</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">name</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> string</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">int</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"> </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">with</span><span class="token plain"> </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token string" style="color:rgb(255, 121, 198)">'connector'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'mysql-cdc-inlong'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'hostname'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'localhost'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'username'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'root'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'password'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'password'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'database-name'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'inlong'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'table-name'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'tableName'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="412-transformnode--described-in-sql">4.1.2 TransformNode Described in SQL<a href="#412-transformnode--described-in-sql" class="hash-link" aria-label="Direct link to 4.1.2 TransformNode Described in SQL" title="Direct link to 4.1.2 TransformNode Described in SQL"></a></h4><p>The node configuration is:</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FilterFunction> filters = Arrays.asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new SingleValueFilterFunction(EmptyOperator.getInstance(),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> LessThanOperator.getInstance(), new ConstantParam(25)),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new SingleValueFilterFunction(AndOperator.getInstance(),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> MoreThanOrEqualOperator.getInstance(), new ConstantParam(18))</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> );</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The generated SQL is:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">SELECT</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">name</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">AS</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">name</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">AS</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">FROM</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">mysql_1</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">WHERE</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token operator"><</span><span class="token plain"> </span><span class="token number">25</span><span class="token plain"> </span><span class="token operator">AND</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token operator">>=</span><span class="token plain"> </span><span class="token number">18</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="413-loadnode-described-in-sql">4.1.3 LoadNode Described in SQL<a href="#413-loadnode-described-in-sql" class="hash-link" aria-label="Direct link to 4.1.3 LoadNode Described in SQL" title="Direct link to 4.1.3 LoadNode Described in SQL"></a></h4><p>The node configuration is:</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> private Node buildKafkaLoadNode(FilterStrategy filterStrategy) {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FieldInfo> fields = Arrays.asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("name", new StringFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> );</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FieldRelation> relations = Arrays</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldRelation(new FieldInfo("name", new StringFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("name", new StringFormatInfo())),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldRelation(new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()))</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> );</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FilterFunction> filters = Arrays.asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new SingleValueFilterFunction(EmptyOperator.getInstance(),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> LessThanOperator.getInstance(), new ConstantParam(25)),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new SingleValueFilterFunction(AndOperator.getInstance(),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> MoreThanOrEqualOperator.getInstance(), new ConstantParam(18))</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> );</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> return new KafkaLoadNode("2", "kafka_output", fields, relations, filters,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> filterStrategy, "topic1", "localhost:9092",</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new CanalJsonFormat(), null,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> null, "id");</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The generated SQL is:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">CREATE</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">TABLE</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">kafka_3</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">name</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> string</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">int</span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><span class="token plain"> </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">with</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'connector'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'kafka-inlong'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'topic'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'topic1'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'properties.bootstrap.servers'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'localhost:9092'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'format'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'canal-json-inlong'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'canal-json-inlong.ignore-parse-errors'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'true'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'canal-json-inlong.map-null-key.mode'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'DROP'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'canal-json-inlong.encode.decimal-as-plain-number'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'true'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'canal-json-inlong.timestamp-format.standard'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'SQL'</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token string" style="color:rgb(255, 121, 198)">'canal-json-inlong.map-null-key.literal'</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">'null'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="42-field-t-described-in-sql">4.2 Field T Described in SQL<a href="#42-field-t-described-in-sql" class="hash-link" aria-label="Direct link to 4.2 Field T Described in SQL" title="Direct link to 4.2 Field T Described in SQL"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="421-filter-operator">4.2.1 Filter operator<a href="#421-filter-operator" class="hash-link" aria-label="Direct link to 4.2.1 Filter operator" title="Direct link to 4.2.1 Filter operator"></a></h4><p>See 4.1 node configuration for relevant configurations</p><p>The generated SQL is:</p><div class="language-sql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">INSERT</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">INTO</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">kafka_3</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">SELECT</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">name</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">AS</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">name</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">AS</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">FROM</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">mysql_1</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">WHERE</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token operator"><</span><span class="token plain"> </span><span class="token number">25</span><span class="token plain"> </span><span class="token operator">AND</span><span class="token plain"> </span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token identifier">age</span><span class="token identifier punctuation" style="color:rgb(248, 248, 242)">`</span><span class="token plain"> </span><span class="token operator">>=</span><span class="token plain"> </span><span class="token number">18</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="422-watermark">4.2.2 Watermark<a href="#422-watermark" class="hash-link" aria-label="Direct link to 4.2.2 Watermark" title="Direct link to 4.2.2 Watermark"></a></h4><p>The complete configuration of GroupInfo is as follows:</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">private Node buildMySqlExtractNode() {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FieldInfo> fields = Arrays.asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("name", new StringFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("ts", new TimestampFormatInfo()));</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> WatermarkField wk = new WatermarkField(new FieldInfo("ts", new TimestampFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new StringConstantParam("1"),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new TimeUnitConstantParam(TimeUnit.MINUTE));</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> return new MySqlExtractNode("1", "mysql_input", fields,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> wk, null, "id",</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Collections.singletonList("tableName"), "localhost", "root", "password",</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> "inlong", null, null,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> null, null);</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> private Node buildKafkaNode() {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FieldInfo> fields = Arrays.asList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("name", new StringFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("ts", new TimestampFormatInfo()));</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<FieldRelation> relations = Arrays</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .asList(new FieldRelation(new FieldInfo("name", new StringFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("name", new StringFormatInfo())),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldRelation(new FieldInfo("age", new IntFormatInfo()),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> new FieldInfo("age", new IntFormatInfo()))</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> );</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> return new KafkaLoadNode("2", "kafka_output", fields, relations, null, null,</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> "topic", "localhost:9092", new JsonFormat(),</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 1, null, "id");</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> private NodeRelation buildNodeRelation(List<Node> inputs, List<Node> outputs) {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<String> inputIds = inputs.stream().map(Node::getId).collect(Collectors.toList());</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> List<String> outputIds = outputs.stream().map(Node::getId).collect(Collectors.toList());</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> return new NodeRelation(inputIds, outputIds);</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @Override</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> public GroupInfo getTestObject() {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Node input = buildMySqlExtractNode();</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Node output = buildKafkaNode();</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> StreamInfo streamInfo = new StreamInfo("1", Arrays.asList(input, output), Collections.singletonList(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> buildNodeRelation(Collections.singletonList(input), Collections.singletonList(output))));</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> return new GroupInfo("1", Collections.singletonList(streamInfo));</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/apache-in-long">Apache InLong</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/sort">Sort</a></li></ul></div><div class="col margin-top--sm"><a href="https://github.com/apache/inlong-website/edit/master/blog/blog/2022-06-16-inlong-sort-etl.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/blog/2022/06/22/release-1.2.0"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">Release 1.2.0</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/blog/2022/04/25/release-1.1.0"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">Release 1.1.0</div></a></nav></main><div class="col col--2"><div class="tableOfContents_bqdL thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-background" class="table-of-contents__link toc-highlight">1. Background</a></li><li><a href="#2-introduction" class="table-of-contents__link toc-highlight">2. Introduction</a><ul><li><a href="#21-requirements" class="table-of-contents__link toc-highlight">2.1 Requirements</a></li><li><a href="#22-usage-scenarios" class="table-of-contents__link toc-highlight">2.2 Usage Scenarios</a></li><li><a href="#23-design-goal" class="table-of-contents__link toc-highlight">2.3 Design Goal</a></li><li><a href="#24-basic-concepts" class="table-of-contents__link toc-highlight">2.4 Basic Concepts</a></li><li><a href="#25-domain-model" class="table-of-contents__link toc-highlight">2.5 Domain Model</a></li><li><a href="#26-function-use-case-diagram" class="table-of-contents__link toc-highlight">2.6 Function Use-case Diagram</a></li></ul></li><li><a href="#3-system-outline-design" class="table-of-contents__link toc-highlight">3. System Outline Design</a><ul><li><a href="#31-system-architecture-diagram" class="table-of-contents__link toc-highlight">3.1 System Architecture Diagram</a></li><li><a href="#32-inlong-sort-internal-operation-flow-chart" class="table-of-contents__link toc-highlight">3.2 InLong Sort Internal Operation Flow Chart</a></li><li><a href="#33-module-design" class="table-of-contents__link toc-highlight">3.3 Module Design</a></li></ul></li><li><a href="#4-detailed-system-design" class="table-of-contents__link toc-highlight">4. Detailed System Design</a><ul><li><a href="#41-node-described-in-sql" class="table-of-contents__link toc-highlight">4.1 Node Described in SQL</a></li><li><a href="#42-field-t-described-in-sql" class="table-of-contents__link toc-highlight">4.2 Field T Described in SQL</a></li></ul></li></ul></div></div></div></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Events</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apachecon.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">ApacheCon<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="acevent" data-format="square" data-mode="dark" data-event="random"></a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://twitter.com/ApacheInlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://inlong.apache.org/img/apache-inlong-wechat.jpg" target="_blank" rel="noopener noreferrer" class="footer__link-item">WeChat<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="mailto:dev@inlong.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Email</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/blog">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright"><div style="font-family: Avenir-Medium;font-size: 14px;color: #999;"> |
