| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache InLong</b></a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/introduction">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/introduction">Next</a></li><li><a class="dropdown__link" href="/docs/introduction">1.11.0</a></li><li><a class="dropdown__link" href="/docs/1.10.0/introduction">1.10.0</a></li><li><a class="dropdown__link" href="/docs/1.9.0/introduction">1.9.0</a></li><li><a class="dropdown__link" href="/docs/1.8.0/introduction">1.8.0</a></li><li><a class="dropdown__link" href="/versions/">All versions</a></li></ul></div><a class="navbar__item navbar__link" href="/downloads">Download</a><a class="navbar__item navbar__link" href="/community/how-to-contribute">Community</a><a class="navbar__item navbar__link" href="/blog">Blog</a><a class="navbar__item navbar__link" href="/team">Team</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">ASF</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li><li><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Privacy</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/docs/1.7.0/modules/sort-standalone/overview" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li><li><a href="/zh-CN/docs/1.7.0/modules/sort-standalone/overview" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="zh-CN">简体中文</a></li></ul></div><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.7.0/introduction">Introduction</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/design_and_concept/basic_concept">Design and Concept</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/quick_start/how_to_build">Quick Start</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/deployment/standalone">Deployment</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/1.7.0/modules/agent/overview">Components</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/agent/overview">Agent</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/dataproxy/overview">DataProxy</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/tubemq/overview">TubeMQ</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/sort/overview">Sort</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/manager/overview">Manager</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/dashboard/overview">Dashboard</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/1.7.0/modules/sort-standalone/overview">Sort Standalone</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/1.7.0/modules/sort-standalone/overview">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.7.0/modules/sort-standalone/quick_start">Deployment</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.7.0/modules/sort-standalone/pulsar2hive">Pulsar2Hive Example</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.7.0/modules/sort-standalone/pulsar2es">Pulsar2Elasticsearch Example</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.7.0/modules/sort-standalone/pulsar2kafka">Pulsar2kafka Example</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.7.0/modules/audit/overview">Audit</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/data_node/extract_node/overview">Data Nodes</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/sdk/dataproxy-sdk/cpp">SDK</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/user_guide/dashboard_usage">User Guide</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/development/inlong_msg">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/administration/user_management">Administration</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.7.0/contact">Contact Us</a></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache InLong<!-- --> <b>1.7.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/modules/sort-standalone/overview">latest version</a></b> (<!-- -->1.11.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Components</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Sort Standalone</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.7.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction"></a></h2><p>Inlong sort standalone is a module responsible for consuming the data stream reported by users from the cache layer and distributing it to different data stores. It supports hive, elasticsearch, CLS and other data stores.<br> |
| <!-- -->If the consumption location at the time of offline is no longer within the life cycle of the cache layer, consumption starts from the latest location of the cache layer.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="message-queues-supported-by-the-cache-layer">message queues supported by the cache layer<a href="#message-queues-supported-by-the-cache-layer" class="hash-link" aria-label="Direct link to message queues supported by the cache layer" title="Direct link to message queues supported by the cache layer"></a></h3><ul><li>Inlong-Tube</li><li>Apache Pulsar</li><li>Apache Kafka</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="supported-data-storage">supported data storage<a href="#supported-data-storage" class="hash-link" aria-label="Direct link to supported data storage" title="Direct link to supported data storage"></a></h3><ul><li>Apache Hive (currently only supports sequence file format)</li><li>Apache Pulsar</li><li>Apache Kafka</li><li>Elasticsearch</li><li>ClickHouse</li><li>Tencent CLS</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="future-planning">Future planning<a href="#future-planning" class="hash-link" aria-label="Direct link to Future planning" title="Direct link to Future planning"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="support-more-kinds-of-data-storage">support more kinds of data storage<a href="#support-more-kinds-of-data-storage" class="hash-link" aria-label="Direct link to support more kinds of data storage" title="Direct link to support more kinds of data storage"></a></h4><p>HBase, etc.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="support-more-file-formats-written-to-apache-hive">support more file formats written to Apache hive<a href="#support-more-file-formats-written-to-apache-hive" class="hash-link" aria-label="Direct link to support more file formats written to Apache hive" title="Direct link to support more file formats written to Apache hive"></a></h4><p>Orc file, etc.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-in-sort-standalone">Configuration in Sort-Standalone<a href="#configuration-in-sort-standalone" class="hash-link" aria-label="Direct link to Configuration in Sort-Standalone" title="Direct link to Configuration in Sort-Standalone"></a></h2><p>The configuration in Sort-Standalone can be divided into three parts:</p><ul><li>Basic node configuration: including the type of source, channel, sink this node try to specify, and how to acquire the metadata of each group id and stream id. These configs will not be updated since the process starts. They will be put in <strong><em>common.properties</em></strong></li><li>Sink configuration: including the metadata of sinks, such as access point, username, password. And the metadata of each id which sinks require. These configs will be put in <strong><em>SortClusterConfig</em></strong></li><li>Source configuration: including the metadata of sources, such as mq type, topic, serviceUrl. These configs will be put in <strong><em>SortSourceConfig</em></strong></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-fileconfcommonproperties">Configuration file:conf/common.properties<a href="#configuration-fileconfcommonproperties" class="hash-link" aria-label="Direct link to Configuration file:conf/common.properties" title="Direct link to Configuration file:conf/common.properties"></a></h3><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>clusterId</td><td>Y</td><td>NA</td><td>inlong-sort-standalone cluster id</td></tr><tr><td>sortSource.type</td><td>N</td><td>org.apache.inlong.sort.standalone.source.readapi.SortSdkSource</td><td>Source class name</td></tr><tr><td>sortChannel.type</td><td>N</td><td>org.apache.inlong.sort.standalone.channel.BufferQueueChannel</td><td>Channel class name</td></tr><tr><td>sortSink.type</td><td>N</td><td>org.apache.inlong.sort.standalone.sink.hive.HiveSink</td><td>Sink class name</td></tr><tr><td>sortClusterConfig.type</td><td>Y</td><td>manager</td><td>ClusterConfig configuration type. File config mode please choose: <strong><em>file</em></strong></td></tr><tr><td>sortSourceConfig.QueryConsumeConfigType</td><td>Y</td><td>manager</td><td>SourceConfig configuration type. File config mode please choose: <strong><em>file</em></strong></td></tr><tr><td>sortClusterConfig.file</td><td>N</td><td>SortClusterConfig.conf</td><td>Sort cluster config file name. <strong><em>Required in file config type.</em></strong></td></tr><tr><td>managerUrlLoaderType</td><td>N</td><td>org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader</td><td>The type of manager url loader. <strong><em>Required in manager config type.</em></strong></td></tr><tr><td>sortClusterConfig.managerUrl</td><td>N</td><td>http://${manager_ip}:{manager_port}/inlong/manager/openapi/sort/getClusterConfig</td><td>The manager config interface to cluster config. <strong><em>Required in manager config type.</em></strong></td></tr><tr><td>sortSourceConfig.managerUrl</td><td>N</td><td>http://${manager_ip}:{manager_port}/inlong/manager/openapi/sort/getSortSource</td><td>The manager config interface to sort source config. <strong><em>Required in manager config mode.</em></strong></td></tr><tr><td>eventFormatHandler</td><td>N</td><td>org.apache.inlong.sort.standalone.sink.hive.DefaultEventFormatHandler</td><td>Formatter class name</td></tr><tr><td>maxThreads</td><td>N</td><td>10</td><td>sink thread number</td></tr><tr><td>reloadInterval</td><td>N</td><td>60000</td><td>interval updating Configuration data(millisecond)</td></tr><tr><td>processInterval</td><td>N</td><td>100</td><td>interval processing data(millisecond)</td></tr><tr><td>metricDomains</td><td>N</td><td>Sort</td><td>domain name of metric</td></tr><tr><td>metricDomains.Sort.domainListeners</td><td>N</td><td>org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener</td><td>class name list of metric listener, separated by space</td></tr><tr><td>prometheusHttpPort</td><td>N</td><td>8080</td><td>HTTP server port of prometheus simple client</td></tr><tr><td>metricDomains.Sort.snapshotInterval</td><td>N</td><td>60000</td><td>interval snapshoting metric data(millisecond)</td></tr><tr><td>sortsdk.consumeStrategy</td><td>N</td><td>latest</td><td>The consume strategy of topic, other option is earliest</td></tr><tr><td>interceptor.type</td><td>N</td><td>org.apache.inlong.sort.standalone.rollback.TimeBasedFilterInterceptor$Builder</td><td>Type of interceptor used in sort-standalone. The default interceptor is based on message time.</td></tr><tr><td>rollback.startTime</td><td>N</td><td>1970-01-01 08:00:00</td><td>Valid message start time. Earlier message will be filtered.</td></tr><tr><td>rollback.stopTime</td><td>N</td><td>NA</td><td>Valid message stop time. Later message will be filtered.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sortclusterconfig">SortClusterConfig<a href="#sortclusterconfig" class="hash-link" aria-label="Direct link to SortClusterConfig" title="Direct link to SortClusterConfig"></a></h3><p>The SortClusterConfig can be acquired from local file or manager, depending on the <strong><em>sortClusterConfig.type</em></strong> configured in common.properties.</p><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>clusterName</td><td>Y</td><td>NA</td><td>inlong-sort-standalone cluster id</td></tr><tr><td>sortTasks</td><td>Y</td><td>NA</td><td>Sort task list</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="sorttask">SortTask<a href="#sorttask" class="hash-link" aria-label="Direct link to SortTask" title="Direct link to SortTask"></a></h4><p>Sort Task includes <strong><em>idParams and sinkParams</em></strong> which represent the config of <strong><em>stream and dataNode</em></strong> respectively</p><p>The idParams and sinkParams are totally different among each type of sort tasks. There are two examples of hive task and pulsar task</p><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>name</td><td>Y</td><td>NA</td><td>sort task name</td></tr><tr><td>type</td><td>Y</td><td>NA</td><td>sort task type, for example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n")</td></tr><tr><td>idParams</td><td>Y</td><td>NA</td><td>List of Inlong DataStream configuration</td></tr><tr><td>sinkParams</td><td>Y</td><td>NA</td><td>Sort task parameters</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="idparams-content-of-hive-sort-task">idParams content of Hive sort task<a href="#idparams-content-of-hive-sort-task" class="hash-link" aria-label="Direct link to idParams content of Hive sort task" title="Direct link to idParams content of Hive sort task"></a></h4><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>inlongGroupId</td><td>Y</td><td>NA</td><td>inlongGroupId</td></tr><tr><td>inlongStreamId</td><td>Y</td><td>NA</td><td>inlongStreamId</td></tr><tr><td>separator</td><td>Y</td><td>NA</td><td>separator</td></tr><tr><td>partitionIntervalMs</td><td>N</td><td>3600000</td><td>partition interval(millisecond)</td></tr><tr><td>idRootPath</td><td>Y</td><td>NA</td><td>HDFS root path of Inlong DataStream</td></tr><tr><td>partitionSubPath</td><td>Y</td><td>NA</td><td>partition sub path of Inlong DataStream</td></tr><tr><td>hiveTableName</td><td>Y</td><td>NA</td><td>Hive table name of Inlong DataStream</td></tr><tr><td>partitionFieldName</td><td>N</td><td>dt</td><td>partition field name of Inlong DataStream</td></tr><tr><td>partitionFieldPattern</td><td>Y</td><td>NA</td><td>Date format of partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm}</td></tr><tr><td>msgTimeFieldPattern</td><td>Y</td><td>NA</td><td>Date format of message generation time, it support Java date format</td></tr><tr><td>maxPartitionOpenDelayHour</td><td>N</td><td>8</td><td>Max delay hour of partition(hour)</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="sinkparams-content-of-hive-sort-task">sinkParams content of Hive sort task<a href="#sinkparams-content-of-hive-sort-task" class="hash-link" aria-label="Direct link to sinkParams content of Hive sort task" title="Direct link to sinkParams content of Hive sort task"></a></h4><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>hdfsPath</td><td>Y</td><td>NA</td><td>NameNode URL of HDFS</td></tr><tr><td>maxFileOpenDelayMinute</td><td>N</td><td>5</td><td>Max writing delay minute of simple HDFS file(minute)</td></tr><tr><td>tokenOvertimeMinute</td><td>N</td><td>60</td><td>token overtime of Inlong Data Stream(minute)</td></tr><tr><td>maxOutputFileSizeGb</td><td>N</td><td>2</td><td>Max file size of simple HDFS file(GB)</td></tr><tr><td>hiveJdbcUrl</td><td>Y</td><td>NA</td><td>JDBC URL of Hive</td></tr><tr><td>hiveDatabase</td><td>Y</td><td>NA</td><td>Hive database</td></tr><tr><td>hiveUsername</td><td>Y</td><td>NA</td><td>Hive username</td></tr><tr><td>hivePassword</td><td>Y</td><td>NA</td><td>Hive password</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="idparams-content-of-pulsar-sort-task">idParams content of Pulsar sort task<a href="#idparams-content-of-pulsar-sort-task" class="hash-link" aria-label="Direct link to idParams content of Pulsar sort task" title="Direct link to idParams content of Pulsar sort task"></a></h4><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>inlongGroupId</td><td>Y</td><td>NA</td><td>inlongGroupId</td></tr><tr><td>inlongStreamId</td><td>Y</td><td>NA</td><td>inlongStreamId</td></tr><tr><td>topic</td><td>Y</td><td>NA</td><td>Topic of MQ</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="sinkparams-content-of-pulsar-sort-task">sinkParams content of Pulsar sort task<a href="#sinkparams-content-of-pulsar-sort-task" class="hash-link" aria-label="Direct link to sinkParams content of Pulsar sort task" title="Direct link to sinkParams content of Pulsar sort task"></a></h4><table><thead><tr><th>Parameter</th><th>Required</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>serviceUrl</td><td>Y</td><td>NA</td><td>Pulsar service URL</td></tr><tr><td>authentication</td><td>Y</td><td>NA</td><td>Pulsar authentication</td></tr><tr><td>enableBatching</td><td>N</td><td>true</td><td>enableBatching</td></tr><tr><td>batchingMaxBytes</td><td>N</td><td>5242880</td><td>batchingMaxBytes</td></tr><tr><td>batchingMaxMessages</td><td>N</td><td>3000</td><td>batchingMaxMessages</td></tr><tr><td>batchingMaxPublishDelay</td><td>N</td><td>1</td><td>batchingMaxPublishDelay</td></tr><tr><td>maxPendingMessages</td><td>N</td><td>1000</td><td>maxPendingMessages</td></tr><tr><td>maxPendingMessagesAcrossPartitions</td><td>N</td><td>50000</td><td>maxPendingMessagesAcrossPartitions</td></tr><tr><td>sendTimeout</td><td>N</td><td>0</td><td>sendTimeout</td></tr><tr><td>compressionType</td><td>N</td><td>NONE</td><td>compressionType</td></tr><tr><td>blockIfQueueFull</td><td>N</td><td>true</td><td>blockIfQueueFull</td></tr><tr><td>roundRobinRouterBatchingPartitionSwitchFrequency</td><td>N</td><td>10</td><td>roundRobinRouterBatchingPartitionSwitchFrequency</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sortsourceconfig">SortSourceConfig<a href="#sortsourceconfig" class="hash-link" aria-label="Direct link to SortSourceConfig" title="Direct link to SortSourceConfig"></a></h3><table><thead><tr><th>Parameter</th><th>Required</th><th>Type</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>sortClusterName</td><td>Y</td><td>String</td><td>NA</td><td>inlong-sort-standalone cluster id</td></tr><tr><td>sortTaskId</td><td>Y</td><td>String</td><td>NA</td><td>Sort task name</td></tr><tr><td>cacheZones</td><td>Y</td><td>JsonObject<!-- --><<!-- -->String, JsonObject<!-- -->></td><td>NA</td><td>Cache cluster list, Map<!-- --><<!-- -->cacheClusterName, CacheCluster<!-- -->></td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="cachezones">CacheZones<a href="#cachezones" class="hash-link" aria-label="Direct link to CacheZones" title="Direct link to CacheZones"></a></h4><table><thead><tr><th>Parameter</th><th>Required</th><th>Type</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>zoneName</td><td>Y</td><td>String</td><td>NA</td><td>cache cluster name</td></tr><tr><td>zoneType</td><td>Y</td><td>String</td><td>NA</td><td>[pulsar,tube,kafka]</td></tr><tr><td>serviceUrl</td><td>Y</td><td>String</td><td>NA</td><td>Pulsar serviceUrl or Kafka broker list</td></tr><tr><td>authentication</td><td>Y</td><td>String</td><td>NA</td><td>Pulsar authentication</td></tr><tr><td>cacheZoneProperties</td><td>N</td><td>Map<!-- --><<!-- -->String,String<!-- -->></td><td>NA</td><td>Cache consumer configuration</td></tr><tr><td>topics</td><td>N</td><td>List<!-- --><<!-- -->Topic<!-- -->></td><td>NA</td><td>Topic list of Cache consumer</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="topics">Topics<a href="#topics" class="hash-link" aria-label="Direct link to Topics" title="Direct link to Topics"></a></h4><table><thead><tr><th>Parameter</th><th>Required</th><th>Type</th><th>DefaultValue</th><th>Remark</th></tr></thead><tbody><tr><td>topic</td><td>Y</td><td>String</td><td>NA</td><td>cache topic name</td></tr><tr><td>partitionCnt</td><td>Y</td><td>Integer</td><td>NA</td><td>cache topic partition count</td></tr><tr><td>topicProperties</td><td>N</td><td>Map<!-- --><<!-- -->String,String<!-- -->></td><td>NA</td><td>Cache topic configuration</td></tr></tbody></table></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/inlong-website/edit/master/versioned_docs/version-1.7.0/modules/sort-standalone/overview.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/1.7.0/modules/dashboard/quick_start"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Deployment</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/1.7.0/modules/sort-standalone/quick_start"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Deployment</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a></li><li><a href="#feature" class="table-of-contents__link toc-highlight">Feature</a><ul><li><a href="#multi-tenant-system" class="table-of-contents__link toc-highlight">Multi tenant system</a></li><li><a href="#distribution-tasks-support-dynamic-updates" class="table-of-contents__link toc-highlight">Distribution tasks support dynamic updates</a></li><li><a href="#message-queues-supported-by-the-cache-layer" class="table-of-contents__link toc-highlight">message queues supported by the cache layer</a></li><li><a href="#supported-data-storage" class="table-of-contents__link toc-highlight">supported data storage</a></li><li><a href="#future-planning" class="table-of-contents__link toc-highlight">Future planning</a></li></ul></li><li><a href="#configuration-in-sort-standalone" class="table-of-contents__link toc-highlight">Configuration in Sort-Standalone</a><ul><li><a href="#configuration-fileconfcommonproperties" class="table-of-contents__link toc-highlight">Configuration file:conf/common.properties</a></li><li><a href="#sortclusterconfig" class="table-of-contents__link toc-highlight">SortClusterConfig</a></li><li><a href="#sortsourceconfig" class="table-of-contents__link toc-highlight">SortSourceConfig</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Events</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apachecon.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">ApacheCon<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="acevent" data-format="square" data-mode="dark" data-event="random"></a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://twitter.com/ApacheInlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://inlong.apache.org/img/apache-inlong-wechat.jpg" target="_blank" rel="noopener noreferrer" class="footer__link-item">WeChat<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="mailto:dev@inlong.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Email</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/blog">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright"><div style="font-family: Avenir-Medium;font-size: 14px;color: #999;"> |
