| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-1.9.0 plugin-docs plugin-id-default docs-doc-id-modules/audit/overview"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.3.1"> |
| <title data-rh="true">Overview | Apache InLong</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://inlong.apache.org/docs/1.9.0/modules/audit/overview"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="1.9.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-1.9.0"><meta data-rh="true" name="docsearch:version" content="1.9.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-1.9.0"><meta data-rh="true" property="og:title" content="Overview | Apache InLong"><meta data-rh="true" name="description" content="InLong audit is a subsystem independent of InLong, which performs real-time audit and reconciliation on the incoming and outgoing traffic of the Agent, DataProxy, and Sort modules of the InLong system."><meta data-rh="true" property="og:description" content="InLong audit is a subsystem independent of InLong, which performs real-time audit and reconciliation on the incoming and outgoing traffic of the Agent, DataProxy, and Sort modules of the InLong system."><link data-rh="true" rel="icon" href="/img/logo.svg"><link data-rh="true" rel="canonical" href="https://inlong.apache.org/docs/1.9.0/modules/audit/overview"><link data-rh="true" rel="alternate" href="https://inlong.apache.org/docs/1.9.0/modules/audit/overview" hreflang="en"><link data-rh="true" rel="alternate" href="https://inlong.apache.org/zh-CN/docs/1.9.0/modules/audit/overview" hreflang="zh-CN"><link data-rh="true" rel="alternate" href="https://inlong.apache.org/docs/1.9.0/modules/audit/overview" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://YUW9QEL53E-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache InLong RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache InLong Atom Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache InLong" href="/opensearch.xml"> |
| |
| |
| |
| <script src="https://www.apachecon.com/event-images/snippet.js" async></script><link rel="stylesheet" href="/assets/css/styles.c64edd51.css"> |
| <link rel="preload" href="/assets/js/runtime~main.63c98e82.js" as="script"> |
| <link rel="preload" href="/assets/js/main.070aef2a.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function e(e){document.documentElement.setAttribute("data-theme",e)}var t=function(){var e=null;try{e=localStorage.getItem("theme")}catch(e){}return e}();null!==t?e(t):window.matchMedia("(prefers-color-scheme: dark)").matches?e("dark"):(window.matchMedia("(prefers-color-scheme: light)").matches,e("light"))}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache InLong</b></a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/introduction">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/introduction">Next</a></li><li><a class="dropdown__link" href="/docs/introduction">1.11.0</a></li><li><a class="dropdown__link" href="/docs/1.10.0/introduction">1.10.0</a></li><li><a class="dropdown__link" href="/docs/1.9.0/introduction">1.9.0</a></li><li><a class="dropdown__link" href="/docs/1.8.0/introduction">1.8.0</a></li><li><a class="dropdown__link" href="/versions/">All versions</a></li></ul></div><a class="navbar__item navbar__link" href="/downloads">Download</a><a class="navbar__item navbar__link" href="/community/how-to-contribute">Community</a><a class="navbar__item navbar__link" href="/blog">Blog</a><a class="navbar__item navbar__link" href="/team">Team</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">ASF</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li><li><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Privacy</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/docs/1.9.0/modules/audit/overview" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li><li><a href="/zh-CN/docs/1.9.0/modules/audit/overview" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="zh-CN">简体中文</a></li></ul></div><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.9.0/introduction">Introduction</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/design_and_concept/basic_concept">Design and Concept</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/quick_start/how_to_build">Quick Start</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/deployment/standalone">Deployment</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/1.9.0/modules/agent/overview">Components</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/agent/overview">Agent</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/dataproxy/overview">DataProxy</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/tubemq/overview">TubeMQ</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/sort/overview">Sort</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/manager/overview">Manager</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/dashboard/overview">Dashboard</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/docs/1.9.0/modules/sort-standalone/overview">Sort Standalone</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/docs/1.9.0/modules/audit/overview">Audit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/1.9.0/modules/audit/overview">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.9.0/modules/audit/configure">Configuration</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.9.0/modules/audit/quick_start">Deployment</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/data_node/extract_node/overview">Data Nodes</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/sdk/dataproxy-sdk/cpp">SDK</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/user_guide/dashboard_usage">User Guide</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/development/inlong_msg">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.9.0/administration/user_management">Administration</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.9.0/contact">Contact Us</a></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache InLong<!-- --> <b>1.9.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/modules/audit/overview">latest version</a></b> (<!-- -->1.11.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Components</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Audit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.9.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><p>InLong audit is a subsystem independent of InLong, which performs real-time audit and reconciliation on the incoming and outgoing traffic of the Agent, DataProxy, and Sort modules of the InLong system. |
| There are three granularities for reconciliation: minutes, hours, and days.</p><p>The audit reconciliation is based on the log reporting time, and each service participating in the audit will conduct real-time reconciliation according to the same log time. Through audit reconciliation, we can clearly understand InLong |
| The transmission status of each module, and whether the data stream is lost or repeated</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="architecture">Architecture<a href="#architecture" class="hash-link" aria-label="Direct link to Architecture" title="Direct link to Architecture"></a></h2><p><img loading="lazy" src="/assets/images/audit_architecture-8402cbb7c4906ca8cf7d26c57c204dbb.png" width="1260" height="213" class="img_ev3q"></p><ol><li>The audit SDK is nested in the service that needs to be audited, audits the service, and sends the audit result to the audit access layer</li><li>The audit proxy writes audit data to MQ (Pulsar, Kafka or TubeMQ)</li><li>The distribution service consumes the audit data of MQ, and writes the audit data to MySQL, Elasticsearch and ClickHouse.</li><li>The interface layer encapsulates the data of MySQL, Elasticsearch and ClickHouse.</li><li>Application scenarios mainly include report display, audit reconciliation, etc.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="audit-dimension">Audit Dimension<a href="#audit-dimension" class="hash-link" aria-label="Direct link to Audit Dimension" title="Direct link to Audit Dimension"></a></h2><table><thead><tr><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead><tbody><tr><td>Machine ip</td><td>Container ID</td><td>Thread ID</td><td>Log time (minutes)</td><td>Audit ID</td><td>inlong_group_id</td><td>inlong_stream_id</td><td>Number of records</td><td>Size</td><td>Transmission delay (ms)</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="audit-id">Audit ID<a href="#audit-id" class="hash-link" aria-label="Direct link to Audit ID" title="Direct link to Audit ID"></a></h2><p>The receiving and sending of each module are respectively an independent audit item ID</p><table><thead><tr><th>Inlong Service Module</th><th>Audit ID</th></tr></thead><tbody><tr><td>Inlong API Received Successfully</td><td>1</td></tr><tr><td>Inlong API Send Successfully</td><td>2</td></tr><tr><td>Inlong Agent Received Successfully</td><td>3</td></tr><tr><td>Inlong Agent Send Successfully</td><td>4</td></tr><tr><td>Inlong DataProxy Received Successfully</td><td>5</td></tr><tr><td>Inlong DataProxy Send Successfully</td><td>6</td></tr><tr><td>Inlong Sort Received Successfully</td><td>7</td></tr><tr><td>Inlong Sort Send Successfully</td><td>8</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-transfer-protocol">Data Transfer Protocol<a href="#data-transfer-protocol" class="hash-link" aria-label="Direct link to Data Transfer Protocol" title="Direct link to Data Transfer Protocol"></a></h2><p>The transmission protocol between sdk, access layer, and distribution layer is Protocol Buffers</p><div class="language-markdown codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-markdown codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">syntax = "proto3";</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">package org.apache.inlong.audit.protocol;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message BaseCommand {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> enum Type {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> PING = 0;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> PONG = 1;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> AUDITREQUEST = 2;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> AUDITREPLY = 3;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Type type = 1;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> optional AuditRequest audit_request = 2;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> optional AuditReply audit_reply = 3;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> optional Ping ping = 4;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> optional Pong pong = 5;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message Ping {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message Pong {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message AuditRequest {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> AuditMessageHeader msg_header = 1; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> repeated AuditMessageBody msg_body = 2; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message AuditMessageHeader {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> string ip = 1; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> string docker_id = 2; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> string thread_id = 3; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> uint64 sdk_ts = 4; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> uint64 packet_id = 5; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message AuditMessageBody {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> uint64 log_ts = 1; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> string inlong_group_id= 2; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> string inlong_stream_id= 3; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> string audit_id = 4; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> uint64 count = 5; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> uint64 size = 6; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> int64 delay = 7; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">message AuditReply {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> enum RSP_CODE {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> SUCCESS = 0; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> FAILED = 1; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> DISASTER = 2; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> RSP_CODE rsp_code = 1; </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> optional string message = 2;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="audit-sdk-implementation-details">Audit SDK Implementation Details<a href="#audit-sdk-implementation-details" class="hash-link" aria-label="Direct link to Audit SDK Implementation Details" title="Direct link to Audit SDK Implementation Details"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="target">Target<a href="#target" class="hash-link" aria-label="Direct link to Target" title="Direct link to Target"></a></h3><p><strong><em>1. Support local disaster recovery</em></strong> |
| <strong><em>2. Data Uniqueness</em></strong> |
| <strong><em>3. Reduce data loss caused by abnormal restart</em></strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram">Main Logic Diagram<a href="#main-logic-diagram" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p><img loading="lazy" src="/assets/images/audit_sdk-76111ecb81d46749cf0daad888a756a0.png" width="848" height="1444" class="img_ev3q"></p><ol><li>The sdk provides the add interface externally. The parameters are: audit_id, inlong_group_id, inlong_stream_id, number, size.</li><li>The sdk uses log time+audit_id+inlong_group_id+inlong_stream_id as the key to perform real-time statistics.</li><li>When the sending cycle is satisfied or the business program is actively triggered, the SDK will package the statistical results with the PB protocol and send the audit access layer.</li><li>If (4) fails to send, put it into the failure queue, and continue to send in the next cycle.</li><li>When the failure queue is greater than the threshold, perform disaster recovery through local files.</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="service-discovery">Service Discovery<a href="#service-discovery" class="hash-link" aria-label="Direct link to Service Discovery" title="Direct link to Service Discovery"></a></h3><p>Audit name discovery between sdk and access layer, support plug-in, including domain name, vip, etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="disaster-recovery">Disaster Recovery<a href="#disaster-recovery" class="hash-link" aria-label="Direct link to Disaster Recovery" title="Direct link to Disaster Recovery"></a></h3><p><img loading="lazy" src="/assets/images/audit_sdk_disaster_recovery-4c48c3a8e253ca1ca7dc99a2be2a4e48.png" width="280" height="824" class="img_ev3q"></p><ol><li>When the SDK fails to send the access layer, it will be placed in the failure queue.</li><li>When the failure queue reaches the threshold, it will be written to the local disaster recovery file.</li><li>When the local disaster recovery file reaches the threshold, the old data will be eliminated (eliminated by time).</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="access-layer-implementation">Access layer Implementation<a href="#access-layer-implementation" class="hash-link" aria-label="Direct link to Access layer Implementation" title="Direct link to Access layer Implementation"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="target-1">Target<a href="#target-1" class="hash-link" aria-label="Direct link to Target" title="Direct link to Target"></a></h3><p><strong><em>1.High reliability</em></strong><br> |
| <strong><em>2.at least once</em></strong> </p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram-1">Main Logic Diagram<a href="#main-logic-diagram-1" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p><img loading="lazy" src="/assets/images/audit_proxy-da4ab5907594ab9e46b70645c9a0ea91.png" width="1192" height="424" class="img_ev3q"></p><ol><li>After the access layer receives the packet sent by the sdk, it writes the message queue.</li><li>After writing the message queue successfully, return success to the sdk.</li><li>The data protocol of the message queue is the PB protocol.</li><li>Set the ack of the write message queue to -1 or all.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="elasticsearch-distribution-implementation">Elasticsearch Distribution Implementation<a href="#elasticsearch-distribution-implementation" class="hash-link" aria-label="Direct link to Elasticsearch Distribution Implementation" title="Direct link to Elasticsearch Distribution Implementation"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="target-2">Target<a href="#target-2" class="hash-link" aria-label="Direct link to Target" title="Direct link to Target"></a></h3><p><strong><em>1. High real-time performance (minute level)</em></strong> |
| <strong><em>2. Can operate tens of billions of audit data per day</em></strong> |
| <strong><em>3. Can be deduplicated</em></strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram-2">Main Logic Diagram<a href="#main-logic-diagram-2" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p><img loading="lazy" src="/assets/images/elasticsearch_overview-baeacc406471a715846369f35fe0d4d7.png" width="1289" height="508" class="img_ev3q"></p><ol><li>Distribution service AuditDds consumes messages in real time.</li><li>According to the audit ID in the audit data, route the data to the corresponding Elasticsearch cluster.</li><li>Each audit ID corresponds to an Elasticsearch index.</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="elasticsearch-index-design">Elasticsearch Index Design<a href="#elasticsearch-index-design" class="hash-link" aria-label="Direct link to Elasticsearch Index Design" title="Direct link to Elasticsearch Index Design"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="index-name">Index Name<a href="#index-name" class="hash-link" aria-label="Direct link to Index Name" title="Direct link to Index Name"></a></h4><p>The index name consists of date + audit item ID, such as 20211019_1, 20211019_2.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="index-field-schema">Index Field Schema<a href="#index-field-schema" class="hash-link" aria-label="Direct link to Index Field Schema" title="Direct link to Index Field Schema"></a></h4><table><thead><tr><th>field</th><th>type</th><th>instruction</th></tr></thead><tbody><tr><td>audit_id</td><td>keyword</td><td>Audit ID</td></tr><tr><td>inlong_group_id</td><td>keyword</td><td>inlong_group_id</td></tr><tr><td>inlong_stream_id</td><td>keyword</td><td>inlong_stream_id</td></tr><tr><td>docker_id</td><td>keyword</td><td>ID of the container where the dk is located</td></tr><tr><td>thread_id</td><td>keyword</td><td>thread ID</td></tr><tr><td>packet_id</td><td>keyword</td><td>Package ID reported by sdk</td></tr><tr><td>ip</td><td>keyword</td><td>Machine IP</td></tr><tr><td>log_ts</td><td>keyword</td><td>log time</td></tr><tr><td>sdk_ts</td><td>long</td><td>Audit SDK reporting time</td></tr><tr><td>count</td><td>long</td><td>Number of logs</td></tr><tr><td>size</td><td>long</td><td>size of log</td></tr><tr><td>delay</td><td>long</td><td>The log transfer time, equal to the current machine time minus the log time</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="elasticsearch-index-storage-period">Elasticsearch Index Storage Period<a href="#elasticsearch-index-storage-period" class="hash-link" aria-label="Direct link to Elasticsearch Index Storage Period" title="Direct link to Elasticsearch Index Storage Period"></a></h4><p>Storage by day, storage period is dynamically configurable</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="elasticsearch-write-design">Elasticsearch Write Design<a href="#elasticsearch-write-design" class="hash-link" aria-label="Direct link to Elasticsearch Write Design" title="Direct link to Elasticsearch Write Design"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="the-relationship-between-inlong_group_id-inlong_stream_id-audit-id-and-elasticsearch-index">The relationship between inlong_group_id, inlong_stream_id, audit ID and Elasticsearch index<a href="#the-relationship-between-inlong_group_id-inlong_stream_id-audit-id-and-elasticsearch-index" class="hash-link" aria-label="Direct link to The relationship between inlong_group_id, inlong_stream_id, audit ID and Elasticsearch index" title="Direct link to The relationship between inlong_group_id, inlong_stream_id, audit ID and Elasticsearch index"></a></h3><p><img loading="lazy" src="/assets/images/elasticsearch_index-e91afcde5316b4c42de87b9e3a0e363d.png" width="1040" height="258" class="img_ev3q"> |
| The relationship between inlong_group_id, inlong_stream_id, audit ID and Elasticsearch index is 1:N in system design and service implementation</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-routing-policy">Write Routing Policy<a href="#write-routing-policy" class="hash-link" aria-label="Direct link to Write Routing Policy" title="Direct link to Write Routing Policy"></a></h3><p><img loading="lazy" src="/assets/images/elasticsearch_write-4408c917c9a09304298f43924a25a625.png" width="1920" height="259" class="img_ev3q"> |
| Use inlong_group_id and inlong_stream_id to route to Elasticsearch shards to ensure that the same inlong_group_id and inlong_stream_id are stored in the same shard |
| When writing the same inlong_group_id and inlong_stream_id to the same shard, when querying and aggregating, only one shard needs to be processed, which can greatly improve performance</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="optional-deduplicationby-doc_id">Optional DeduplicationBy doc_id<a href="#optional-deduplicationby-doc_id" class="hash-link" aria-label="Direct link to Optional DeduplicationBy doc_id" title="Direct link to Optional DeduplicationBy doc_id"></a></h3><p>Elasticsearch is resource-intensive for real-time deduplication. This function is optional through configuration.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-bulk-batch-method">Use bulk batch method<a href="#use-bulk-batch-method" class="hash-link" aria-label="Direct link to Use bulk batch method" title="Direct link to Use bulk batch method"></a></h3><p>Use bulk to write, each batch of 5000, improve the write performance of the Elasticsearch cluster</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mysql-distribution-implementation">MySQL Distribution Implementation<a href="#mysql-distribution-implementation" class="hash-link" aria-label="Direct link to MySQL Distribution Implementation" title="Direct link to MySQL Distribution Implementation"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="target-3">Target<a href="#target-3" class="hash-link" aria-label="Direct link to Target" title="Direct link to Target"></a></h3><p><strong><em>1. High real-time performance (minute level)</em></strong><br> |
| <strong><em>2. Simple to deploy</em></strong><br> |
| <strong><em>3. Can be deduplicated</em></strong> </p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram-3">Main Logic Diagram<a href="#main-logic-diagram-3" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p><img loading="lazy" src="/assets/images/audit_mysql-527cb86d7d377b0a6e944b2ee2930431.png" width="1050" height="250" class="img_ev3q"> |
| MySQL distribution supports distribution to different MySQL instances according to the audit ID, and supports horizontal expansion.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage-introduction">Usage introduction<a href="#usage-introduction" class="hash-link" aria-label="Direct link to Usage introduction" title="Direct link to Usage introduction"></a></h3><ol><li>When the audit scale of the business is relatively small, less than ten million per day, you can consider using MySQL as the audit storage. Because the deployment of MySQL is much simpler than that of Elasticsearch, the resource cost will be much less.</li><li>If the scale of audit data is large and MySQL cannot support it, you can consider using Elasticsearch as storage. After all, a single Elasticsearch cluster can support tens of billions of audit data and horizontal expansion.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="clickhouse-distribution-implementation">ClickHouse Distribution Implementation<a href="#clickhouse-distribution-implementation" class="hash-link" aria-label="Direct link to ClickHouse Distribution Implementation" title="Direct link to ClickHouse Distribution Implementation"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="target-4">Target<a href="#target-4" class="hash-link" aria-label="Direct link to Target" title="Direct link to Target"></a></h3><p><strong><em>1. High real-time performance (minute level)</em></strong><br> |
| <strong><em>2. Simple to deploy</em></strong><br> |
| <strong><em>3. Can be deduplicated</em></strong> </p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram-4">Main Logic Diagram<a href="#main-logic-diagram-4" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p>ClickHouse distribution supports distribution to different ClickHouse instances according to the audit ID, and supports horizontal expansion.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage-introduction-1">Usage introduction<a href="#usage-introduction-1" class="hash-link" aria-label="Direct link to Usage introduction" title="Direct link to Usage introduction"></a></h3><ol><li>When the audit scale of the business is huge and you want to use SQL to access audit data, you can consider using ClickHouse as the audit storage. Because ClickHouse support SQL accessing, and support tens of billions of audit data and horizontal expansion.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="audit-usage-interface-design">Audit Usage Interface Design<a href="#audit-usage-interface-design" class="hash-link" aria-label="Direct link to Audit Usage Interface Design" title="Direct link to Audit Usage Interface Design"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram-5">Main Logic Diagram<a href="#main-logic-diagram-5" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p><img loading="lazy" src="/assets/images/audit_api-2fa08b39a7e31207a2cd8bd104525901.png" width="1718" height="241" class="img_ev3q"> |
| The audit interface layer uses SQL to check MySQL/ClickHouse or restful to check Elasticsearch. How to check which type of storage the interface uses depends on which type of storage is used.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ui-interface-display">UI Interface Display<a href="#ui-interface-display" class="hash-link" aria-label="Direct link to UI Interface Display" title="Direct link to UI Interface Display"></a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="main-logic-diagram-6">Main Logic Diagram<a href="#main-logic-diagram-6" class="hash-link" aria-label="Direct link to Main Logic Diagram" title="Direct link to Main Logic Diagram"></a></h3><p><img loading="lazy" src="/assets/images/audit_ui-51d52c5abdbf1615d329fe78b407e151.png" width="1735" height="931" class="img_ev3q"> |
| The front-end page pulls the audit data of each module through the interface layer and displays it.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/inlong-website/edit/master/versioned_docs/version-1.9.0/modules/audit/overview.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/1.9.0/modules/sort-standalone/pulsar2kafka"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Pulsar2kafka Example</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/1.9.0/modules/audit/configure"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Configuration</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#audit-dimension" class="table-of-contents__link toc-highlight">Audit Dimension</a></li><li><a href="#audit-id" class="table-of-contents__link toc-highlight">Audit ID</a></li><li><a href="#data-transfer-protocol" class="table-of-contents__link toc-highlight">Data Transfer Protocol</a></li><li><a href="#audit-sdk-implementation-details" class="table-of-contents__link toc-highlight">Audit SDK Implementation Details</a><ul><li><a href="#target" class="table-of-contents__link toc-highlight">Target</a></li><li><a href="#main-logic-diagram" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li><li><a href="#service-discovery" class="table-of-contents__link toc-highlight">Service Discovery</a></li><li><a href="#disaster-recovery" class="table-of-contents__link toc-highlight">Disaster Recovery</a></li></ul></li><li><a href="#access-layer-implementation" class="table-of-contents__link toc-highlight">Access layer Implementation</a><ul><li><a href="#target-1" class="table-of-contents__link toc-highlight">Target</a></li><li><a href="#main-logic-diagram-1" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li></ul></li><li><a href="#elasticsearch-distribution-implementation" class="table-of-contents__link toc-highlight">Elasticsearch Distribution Implementation</a><ul><li><a href="#target-2" class="table-of-contents__link toc-highlight">Target</a></li><li><a href="#main-logic-diagram-2" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li><li><a href="#elasticsearch-index-design" class="table-of-contents__link toc-highlight">Elasticsearch Index Design</a></li></ul></li><li><a href="#elasticsearch-write-design" class="table-of-contents__link toc-highlight">Elasticsearch Write Design</a><ul><li><a href="#the-relationship-between-inlong_group_id-inlong_stream_id-audit-id-and-elasticsearch-index" class="table-of-contents__link toc-highlight">The relationship between inlong_group_id, inlong_stream_id, audit ID and Elasticsearch index</a></li><li><a href="#write-routing-policy" class="table-of-contents__link toc-highlight">Write Routing Policy</a></li><li><a href="#optional-deduplicationby-doc_id" class="table-of-contents__link toc-highlight">Optional DeduplicationBy doc_id</a></li><li><a href="#use-bulk-batch-method" class="table-of-contents__link toc-highlight">Use bulk batch method</a></li></ul></li><li><a href="#mysql-distribution-implementation" class="table-of-contents__link toc-highlight">MySQL Distribution Implementation</a><ul><li><a href="#target-3" class="table-of-contents__link toc-highlight">Target</a></li><li><a href="#main-logic-diagram-3" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li><li><a href="#usage-introduction" class="table-of-contents__link toc-highlight">Usage introduction</a></li></ul></li><li><a href="#clickhouse-distribution-implementation" class="table-of-contents__link toc-highlight">ClickHouse Distribution Implementation</a><ul><li><a href="#target-4" class="table-of-contents__link toc-highlight">Target</a></li><li><a href="#main-logic-diagram-4" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li><li><a href="#usage-introduction-1" class="table-of-contents__link toc-highlight">Usage introduction</a></li></ul></li><li><a href="#audit-usage-interface-design" class="table-of-contents__link toc-highlight">Audit Usage Interface Design</a><ul><li><a href="#main-logic-diagram-5" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li><li><a href="#ui-interface-display" class="table-of-contents__link toc-highlight">UI Interface Display</a></li><li><a href="#main-logic-diagram-6" class="table-of-contents__link toc-highlight">Main Logic Diagram</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Events</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apachecon.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">ApacheCon<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="acevent" data-format="square" data-mode="dark" data-event="random"></a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://twitter.com/ApacheInlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://inlong.apache.org/img/apache-inlong-wechat.jpg" target="_blank" rel="noopener noreferrer" class="footer__link-item">WeChat<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="mailto:dev@inlong.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Email</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/blog">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright"><div style="font-family: Avenir-Medium;font-size: 14px;color: #999;"> |
| <div>Copyright © 2020-2024 The Apache Software Foundation. Licensed under the Apache License, Version 2.0.</div> |
| <div style="margin-top: 20px; padding-top: 20px; border-top: 1px solid #666;line-height: 20px;">The Apache Software Foundation Apache InLong, InLong, Apache, the Apache feather, and the Apache InLong project logo are either registered trademarks or trademarks of the Apache Software Foundation.</div> |
| </div></div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.63c98e82.js"></script> |
| <script src="/assets/js/main.070aef2a.js"></script> |
| </body> |
| </html> |
