| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-1.7.0 plugin-docs plugin-id-default docs-doc-id-quick_start/pulsar_example"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.3.1"> |
| <title data-rh="true">Pulsar Example | Apache InLong</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://inlong.apache.org/docs/1.7.0/quick_start/pulsar_example"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="1.7.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-1.7.0"><meta data-rh="true" name="docsearch:version" content="1.7.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-1.7.0"><meta data-rh="true" property="og:title" content="Pulsar Example | Apache InLong"><meta data-rh="true" name="description" content="Apache InLong has increased the ability to access data through Apache Pulsar, taking full advantage of Pulsar's technical advantages that are different from other MQ, and providing complete solutions for data integration scenarios with higher data quality requirements such as finance and billing."><meta data-rh="true" property="og:description" content="Apache InLong has increased the ability to access data through Apache Pulsar, taking full advantage of Pulsar's technical advantages that are different from other MQ, and providing complete solutions for data integration scenarios with higher data quality requirements such as finance and billing."><link data-rh="true" rel="icon" href="/img/logo.svg"><link data-rh="true" rel="canonical" href="https://inlong.apache.org/docs/1.7.0/quick_start/pulsar_example"><link data-rh="true" rel="alternate" href="https://inlong.apache.org/docs/1.7.0/quick_start/pulsar_example" hreflang="en"><link data-rh="true" rel="alternate" href="https://inlong.apache.org/zh-CN/docs/1.7.0/quick_start/pulsar_example" hreflang="zh-CN"><link data-rh="true" rel="alternate" href="https://inlong.apache.org/docs/1.7.0/quick_start/pulsar_example" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://YUW9QEL53E-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache InLong RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache InLong Atom Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache InLong" href="/opensearch.xml"> |
| |
| |
| |
| <script src="https://www.apachecon.com/event-images/snippet.js" async></script><link rel="stylesheet" href="/assets/css/styles.c64edd51.css"> |
| <link rel="preload" href="/assets/js/runtime~main.63c98e82.js" as="script"> |
| <link rel="preload" href="/assets/js/main.070aef2a.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function e(e){document.documentElement.setAttribute("data-theme",e)}var t=function(){var e=null;try{e=localStorage.getItem("theme")}catch(e){}return e}();null!==t?e(t):window.matchMedia("(prefers-color-scheme: dark)").matches?e("dark"):(window.matchMedia("(prefers-color-scheme: light)").matches,e("light"))}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/logo.svg" alt="Apache" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache InLong</b></a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/introduction">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/introduction">Next</a></li><li><a class="dropdown__link" href="/docs/introduction">1.11.0</a></li><li><a class="dropdown__link" href="/docs/1.10.0/introduction">1.10.0</a></li><li><a class="dropdown__link" href="/docs/1.9.0/introduction">1.9.0</a></li><li><a class="dropdown__link" href="/docs/1.8.0/introduction">1.8.0</a></li><li><a class="dropdown__link" href="/versions/">All versions</a></li></ul></div><a class="navbar__item navbar__link" href="/downloads">Download</a><a class="navbar__item navbar__link" href="/community/how-to-contribute">Community</a><a class="navbar__item navbar__link" href="/blog">Blog</a><a class="navbar__item navbar__link" href="/team">Team</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">ASF</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li><li><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Privacy</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/docs/1.7.0/quick_start/pulsar_example" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li><li><a href="/zh-CN/docs/1.7.0/quick_start/pulsar_example" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="zh-CN">简体中文</a></li></ul></div><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.7.0/introduction">Introduction</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/design_and_concept/basic_concept">Design and Concept</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/1.7.0/quick_start/how_to_build">Quick Start</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.7.0/quick_start/how_to_build">How to Build</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.7.0/quick_start/hive_example">Hive Example</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/1.7.0/quick_start/pulsar_example">Pulsar Example</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/deployment/standalone">Deployment</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/modules/agent/overview">Components</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/data_node/extract_node/overview">Data Nodes</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/sdk/dataproxy-sdk/cpp">SDK</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/user_guide/dashboard_usage">User Guide</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/development/inlong_msg">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.7.0/administration/user_management">Administration</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.7.0/contact">Contact Us</a></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache InLong<!-- --> <b>1.7.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/introduction">latest version</a></b> (<!-- -->1.11.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Quick Start</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Pulsar Example</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.7.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Pulsar Example</h1></header><p>Apache InLong has increased the ability to access data through Apache Pulsar, taking full advantage of Pulsar's technical advantages that are different from other MQ, and providing complete solutions for data integration scenarios with higher data quality requirements such as finance and billing. |
| In the following content, we will use a complete example to introduce Apache Pulsar to access data through Apache InLong.</p><p><img loading="lazy" alt="Create Group" src="/assets/images/pulsar-arch-fc9241768fd9bc0c2f8224734e53e202.png" width="1751" height="306" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="install-pulsar">Install Pulsar<a href="#install-pulsar" class="hash-link" aria-label="Direct link to Install Pulsar" title="Direct link to Install Pulsar"></a></h2><p>Please refer to <a href="https://pulsar.apache.org/docs/en/standalone/" target="_blank" rel="noopener noreferrer">Official Installation Guidelines</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="install-hive">Install Hive<a href="#install-hive" class="hash-link" aria-label="Direct link to Install Hive" title="Direct link to Install Hive"></a></h2><p>Hive is the necessary component. If you don't have Hive in your machine, we recommand using Docker to install it. Details can be found <a href="https://github.com/big-data-europe/docker-hive" target="_blank" rel="noopener noreferrer">here</a>.</p><blockquote><p>Note that if you use Docker, you need to add a port mapping <code>8020:8020</code>, because it's the port of HDFS DefaultFS, and we need to use it later.</p></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="install-inlong">Install InLong<a href="#install-inlong" class="hash-link" aria-label="Direct link to Install InLong" title="Direct link to Install InLong"></a></h2><p>Before we begin, we need to install InLong. Here we provide two ways:</p><ol><li>Install InLong with Docker by according to the <a href="/docs/1.7.0/deployment/docker">instructions here</a>.(Recommanded)</li><li>Install InLong binary according to the <a href="/docs/1.7.0/deployment/bare_metal">instructions here</a>.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-data-ingestion">Create a data ingestion<a href="#create-a-data-ingestion" class="hash-link" aria-label="Direct link to Create a data ingestion" title="Direct link to Create a data ingestion"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-data-streams-group-information">Configure data streams group information<a href="#configure-data-streams-group-information" class="hash-link" aria-label="Direct link to Configure data streams group information" title="Direct link to Configure data streams group information"></a></h3><p><img loading="lazy" src="/assets/images/pulsar-group-dfd5cf3d140dc22db1a7a99a230a4624.png" width="1431" height="653" class="img_ev3q"> |
| When creating data ingestion, the message middleware that the data stream group can use is Pulsar, |
| and other configuration items related to Pulsar include:</p><ul><li>Queue module: Parallel or Serial, when selecting parallel, you can set the number of topic partitions</li><li>Write quorum: Number of copies to store for each message</li><li>Ack quorum: Number of guaranteed copies (acks to wait before write is complete)</li><li>retention time: retention time for the consumed message</li><li>ttl: The default Time to Live for message</li><li>retention size: retention size for the consumed message</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-data-stream">Configure data stream<a href="#configure-data-stream" class="hash-link" aria-label="Direct link to Configure data stream" title="Direct link to Configure data stream"></a></h3><p><img loading="lazy" src="/assets/images/pulsar-stream-b525a38e7ff3475e180a71362180faf8.png" width="970" height="443" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-file-agent">Configure File Agent<a href="#configure-file-agent" class="hash-link" aria-label="Direct link to Configure File Agent" title="Direct link to Configure File Agent"></a></h3><p><img loading="lazy" src="/assets/images/file-source-e62466d16f0beb72ff66134bd9720f97.png" width="658" height="458" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-data-information">Configure data information<a href="#configure-data-information" class="hash-link" aria-label="Direct link to Configure data information" title="Direct link to Configure data information"></a></h3><p><img loading="lazy" src="/assets/images/pulsar-data-02f0dd9ecd75f6545b6a8ff993b178f2.png" width="979" height="671" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-hive-cluster">Configure Hive cluster<a href="#configure-hive-cluster" class="hash-link" aria-label="Direct link to Configure Hive cluster" title="Direct link to Configure Hive cluster"></a></h3><p>Save Hive cluster information, click "Ok" to submit. |
| <img loading="lazy" src="/assets/images/pulsar-hive-8170f49696fc889e32eb762a4d7a748d.png" width="1006" height="703" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-ingestion-approval">Data ingestion Approval<a href="#data-ingestion-approval" class="hash-link" aria-label="Direct link to Data ingestion Approval" title="Direct link to Data ingestion Approval"></a></h2><p>Enter <strong>Approval</strong> page, click <strong>My Approval</strong>, abd approve the data ingestion application. After the approval is over, |
| the topics and subscriptions required for the data stream will be created in the Pulsar cluster synchronously. |
| We can use the command-line tool in the Pulsar cluster to check whether the topic is created successfully.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configure-file-agent-1">Configure File Agent<a href="#configure-file-agent-1" class="hash-link" aria-label="Direct link to Configure File Agent" title="Direct link to Configure File Agent"></a></h2><p>Then we need to create a new file <code>/data/collect-data/test.log</code> and add content to it to trigger the agent to send data to the dataproxy.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token function" style="color:rgb(80, 250, 123)">mkdir</span><span class="token plain"> collect-data</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token assign-left variable" style="color:rgb(189, 147, 249);font-style:italic">END</span><span class="token operator">=</span><span class="token number">100000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">for</span><span class="token plain"> </span><span class="token variable punctuation" style="color:rgb(248, 248, 242);font-style:italic">((</span><span class="token variable" style="color:rgb(189, 147, 249);font-style:italic">i</span><span class="token variable operator" style="color:rgb(189, 147, 249);font-style:italic">=</span><span class="token variable number" style="color:rgb(189, 147, 249);font-style:italic">1</span><span class="token variable punctuation" style="color:rgb(248, 248, 242);font-style:italic">;</span><span class="token variable" style="color:rgb(189, 147, 249);font-style:italic">i</span><span class="token variable operator" style="color:rgb(189, 147, 249);font-style:italic"><=</span><span class="token variable" style="color:rgb(189, 147, 249);font-style:italic">END</span><span class="token variable punctuation" style="color:rgb(248, 248, 242);font-style:italic">;</span><span class="token variable" style="color:rgb(189, 147, 249);font-style:italic">i</span><span class="token variable operator" style="color:rgb(189, 147, 249);font-style:italic">++</span><span class="token variable punctuation" style="color:rgb(248, 248, 242);font-style:italic">))</span><span class="token punctuation" style="color:rgb(248, 248, 242)">;</span><span class="token plain"> </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">do</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token function" style="color:rgb(80, 250, 123)">sleep</span><span class="token plain"> </span><span class="token number">3</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token builtin class-name" style="color:rgb(189, 147, 249)">echo</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"name_</span><span class="token string variable" style="color:rgb(189, 147, 249);font-style:italic">$i</span><span class="token string" style="color:rgb(255, 121, 198)"> | </span><span class="token string variable" style="color:rgb(189, 147, 249);font-style:italic">$i</span><span class="token string" style="color:rgb(255, 121, 198)">"</span><span class="token plain"> </span><span class="token operator">>></span><span class="token plain"> /data/collect-data/test.log</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">done</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then you can observe the Audit Data Pages, and see that the data has been collected and sent successfully.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-check">Data Check<a href="#data-check" class="hash-link" aria-label="Direct link to Data Check" title="Direct link to Data Check"></a></h2><p>Finally, we log in to the Hive cluster and use Hive SQL commands to check |
| whether data is successfully inserted in the <code>test_stream</code> table.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="troubleshooting">Troubleshooting<a href="#troubleshooting" class="hash-link" aria-label="Direct link to Troubleshooting" title="Direct link to Troubleshooting"></a></h2><p>If data is not correctly written to the Hive cluster, you can check whether the <code>DataProxy</code> and <code>Sort</code> related information are synchronized:</p><ul><li>Check whether the topic information corresponding to the data stream is correctly written in the <code>conf/topics.properties</code> folder of <code>InLong DataProxy</code>:</li></ul><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">b_test_group/test_stream=persistent://public/b_test_group/test_stream</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/inlong-website/edit/master/versioned_docs/version-1.7.0/quick_start/pulsar_example.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/1.7.0/quick_start/hive_example"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Hive Example</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/1.7.0/deployment/standalone"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Standalone</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#install-pulsar" class="table-of-contents__link toc-highlight">Install Pulsar</a></li><li><a href="#install-hive" class="table-of-contents__link toc-highlight">Install Hive</a></li><li><a href="#install-inlong" class="table-of-contents__link toc-highlight">Install InLong</a></li><li><a href="#create-a-data-ingestion" class="table-of-contents__link toc-highlight">Create a data ingestion</a><ul><li><a href="#configure-data-streams-group-information" class="table-of-contents__link toc-highlight">Configure data streams group information</a></li><li><a href="#configure-data-stream" class="table-of-contents__link toc-highlight">Configure data stream</a></li><li><a href="#configure-file-agent" class="table-of-contents__link toc-highlight">Configure File Agent</a></li><li><a href="#configure-data-information" class="table-of-contents__link toc-highlight">Configure data information</a></li><li><a href="#configure-hive-cluster" class="table-of-contents__link toc-highlight">Configure Hive cluster</a></li></ul></li><li><a href="#data-ingestion-approval" class="table-of-contents__link toc-highlight">Data ingestion Approval</a></li><li><a href="#configure-file-agent-1" class="table-of-contents__link toc-highlight">Configure File Agent</a></li><li><a href="#data-check" class="table-of-contents__link toc-highlight">Data Check</a></li><li><a href="#troubleshooting" class="table-of-contents__link toc-highlight">Troubleshooting</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Events</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apachecon.com/" target="_blank" rel="noopener noreferrer" class="footer__link-item">ApacheCon<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="acevent" data-format="square" data-mode="dark" data-event="random"></a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://twitter.com/ApacheInlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://inlong.apache.org/img/apache-inlong-wechat.jpg" target="_blank" rel="noopener noreferrer" class="footer__link-item">WeChat<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="mailto:dev@inlong.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Email</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/blog">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/inlong" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/asf_logo.svg" alt="Apache InLong" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></div><div class="footer__copyright"><div style="font-family: Avenir-Medium;font-size: 14px;color: #999;"> |
| <div>Copyright © 2020-2024 The Apache Software Foundation. Licensed under the Apache License, Version 2.0.</div> |
| <div style="margin-top: 20px; padding-top: 20px; border-top: 1px solid #666;line-height: 20px;">The Apache Software Foundation Apache InLong, InLong, Apache, the Apache feather, and the Apache InLong project logo are either registered trademarks or trademarks of the Apache Software Foundation.</div> |
| </div></div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.63c98e82.js"></script> |
| <script src="/assets/js/main.070aef2a.js"></script> |
| </body> |
| </html> |
