| <!doctype html> |
| <html lang="en-US" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-community docs-doc-id-developer-guide/debug-tool" data-has-hydrated="false"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1,minimum-scale=1,maximum-scale=1,user-scalable=no"> |
| <meta name="generator" content="Docusaurus v2.4.3"> |
| <link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Doris RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Doris Atom Feed"> |
| |
| <link rel="preconnect" href="https://www.google-analytics.com"> |
| <link rel="preconnect" href="https://www.googletagmanager.com"> |
| <script async src="https://www.googletagmanager.com/gtag/js?id=G-DT7W9E9722"></script> |
| <script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-DT7W9E9722",{anonymize_ip:!0})</script> |
| |
| |
| |
| <link rel="preconnect" href="https://analytics.apache.org/"> |
| <script>var _paq=window._paq=window._paq||[];_paq.push(["setRequestMethod","POST"]),_paq.push(["trackPageView"]),_paq.push(["enableLinkTracking"]),_paq.push(["enableHeartBeatTimer"]),function(){var e="https://analytics.apache.org/";_paq.push(["setRequestMethod","POST"]),_paq.push(["setTrackerUrl",e+"matomo.php"]),_paq.push(["setSiteId","43"]);var a=document,t=a.createElement("script"),p=a.getElementsByTagName("script")[0];t.type="text/javascript",t.async=!0,t.src=e+"matomo.js",p.parentNode.insertBefore(t,p)}()</script> |
| |
| |
| |
| <link rel="icon" href="/images/logo-only.png"> |
| <link rel="manifest" href="/manifest.json"> |
| <meta name="theme-color" content="#FFFFFF"> |
| <meta name="apple-mobile-web-app-capable" content="yes"> |
| <meta name="apple-mobile-web-app-status-bar-style" content="#000"> |
| <link rel="apple-touch-icon" href="/img/docusaurus.png"> |
| <link rel="mask-icon" href="/img/docusaurus.svg" color="rgb(37, 194, 160)"> |
| <meta name="msapplication-TileImage" content="/img/docusaurus.png"> |
| <meta name="msapplication-TileColor" content="#000"> |
| |
| |
| |
| <link rel="stylesheet" href="https://cdn-font.hyperos.mi.com/font/css?family=MiSans:100,200,300,400,450,500,600,650,700,900:Chinese_Simplify,Latin&display=swap"> |
| <link rel="stylesheet" href="https://cdn-font.hyperos.mi.com/font/css?family=MiSans_Latin:100,200,300,400,450,500,600,650,700,900:Latin&display=swap"> |
| <script src="/js/custom-script.js"></script><title data-rh="true">Debug Tool - Apache Doris</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://doris.apache.org/community/developer-guide/debug-tool"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-community-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-community-current"><meta data-rh="true" property="og:title" content="Debug Tool - Apache Doris"><meta data-rh="true" name="description" content="<!--"><meta data-rh="true" property="og:description" content="<!--"><link data-rh="true" rel="icon" href="/images/favicon.ico"><link data-rh="true" rel="canonical" href="https://doris.apache.org/community/developer-guide/debug-tool"><link data-rh="true" rel="alternate" href="https://doris.apache.org/community/developer-guide/debug-tool" hreflang="en-US"><link data-rh="true" rel="alternate" href="https://doris.apache.org/zh-CN/community/developer-guide/debug-tool" hreflang="zh-Hans-CN"><link data-rh="true" rel="alternate" href="https://doris.apache.org/community/developer-guide/debug-tool" hreflang="x-default"><link rel="stylesheet" href="https://cdnd.selectdb.com/assets/css/styles.2bd95c40.css"> |
| <link rel="preload" href="https://cdnd.selectdb.com/assets/js/runtime~main.eb208fba.js" as="script"> |
| <link rel="preload" href="https://cdnd.selectdb.com/assets/js/main.fa7fcb85.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_s0pr" style="background-color:#3C2FD4;color:#FFFFFF" role="banner"><div class="announcementBarPlaceholder_qxfj"></div><div class="announcementBarContent_dpRF"><a href="https://github.com/apache/doris" target="_blank" style="display: flex; width: 100%; align-items: center; justify-content: center; margin-left: 4px; text-decoration: none; color: white">Do you ❤️ Doris? Give us a 🌟 on GitHub |
| <img style="width: 1.2rem; height: 1.2rem; margin-left: 0.4rem;" src="/images/github-white-icon.svg"> |
| </a></div><button type="button" class="clean-btn close announcementBarClose_iXyO" aria-label="Close"><svg viewBox="0 0 15 15" width="14" height="14" style="color:white"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner" style="padding:"><div class="navbar__items"><div class="navbar-left"><div class="navbar-logo-wrapper flex items-center"><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="https://cdnd.selectdb.com/images/logo.svg" alt="Apache Doris" class="themedImage_ToTc themedImage--light_HNdA"><img src="https://cdnd.selectdb.com/images/logo.svg" alt="Apache Doris" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate"></b></a></div><a class="navbar__item navbar__link" style="text-align:center" href="/docs/get-starting/quick-start">Docs</a><a class="navbar__item navbar__link" style="text-align:center" href="/blog">Blog</a><a class="navbar__item navbar__link" style="text-align:center" href="/users">Users</a><a href="https://github.com/apache/doris/discussions" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link" style="text-align:center">Discussions</a><a class="navbar__item navbar__link" style="text-align:center" href="/ecosystem/cluster-management">Ecosystem</a><a class="navbar__item navbar__link" style="text-align:center" href="/community/join-community">Community</a></div></div><div class="navbar__items navbar__items--right"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><div class="docs-search searchBox_H2mL"><div class="navbar__search searchBarContainer_PzyC"><input placeholder="Search" aria-label="Search" class="navbar__search-input navbarSearchInput_tb6T"><div class="loadingRing__K5d searchBarLoadingRing_e2f0"><div></div><div></div><div></div><div></div></div><div class="searchHintContainer_m7ml"><kbd class="searchHint_zuPL">ctrl</kbd><kbd class="searchHint_zuPL">K</kbd></div></div></div><div class="custom-navbar-item navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg class="icon-language" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16" fill="none"><path d="M7.75756 14.3L10.5816 6.91667H11.8759L14.7 14.3H13.4057L12.7501 12.4167H9.74113L9.06873 14.3H7.75756ZM10.1109 11.35H12.3467L11.254 8.3H11.2036L10.1109 11.35ZM2.84908 12.45L1.97498 11.5833L5.11841 8.48333C4.72618 8.05 4.38439 7.60267 4.09302 7.14133C3.80165 6.68044 3.54389 6.19444 3.31976 5.68333H4.61412C4.80463 6.06111 5.00635 6.39711 5.21927 6.69133C5.43219 6.986 5.68434 7.29444 5.97571 7.61667C6.43519 7.12778 6.81621 6.62511 7.11879 6.10867C7.42137 5.59178 7.67352 5.03889 7.87523 4.45H1V3.23333H5.33694V2H6.58087V3.23333H10.9178V4.45H9.11916C8.89503 5.18333 8.59805 5.89155 8.22824 6.57467C7.85842 7.25822 7.39895 7.90555 6.84983 8.51667L8.3459 10.0167L7.87523 11.2833L5.95891 9.38333L2.84908 12.45Z" fill="#4C576C"></path></svg></a><ul class="dropdown__menu"><li><a href="/community/developer-guide/debug-tool" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" style="text-align:center">English</a></li><li><a href="/zh-CN/community/developer-guide/debug-tool" target="_self" rel="noopener noreferrer" class="dropdown__link" style="text-align:center">中文</a></li></ul></div><div class="custom-navbar-item navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/get-starting/what-is-apache-doris"><span class="text-sm">Versions: </span></a><ul class="dropdown__menu"><li><a class="dropdown__link" style="text-align:center" href="/docs/dev/get-starting/what-is-apache-doris">Dev</a></li><li><a class="dropdown__link" style="text-align:center" href="/docs/get-starting/what-is-apache-doris">2.1</a></li><li><a class="dropdown__link" style="text-align:center" href="/docs/2.0/get-starting/what-is-apache-doris">2.0</a></li><li><a class="dropdown__link" style="text-align:center" href="/docs/1.2/get-starting/">1.2</a></li></ul></div><a class="navbar__item navbar__link header-right-button-primary navbar-download-mobile" style="text-align:center" href="/download">Download</a><a href="https://github.com/apache/doris" target="_blank" rel="noopener noreferrer" class="github-btn desktop header-right-button-github"></a><a href="https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-2kl08hzc0-SPJe4VWmL_qzrFd2u2XYQA" target="_blank" rel="noopener noreferrer" class="slack-btn desktop header-right-button-slack"></a><a class="header-right-button-primary navbar-download-desktop" href="/download">Download</a></div></div><div class="navbar__bottom"></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="main-wrapper docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar"><nav aria-label="Docs sidebar" class="menu custom-scrollbar menu_Y1UP menuWithAnnouncementBar_fPny"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/community/join-community">Join Community</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/community/subscribe-mail-list">Subscribe Mailing Lists</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/community/feedback">Feedback</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/community/doris-future">Doris Future Project</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a id="menu__group" class="menu__link menu__link_level_1 menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/community/how-to-contribute/">How to Contribute</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a id="menu__group" class="menu__link menu__link_level_1 menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/community/release-and-verify/release-prepare">Release Process & Verification</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/community/security">Security</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a id="menu__group" class="menu__link menu__link_level_1 menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/community/design/doris_storage_optimization">Design Documents</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a id="menu__group" class="menu__link menu__link_level_1 menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/community/developer-guide/debug-tool">Developer Guide</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list menu__list_level_2"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/community/developer-guide/debug-tool">Debug Tool</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/docker-dev">Doris Docker quick build development environment</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/benchmark-tool">Doris BE Storage Layer Benchmark Tool</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/pipeline-tracing">Pipeline Tracing</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/fe-eclipse-dev">Setting FE Dev Env - Eclipse</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/fe-profiler">Use FE profiler to generate flame graph</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/fe-idea-dev">Setting Up Dec Env for FE - IntelliJ IDEA</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/fe-vscode-dev">FE Development and Debugging Environment - Visual Studio Code (VSCode)</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/be-vscode-dev">BE development and debugging environment under Linux</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/be-clion-dev">Development and Debugging of Apache Doris BE -- Clion</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link_level_2 menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/community/developer-guide/mac-dev/dev-prepare">Mac Local Dev</a></div></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/java-format-code">Java Format Code</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/cpp-format-code">C++ Format Code</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/cpp-diagnostic-code">C++ Code Diagnostic</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/bitmap-hll-file-format">Bitmap/HLL data format</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/github-checks">Github Checks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/community/developer-guide/regression-testing">Regression Testing</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/community/team">Doris Team</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a href="https://cwiki.apache.org/confluence/display/DORIS/Doris+Improvement+Proposals" target="_blank" rel="noopener noreferrer" class="menu__link menuExternalLink_NmtK">Improvement Proposals<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_z5aJ"><div class="docItemContainer_c0TR"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Developer Guide</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Debug Tool</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Debug Tool</h1></header><h1>Debug Tool</h1><p>In the process of using and developing Doris, we often encounter scenarios that need to debug Doris. Here are some common debugging tools.</p><p><strong>The name of the BE binary that appears in this doc is <code>doris_be</code>, which was <code>palo_be</code> in previous versions.</strong></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fe-debugging">FE debugging<a href="#fe-debugging" class="hash-link" aria-label="Direct link to FE debugging" title="Direct link to FE debugging"></a></h2><p>Fe is a java process. Here are just a few simple and commonly used java debugging commands.</p><ol><li><p>Statistics of current memory usage details</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jmap -histo:live pid > 1. jmp</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This command can enumerate and sort the memory occupation of living objects. (replace PID with Fe process ID)</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">num #instances #bytes class name</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">----------------------------------------------</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">1: 33528 10822024 [B</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">2: 80106 8662200 [C</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">3: 143 4688112 [Ljava.util.concurrent.ForkJoinTask;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">4: 80563 1933512 java. lang.String</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">5: 15295 1714968 java. lang.Class</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">6: 45546 1457472 java. util. concurrent. ConcurrentHashMap$Node</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">7: 15483 1057416 [Ljava.lang.Object;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can use this method to view the total memory occupied by the currently living objects (at the end of the file) and analyze which objects occupy more memory.</p><p>Note that this method will trigger fullgc because `: live 'is specified.</p></li><li><p>Check JVM memory usage</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jstat -gcutil pid 1000 1000</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This command can scroll through the memory usage of each region of the current JVM. (replace PID with Fe process ID)</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">S0 S1 E O M CCS YGC YGCT FGC FGCT GCT</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">0.00 0.00 22.61 3.03 95.74 92.77 68 1.249 5 0.794 2.043</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">0.00 0.00 22.61 3.03 95.74 92.77 68 1.249 5 0.794 2.043</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">0.00 0.00 22.61 3.03 95.74 92.77 68 1.249 5 0.794 2.043</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">0.00 0.00 22.92 3.03 95.74 92.77 68 1.249 5 0.794 2.043</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">0.00 0.00 22.92 3.03 95.74 92.77 68 1.249 5 0.794 2.043</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The main focus is on the percentage of old area (o) (3% in the example). If the occupancy is too high, oom or fullgc may occur.</p></li><li><p>Print Fe thread stack</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jstack -l pid > 1. js</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This command can print the thread stack of the current Fe. (replace PID with Fe process ID). |
| <code>-L </code> the parameter will detect whether there is deadlock at the same time. This method can check the operation of Fe thread, whether there is deadlock, where it is stuck, etc.</p></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="be-debugging">BE debugging<a href="#be-debugging" class="hash-link" aria-label="Direct link to BE debugging" title="Direct link to BE debugging"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="preparing">Preparing<a href="#preparing" class="hash-link" aria-label="Direct link to Preparing" title="Direct link to Preparing"></a></h3><p><a href="https://github.com/google/pprof" target="_blank" rel="noopener noreferrer">pprof</a>: from gperftools, it is used to transform the content generated by gperftools into a format that is easy for people to read, such as PDF, SVG, text, etc.</p><p><a href="http://www.graphviz.org/" target="_blank" rel="noopener noreferrer">graphviz</a>: in the absence of this library, pprof can only be converted to text format, but this way is not easy to view. After the library is installed, pprof can be converted to SVG, PDF and other formats, and the call relationship is clearer.</p><p><a href="https://perf.wiki.kernel.org/index.php/main_page" target="_blank" rel="noopener noreferrer">perf</a>: Linux kernel comes with performance analysis tool. <a href="http://www.brendangregg.com/perf.html" target="_blank" rel="noopener noreferrer">here</a> there are some examples of perf usage.</p><p><a href="https://github.com/brendangregg/flamegraph" target="_blank" rel="noopener noreferrer">flamegraph</a>: a visualization tool used to show the output of perf in the form of flame graph.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="memory">Memory<a href="#memory" class="hash-link" aria-label="Direct link to Memory" title="Direct link to Memory"></a></h3><p>Debugging memory is generally divided into two aspects. One is whether the total amount of memory use is reasonable. On the one hand, the excessive amount of memory use may be due to memory leak in the system, on the other hand, it may be due to improper use of program memory. The second is whether there is a problem of memory overrun and illegal access, such as program access to memory with an illegal address, use of uninitialized memory, etc. For the debugging of memory, we usually use the following ways to track the problems.</p><p>Doris 1.2.1 and previous versions use TCMalloc. Doris 1.2.2 starts to use Jemalloc by default. Select the memory debugging method according to the Doris version used. If you need to switch TCMalloc, you can compile <code>USE_JEMALLOC=OFF sh build.sh --be</code>.</p><p>When we find that the memory usage is too large, we can first check the BE log to see if there is a large memory application.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="tcmalloc">TCMalloc<a href="#tcmalloc" class="hash-link" aria-label="Direct link to TCMalloc" title="Direct link to TCMalloc"></a></h6><p>When using TCMalloc, when a large memory application is encountered, the application stack will be printed to the be.out file, and the general expression is as follows:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">tcmalloc: large alloc 1396277248 bytes == 0x3f3488000 @ 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This indicates that Doris be is trying to apply memory of '1396277248 bytes' on this stack. We can use the 'addr2line' command to restore the stack to a letter that we can understand. The specific example is shown below.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">$ addr2line -e lib/doris_be 0x2af6f63 0x2c4095b 0x134d278 0x134bdcb 0x133d105 0x133d1d0 0x19930ed</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1335</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">/home/ssd0/zc/palo/doris/core/thirdparty/src/gperftools-gperftools-2.7/src/tcmalloc.cc:1357</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.cpp:267</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_table.hpp:86</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:239</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">/home/disk0/baidu-doris/baidu/bdg/doris-baidu/core/be/src/exec/hash_join_node.cpp:213</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">thread.cpp:?</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="jemalloc">JEMALLOC<a href="#jemalloc" class="hash-link" aria-label="Direct link to JEMALLOC" title="Direct link to JEMALLOC"></a></h5><p>Most of Doris's large memory applications use Allocator, such as HashTable and data serialization. This part of the memory application is expected and will be effectively managed. Other large memory applications are not expected and will be applied The stack is printed to the be.INFO file, which is usually used for debugging, and the general expression is as follows:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">MemHook alloc large memory: 8.2GB, stacktrace:</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Alloc Stacktrace:</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @ 0x55a6a5cf6b4d doris::ThreadMemTrackerMgr::consume()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @ 0x55a6a5cf99bf malloc</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @ 0x55a6ae0caf98 operator new()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @ 0x55a6a57cb013 doris::segment_v2::PageIO::read_and_decompress_page()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @ 0x55a6a57719c0 doris::segment_v2::ColumnReader::read_page()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ……</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="heap-profile">HEAP PROFILE<a href="#heap-profile" class="hash-link" aria-label="Direct link to HEAP PROFILE" title="Direct link to HEAP PROFILE"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="tcmalloc-1">TCMalloc<a href="#tcmalloc-1" class="hash-link" aria-label="Direct link to TCMalloc" title="Direct link to TCMalloc"></a></h5><p>Sometimes the application of memory is not caused by the application of large memory, but by the continuous accumulation of small memory. Then there is no way to locate the specific application information by viewing the log, so you need to get the information through other ways.</p><p>At this time, we can take advantage of TCMalloc's <a href="https://gperftools.github.io/gperftools/heapprofile.html" target="_blank" rel="noopener noreferrer">heapprofile</a>. If the heapprofile function is set, we can get the overall memory application usage of the process. The usage is to set the 'heapprofile' environment variable before starting Doris be. For example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">export HEAPPROFILE=/tmp/doris_be.hprof</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">./bin/start_be.sh --daemon</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In this way, when the dump condition of the heapprofile is met, the overall memory usage will be written to the file in the specified path. Later, we can use the 'pprof' tool to analyze the output content.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">$ pprof --text lib/doris_be /tmp/doris_be.hprof.0012.heap | head -30</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Using local file lib/doris_be.</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Using local file /tmp/doris_be.hprof.0012.heap.</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Total: 668.6 MB</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 610.6 91.3% 91.3% 610.6 91.3% doris::SystemAllocator::allocate_via_malloc (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 18.1 2.7% 94.0% 18.1 2.7% _objalloc_alloc</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 5.6 0.8% 94.9% 63.4 9.5% doris::RowBatch::RowBatch</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 5.1 0.8% 95.6% 7.1 1.1% butil::ResourcePool::add_block (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 3.7 0.5% 96.2% 3.7 0.5% butil::iobuf::create_block (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 3.4 0.5% 96.7% 3.4 0.5% butil::FlatMap::init</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 3.2 0.5% 97.2% 5.2 0.8% butil::ObjectPool::add_block (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 2.6 0.4% 97.6% 2.6 0.4% __gnu_cxx::new_allocator::allocate (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 2.0 0.3% 97.9% 2.0 0.3% butil::ObjectPool::add_block_group (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 2.0 0.3% 98.2% 2.0 0.3% butil::ResourcePool::add_block_group (inline)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 1.7 0.3% 98.4% 1.7 0.3% doris::SegmentReader::_load_index</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Contents of each column of the above documents:</p><ul><li>Column 1: the memory size directly applied by the function, in MB</li><li>Column 4: the total memory size of the function and all the functions it calls.</li><li>The second column and the fifth column are the proportion values of the first column and the fourth column respectively.</li><li>The third column is the cumulative value of the second column.</li></ul><p>Of course, it can also generate call relation pictures, which is more convenient for analysis. For example, the following command can generate a call graph in SVG format.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">pprof --svg lib/doris_be /tmp/doris_be.hprof.0012.heap > heap.svg </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><strong>NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.</strong></p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="pprof-remote-server">pprof remote server<a href="#pprof-remote-server" class="hash-link" aria-label="Direct link to pprof remote server" title="Direct link to pprof remote server"></a></h6><p>Although heapprofile can get all the memory usage information, it has some limitations. 1. Restart be. 2. You need to enable this command all the time, which will affect the performance of the whole process.</p><p>For Doris be, you can also use the way of opening and closing the heap profile dynamically to analyze the memory application of the process. Doris supports the <a href="https://gperftools.github.io/gperftools/pprof_remote_servers.html" target="_blank" rel="noopener noreferrer">remote server debugging of gperftools</a>. Then you can use 'pprof' to directly perform dynamic head profile on the remote running Doris be. For example, we can check the memory usage increment of Doris through the following command</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">$ pprof --text --seconds=60 http://be_host:be_webport/pprof/heap </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Total: 1296.4 MB</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 484.9 37.4% 37.4% 484.9 37.4% doris::StorageByteBuffer::create</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 272.2 21.0% 58.4% 273.3 21.1% doris::RowBlock::init</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 157.5 12.1% 70.5% 157.5 12.1% doris::RowBatch::RowBatch</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 90.7 7.0% 77.5% 90.7 7.0% doris::SystemAllocator::allocate_via_malloc</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 66.6 5.1% 82.7% 66.6 5.1% doris::IntegerColumnReader::init</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 47.9 3.7% 86.4% 47.9 3.7% __gnu_cxx::new_allocator::allocate</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 20.8 1.6% 88.0% 35.4 2.7% doris::SegmentReader::_load_index</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 12.7 1.0% 89.0% 12.7 1.0% doris::DecimalColumnReader::init</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 12.7 1.0% 89.9% 12.7 1.0% doris::LargeIntColumnReader::init</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 12.7 1.0% 90.9% 12.7 1.0% doris::StringColumnDirectReader::init</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 12.3 0.9% 91.9% 12.3 0.9% std::__cxx11::basic_string::_M_mutate</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 10.4 0.8% 92.7% 10.4 0.8% doris::VectorizedRowBatch::VectorizedRowBatch</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> 10.0 0.8% 93.4% 10.0 0.8% doris::PlainTextLineReader::PlainTextLineReader</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output of this command is the same as the output and view mode of heap profile, which will not be described in detail here. Statistics will be enabled only during execution of this command, which has a limited impact on process performance compared with heap profile.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="jemalloc-1">JEMALLOC<a href="#jemalloc-1" class="hash-link" aria-label="Direct link to JEMALLOC" title="Direct link to JEMALLOC"></a></h5><h6 class="anchor anchorWithStickyNavbar_LWe7" id="1-realtime-heap-dump">1. realtime heap dump<a href="#1-realtime-heap-dump" class="hash-link" aria-label="Direct link to 1. realtime heap dump" title="Direct link to 1. realtime heap dump"></a></h6><p>Change <code>prof:false</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code> to <code>prof:true</code> and restart BE, then use the jemalloc heap dump http interface to generate a heap dump file on the corresponding BE machine.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token function" style="color:rgb(80, 250, 123)">curl</span><span class="token plain"> http://be_host:be_webport/jeheap/dump</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The directory where the heap dump file is located can be configured through the <code>jeprofile_dir</code> variable in <code>be.conf</code>, and the default is <code>${DORIS_HOME}/log</code></p><p>The default sampling interval is 512K, usually only 10% of memory is recorded by heap dump, and the impact on performance is usually less than 10%. You can modify <code>lg_prof_sample</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code>, and the default is <code>19</code> (2^19 B = 512K), reducing <code>lg_prof_sample</code> can sample more frequently to make the heap profile close to the real memory, but this will bring greater performance loss.</p><p>If you are doing profiling, keep <code>prof:false</code> to avoid the performance penalty of heap dump.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="2-regular-heap-dump">2. regular heap dump<a href="#2-regular-heap-dump" class="hash-link" aria-label="Direct link to 2. regular heap dump" title="Direct link to 2. regular heap dump"></a></h6><p>Also change <code>prof:false</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code> to <code>prof:true</code>, and modify <code>JEMALLOC_PROF_PRFIX</code> in <code>be.conf</code> to any value and restart BE.</p><p>The directory where the heap dump file is located is <code>${DORIS_HOME}/log</code> by default, and the file name prefix is <code>JEMALLOC_PROF_PRFIX</code>.</p><ol><li><p>Dump when accumulatively applying for a certain value of memory:</p><p>The default memory accumulatively applies for 4GB to generate a dump. You can modify the <code>lg_prof_interval</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code> to adjust the dump interval. The default value is <code>32</code> (2^32 B = 4GB).</p></li><li><p>Dump every time the memory reaches a new high:</p><p>Change <code>prof_gdump</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code> to <code>true</code> and restart BE.</p></li><li><p>Dump when the program exits, and detect memory leaks:</p><p>Change <code>prof_leak</code> and <code>prof_final</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code> to <code>true</code> and restart BE.</p></li><li><p>Dump memory cumulative value (growth), not real-time value:</p><p>Change <code>prof_accum</code> of <code>JEMALLOC_CONF</code> in <code>be.conf</code> to <code>true</code> and restart BE. |
| Use <code>jeprof --alloc_space</code> to display heap dump accumulation.</p></li></ol><h5 class="anchor anchorWithStickyNavbar_LWe7" id="3-jemalloc-heap-dump-profiling">3. jemalloc heap dump profiling<a href="#3-jemalloc-heap-dump-profiling" class="hash-link" aria-label="Direct link to 3. jemalloc heap dump profiling" title="Direct link to 3. jemalloc heap dump profiling"></a></h5><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">Addr2line version 2.35.2 or above is required, see QA 1 below.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol><li>Analyze a single heap dump file</li></ol><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jeprof --dot lib/doris_be heap_dump_file_1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After executing the above command, a dot syntax graph will be output in the terminal. Paste it to the <a href="http://www.webgraphviz.com/" target="_blank" rel="noopener noreferrer">online dot drawing website</a> to generate a memory allocation graph, and then analyze it.</p><p>If the server is convenient for file transfer, you can also use the following command to directly generate a call relationship graph result.pdf file and transfer it to the local computer for viewing. You need to install the dependencies required for drawing.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">yum </span><span class="token function" style="color:rgb(80, 250, 123)">install</span><span class="token plain"> ghostscript graphviz</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">jeprof --pdf lib/doris_be heap_dump_file_1 </span><span class="token operator">></span><span class="token plain"> result.pdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="2"><li>Analyze the diff of two heap dump files</li></ol><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jeprof --dot lib/doris_be --base</span><span class="token operator">=</span><span class="token plain">heap_dump_file_1 heap_dump_file_2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Multiple heap files can be generated by running the above command multiple times over a period of time. You can select an earlier heap file as a baseline and compare it with a later heap file to analyze their diff. The method for generating a call relationship graph is the same as above.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="4-qa">4. QA<a href="#4-qa" class="hash-link" aria-label="Direct link to 4. QA" title="Direct link to 4. QA"></a></h5><ol><li>Many errors occurred after running jeprof: <code>addr2line: Dwarf Error: found dwarf version xxx, this reader only handles version xxx</code></li></ol><p>After GCC 11, DWARF-v5 is used by default, which requires Binutils 2.35.2 and above. Doris Ldb_toolchain uses GCC 11. See: <a href="https://gcc.gnu.org/gcc-11/changes.html" target="_blank" rel="noopener noreferrer">https://gcc.gnu.org/gcc-11/changes.html</a>.</p><p>Replace addr2line to 2.35.2, refer to:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">// Download addr2line source code</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.bz2</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">//Install dependencies, if needed</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">yum install make gcc gcc-c++ binutils</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">// Compile & install addr2line</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">tar -xvf binutils-2.35.tar.bz2</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">cd binutils-2.35</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">./configure --prefix=/usr/local</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">make</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">make install</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">// verify</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">addr2line -h</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">// Replace addr2line</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">chmod +x addr2line</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">mv /usr/bin/addr2line /usr/bin/addr2line.bak</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">mv /bin/addr2line /bin/addr2line.bak</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">cp addr2line /bin/addr2line</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">cp addr2line /usr/bin/addr2line</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">hash -r</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that you cannot use addr2line 2.3.9, which may be incompatible and cause memory to keep growing.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="lsan">LSAN<a href="#lsan" class="hash-link" aria-label="Direct link to LSAN" title="Direct link to LSAN"></a></h4><p><a href="https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer" target="_blank" rel="noopener noreferrer">LSAN</a> is an address checking tool, GCC has been integrated. When we compile the code, we can enable this function by turning on the corresponding compilation options. When the program has a determinable memory leak, it prints the leak stack. Doris be has integrated this tool, only need to compile with the following command to generate be binary with memory leak detection version.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">BUILD_TYPE=LSAN ./build.sh</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When the system detects a memory leak, it will output the corresponding information in be. Out. For the following demonstration, we intentionally insert a memory leak code into the code. We insert the following code into the <code>open</code> function of <code>StorageEngine</code>.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> char* leak_buf = new char[1024];</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> strcpy(leak_buf, "hello world");</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> LOG(INFO) << leak_buf;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We get the following output in be.out</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">=================================================================</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">==24732==ERROR: LeakSanitizer: detected memory leaks</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Direct leak of 1024 byte(s) in 1 object(s) allocated from:</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #0 0xd10586 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/lsan/lsan_interceptors.cc:164</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #1 0xe333a2 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #2 0xd3cc96 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #3 0x7f573b5eebd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">SUMMARY: LeakSanitizer: 1024 byte(s) leaked in 1 allocation(s).</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>From the above output, we can see that 1024 bytes have been leaked, and the stack information of memory application has been printed out.</p><p><strong>NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.</strong></p><p><strong>NOTE: if the LSAN switch is turned on, the TCMalloc will be automatically turned off</strong></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="jemalloc-heap-profile">JEMALLOC HEAP PROFILE<a href="#jemalloc-heap-profile" class="hash-link" aria-label="Direct link to JEMALLOC HEAP PROFILE" title="Direct link to JEMALLOC HEAP PROFILE"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="1-runtime-heap-dump-by-http">1. runtime heap dump by http<a href="#1-runtime-heap-dump-by-http" class="hash-link" aria-label="Direct link to 1. runtime heap dump by http" title="Direct link to 1. runtime heap dump by http"></a></h5><p>Add <code>,prof:true,lg_prof_sample:10</code> to <code>JEMALLOC_CONF</code> in <code>start_be.sh</code> and restart BE, then use the jemalloc heap dump http interface to generate a heap dump file on the corresponding BE machine.</p><p>The directory where the heap dump file is located can be configured through the <code>jeprofile_dir</code> variable in <code>be.conf</code>, and the default is <code>${DORIS_HOME}/log</code></p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token function" style="color:rgb(80, 250, 123)">curl</span><span class="token plain"> http://be_host:be_webport/jeheap/dump</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>prof</code>: After opening, jemalloc will generate a heap dump file according to the current memory usage. There is a small amount of performance loss in heap profile sampling, which can be turned off during performance testing. |
| <code>lg_prof_sample</code>: heap profile sampling interval, the default value is 19, that is, the default sampling interval is 512K (2^19 B), which will result in only 10% of the memory recorded by the heap profile, <code>lg_prof_sample:10</code> can reduce the sampling interval to 1K (2^10 B), more frequent sampling will make the heap profile close to real memory, but this will bring greater performance loss.</p><p>For detailed parameter description, refer to <a href="https://linux.die.net/man/3/jemalloc" target="_blank" rel="noopener noreferrer">https://linux.die.net/man/3/jemalloc</a>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="2-jemalloc-heap-dump-profiling">2. jemalloc heap dump profiling<a href="#2-jemalloc-heap-dump-profiling" class="hash-link" aria-label="Direct link to 2. jemalloc heap dump profiling" title="Direct link to 2. jemalloc heap dump profiling"></a></h4><p>3.1 Generating plain text analysis results</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jeprof lib/doris_be --base</span><span class="token operator">=</span><span class="token plain">heap_dump_file_1 heap_dump_file_2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>3.2 Generate call relationship picture</p><p> Install dependencies required for plotting</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">yum </span><span class="token function" style="color:rgb(80, 250, 123)">install</span><span class="token plain"> ghostscript graphviz</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> Multiple dump files can be generated by running the above command multiple times in a short period of time, and the first dump file can be selected as the baseline for diff comparison analysis</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jeprof --dot lib/doris_be --base</span><span class="token operator">=</span><span class="token plain">heap_dump_file_1 heap_dump_file_2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> After executing the above command, the terminal will output a diagram of dot syntax, and paste it to <a href="http://www.webgraphviz.com/" target="_blank" rel="noopener noreferrer">online dot drawing website</a>, generate a memory allocation diagram, and then analyze it. This method can Drawing directly through the terminal output results is more suitable for servers where file transfer is not very convenient.</p><p> You can also use the following command to directly generate the call relationship result.pdf file and transfer it to the local for viewing</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">jeprof --pdf lib/doris_be --base</span><span class="token operator">=</span><span class="token plain">heap_dump_file_1 heap_dump_file_2 </span><span class="token operator">></span><span class="token plain"> result.pdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the above jeprof related commands, remove the <code>--base</code> option to analyze only a single heap dump file</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="3-heap-dump-by-jemalloc_conf">3. heap dump by JEMALLOC_CONF<a href="#3-heap-dump-by-jemalloc_conf" class="hash-link" aria-label="Direct link to 3. heap dump by JEMALLOC_CONF" title="Direct link to 3. heap dump by JEMALLOC_CONF"></a></h5><p>Perform heap dump by restarting BE after changing the <code>JEMALLOC_CONF</code> variable in <code>start_be.sh</code></p><ol><li><p>Dump every 1MB:</p><p>Two new variable settings <code>prof:true,lg_prof_interval:20</code> have been added to the <code>JEMALLOC_CONF</code> variable, where <code>prof:true</code> is to enable profiling, and <code>lg_prof_interval:20</code> means that a dump is generated every 1MB (2^20)</p></li><li><p>Dump each time a new high is reached:</p><p>Added two variable settings <code>prof:true,prof_gdump:true</code> in the <code>JEMALLOC_CONF</code> variable, where <code>prof:true</code> is to enable profiling, and <code>prof_gdump:true</code> means to generate a dump when the memory usage reaches a new high</p></li><li><p>Memory leak dump when the program exits:</p><p>Added three new variable settings <code>prof_leak: true, lg_prof_sample: 0, prof_final: true</code> in the <code>JEMALLOC_CONF</code> variable</p></li></ol><h4 class="anchor anchorWithStickyNavbar_LWe7" id="asan">ASAN<a href="#asan" class="hash-link" aria-label="Direct link to ASAN" title="Direct link to ASAN"></a></h4><p>Except for the unreasonable use and leakage of memory. Sometimes there will be memory access illegal address and other errors. At this time, we can use <a href="https://github.com/google/sanitizers/wiki/addresssanitizer" target="_blank" rel="noopener noreferrer">ASAN</a> to help us find the cause of the problem. Like LSAN, ASAN is integrated into GCC. Doris can open this function by compiling as follows</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">BUILD_TYPE=ASAN ./build.sh</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Execute the binary generated by compilation. When the detection tool finds any abnormal access, it will immediately exit and output the stack illegally accessed in be.out. The output of ASAN is the same as that of LSAN. Here we also actively inject an address access error to show the specific content output. We still inject an illegal memory access into the 'open' function of 'storageengine'. The specific error code is as follows</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> char* invalid_buf = new char[1024];</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> for (int i = 0; i < 1025; ++i) {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> invalid_buf[i] = i;</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> LOG(INFO) << invalid_buf;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We get the following output in be.out</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">=================================================================</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">==23284==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x61900008bf80 at pc 0x00000129f56a bp 0x7fff546eed90 sp 0x7fff546eed88</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">WRITE of size 1 at 0x61900008bf80 thread T0</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #0 0x129f569 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #1 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #2 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #3 0xd30794 (/home/ssd0/zc/palo/doris/core/output3/be/lib/doris_be+0xd30794)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">0x61900008bf80 is located 0 bytes to the right of 1024-byte region [0x61900008bb80,0x61900008bf80)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">allocated by thread T0 here:</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #0 0xdeb040 in operator new[](unsigned long) ../../../../gcc-7.3.0/libsanitizer/asan/asan_new_delete.cc:82</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #1 0x129f50d in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**) /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:104</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #2 0xe2c1e3 in main /home/ssd0/zc/palo/doris/core/be/src/service/doris_main.cpp:159</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> #3 0x7fa5580fbbd4 in __libc_start_main (/opt/compiler/gcc-4.8.2/lib64/libc.so.6+0x21bd4)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">SUMMARY: AddressSanitizer: heap-buffer-overflow /home/ssd0/zc/palo/doris/core/be/src/olap/storage_engine.cpp:106 in doris::StorageEngine::open(doris::EngineOptions const&, doris::StorageEngine**)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>From this message, we can see that at the address of <code>0x61900008bf80</code>, we tried to write a byte, but this address is illegal. We can also see the application stack of the address <code>[0x61900008bb80, 0x61900008bf80]</code>.</p><p><strong>NOTE: turning on this option will affect the execution performance of the program. Please be careful to turn on the online instance.</strong></p><p><strong>NOTE: if the ASAN switch is turned on, the TCMalloc will be automatically turned off</strong></p><p>In addition, if stack information is output in be.out, but there is no function symbol, then we need to handle it manually to get readable stack information. The specific processing method needs a script to parse the output of ASAN. At this time, we need to use <a href="https://llvm.org/svn/llvm-project/compiler-rt/trunk/lib/asan/scripts/asan_symbolize.py" target="_blank" rel="noopener noreferrer">asan_symbolize</a> to help with parsing. The specific usage is as follows:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">cat be.out | python asan_symbolize.py | c++filt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the above command, we can get readable stack information.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cpu">CPU<a href="#cpu" class="hash-link" aria-label="Direct link to CPU" title="Direct link to CPU"></a></h3><p>When the CPU idle of the system is very low, it means that the CPU of the system has become the main bottleneck. At this time, it is necessary to analyze the current CPU usage. For the be of Doris, there are two ways to analyze the CPU bottleneck of Doris.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="pprof">pprof<a href="#pprof" class="hash-link" aria-label="Direct link to pprof" title="Direct link to pprof"></a></h4><p>Because Doris has integrated and compatible with GPERF rest interface, users can analyze remote Doris be through the 'pprof' tool. The specific usage is as follows:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">pprof --svg --seconds=60 http://be_host:be_webport/pprof/profile > be.svg </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In this way, a CPU consumption graph of be execution can be generated.</p><p><img loading="lazy" alt="CPU Pprof" src="https://cdnd.selectdb.com/assets/images/cpu-pprof-demo-46064afc3672559af190f0698ed9ef52.png" width="1196" height="1488" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="perf--flamegragh">perf + flamegragh<a href="#perf--flamegragh" class="hash-link" aria-label="Direct link to perf + flamegragh" title="Direct link to perf + flamegragh"></a></h4><p>This is a quite common CPU analysis method. Compared with <code>pprof</code>, this method must be able to log in to the physical machine of the analysis object. However, compared with pprof, which can only collect points on time, perf can collect stack information through different events. The specific usage is as follows:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">perf record -g -p be_pid -- sleep 60</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This command counts the CPU operation of be for 60 seconds and generates perf.data. For the analysis of perf.data, the command of perf can be used for analysis.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">perf report</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The analysis results in the following pictures</p><p><img loading="lazy" alt="Perf Report" src="https://cdnd.selectdb.com/assets/images/perf-report-demo-5987234ce899968a08be49a29f51acb1.png" width="1102" height="412" class="img_ev3q"></p><p>To analyze the generated content. Of course, you can also use flash graph to complete the visual display.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">perf script | ./FlameGraph/stackcollapse-perf.pl | ./FlameGraph/flamegraph.pl > be.svg</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This will also generate a graph of CPU consumption at that time.</p><p><img loading="lazy" alt="CPU Flame" src="https://cdnd.selectdb.com/assets/images/cpu-flame-demo-727a5b8c32db9b3b129481ac75405b5d.svg" width="1200" height="418" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/community/design/spark_load"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Spark Load</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/community/developer-guide/docker-dev"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Doris Docker quick build development environment</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_jeP5 thin-scrollbar theme-doc-toc-desktop"><a href="https://github.com/apache/doris/discussions" target="_blank" rel="noopener noreferrer" class="ml-4 mb-8 flex items-center hover:no-underline"><span class="pr-2">Join Discussion</span><svg viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8500" id="mx_n_1711090272569" width="16" height="16"><path d="M522.24 896.512c-25.6 4.608-51.712 7.168-78.336 7.168-79.36 0-157.696-21.504-225.792-62.464l-18.432-10.752-103.936 28.16c-28.672 7.68-54.784-18.432-47.104-47.104l28.16-103.936c-10.752-17.92-17.408-30.208-20.992-36.864C20.992 607.232 3.072 536.064 3.584 463.36c0-243.2 197.12-440.32 440.32-440.32 221.696 0 405.504 164.352 435.712 377.856 90.112 55.808 144.896 154.112 144.896 260.096 0 51.2-12.8 100.352-36.352 144.384-2.048 4.096-6.144 10.752-11.776 20.48l17.408 64c7.68 28.672-18.432 54.784-47.104 47.104l-64-17.408-7.68 4.608c-47.616 28.672-101.888 43.52-157.184 43.52-71.68-0.512-140.8-25.088-195.584-71.168z m95.232-28.672c31.232 15.36 65.536 23.04 100.352 23.04 41.472 0 82.432-11.264 117.76-32.768 2.56-1.536 9.728-5.632 22.016-12.8 8.704-5.12 19.456-6.656 29.184-3.584l14.848 4.096-4.096-14.848c-2.56-10.24-1.536-20.48 4.096-29.696 6.144-10.24 12.288-20.992 18.432-31.232 17.92-33.28 27.136-70.656 27.136-108.544 0-59.904-23.552-117.76-65.536-160.256-13.312 164.352-118.272 303.616-264.192 366.592z m-462.848-155.648l-14.848 54.784 54.784-14.848c9.728-2.56 20.48-1.536 29.184 4.096 18.432 10.752 29.184 16.896 32.768 19.456 56.32 33.792 120.832 51.712 186.88 51.712 200.704 0 363.52-162.816 363.52-363.52s-162.816-363.52-363.52-363.52-363.52 162.816-363.52 363.52c0 60.928 14.848 119.296 43.008 171.52 3.584 7.168 13.312 23.04 27.648 47.616 5.632 8.704 6.656 19.456 4.096 29.184z m448.512-382.976c20.992 0 38.4 16.896 38.4 38.4 0 20.992-16.896 38.4-38.4 38.4H284.16c-20.992 0-38.4-16.896-38.4-38.4 0-20.992 16.896-38.4 38.4-38.4h318.976z m-153.088 191.488c20.992 0 38.4 16.896 38.4 38.4 0 20.992-16.896 38.4-38.4 38.4H284.16c-20.992 0-38.4-16.896-38.4-38.4 0-20.992 16.896-38.4 38.4-38.4h165.888z m0 0" p-id="8501" fill="currentColor"></path></svg></a><span class="ml-4">On This Page</span><ul class="table-of-contents table-of-contents__left-border"><li><a href="#fe-debugging" class="table-of-contents__link toc-highlight">FE debugging</a></li><li><a href="#be-debugging" class="table-of-contents__link toc-highlight">BE debugging</a><ul><li><a href="#preparing" class="table-of-contents__link toc-highlight">Preparing</a></li><li><a href="#memory" class="table-of-contents__link toc-highlight">Memory</a></li><li><a href="#cpu" class="table-of-contents__link toc-highlight">CPU</a></li></ul></li></ul></div></div></div></div></main></div></div></div><div class="footer pt-16 pb-10"><div class="container"><div class="footer-box"><div class="left"><img src="/images/asf_logo_apache.svg" alt="" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/images/asf_logo_apache.svg" alt="" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">ASF</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Privacy<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Security<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">Resources</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/download">Download</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/get-starting/quick-start">Docs</a></li><li class="footer__item"><a class="footer__link-item" href="/blog">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/ecosystem/cluster-management">Ecosystem</a></li><li class="footer__item"><a class="footer__link-item" href="/users">Users</a></li><li class="footer__item"><a href="https://github.com/apache/doris/discussions" target="_blank" rel="noopener noreferrer" class="footer__link-item">Discussions<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/community/how-to-contribute/">How to contribute</a></li><li class="footer__item"><a href="https://github.com/apache/doris/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Source code<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://cwiki.apache.org/confluence/display/DORIS/Doris+Improvement+Proposals" target="_blank" rel="noopener noreferrer" class="footer__link-item">Improvement proposal<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/community/team">Doris team</a></li><li class="footer__item"><a href="https://github.com/apache/doris/issues/30669" target="_blank" rel="noopener noreferrer" class="footer__link-item">Roadmap<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div></div><div class="right"><div class="footer__title">Join the community</div><div class="social-list"><div class="social"><a href="mailto:dev@doris.apache.org" target="_blank" title="mail" class="item"><svg xmlns="http://www.w3.org/2000/svg" width="2em" height="2em" viewBox="0 0 32 32" fill="none"><path d="M5.6003 6H26.3997C27.8186 6 28.982 7.10964 29 8.46946L16.0045 15.454L3.01202 8.47829C3.02405 7.11258 4.1784 6 5.6003 6ZM3.01202 11.1508L3 23.5011C3 24.8756 4.16938 26 5.6003 26H26.3997C27.8306 26 29 24.8756 29 23.5011V11.145L16.3111 17.8028C16.1157 17.9058 15.8813 17.9058 15.6889 17.8028L3.01202 11.1508Z" fill="currentColor"></path></svg></a><a href="https://github.com/apache/doris" target="_blank" title="github" class="item"><svg width="2em" height="2em" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M16.0001 2.66675C8.63342 2.66675 2.66675 8.63341 2.66675 16.0001C2.66524 18.7991 3.54517 21.5276 5.1817 23.7983C6.81824 26.0691 9.12828 27.7668 11.7841 28.6508C12.4508 28.7668 12.7001 28.3668 12.7001 28.0161C12.7001 27.7001 12.6828 26.6508 12.6828 25.5334C9.33342 26.1508 8.46675 24.7174 8.20008 23.9668C8.04942 23.5828 7.40008 22.4001 6.83342 22.0828C6.36675 21.8334 5.70008 21.2161 6.81608 21.2001C7.86675 21.1828 8.61608 22.1668 8.86675 22.5668C10.0668 24.5828 11.9841 24.0161 12.7494 23.6668C12.8668 22.8001 13.2161 22.2174 13.6001 21.8841C10.6334 21.5508 7.53342 20.4001 7.53342 15.3001C7.53342 13.8494 8.04942 12.6507 8.90008 11.7161C8.76675 11.3827 8.30008 10.0161 9.03342 8.18275C9.03342 8.18275 10.1494 7.83342 12.7001 9.55075C13.7855 9.2495 14.907 9.09787 16.0334 9.10008C17.1668 9.10008 18.3001 9.24942 19.3668 9.54942C21.9161 7.81608 23.0334 8.18408 23.0334 8.18408C23.7668 10.0174 23.3001 11.3841 23.1668 11.7174C24.0161 12.6507 24.5334 13.8334 24.5334 15.3001C24.5334 20.4174 21.4174 21.5508 18.4508 21.8841C18.9334 22.3001 19.3508 23.1001 19.3508 24.3508C19.3508 26.1334 19.3334 27.5668 19.3334 28.0174C19.3334 28.3668 19.5841 28.7828 20.2508 28.6494C22.8975 27.7558 25.1973 26.0547 26.8266 23.7856C28.4559 21.5165 29.3327 18.7936 29.3334 16.0001C29.3334 8.63341 23.3668 2.66675 16.0001 2.66675V2.66675Z" fill="currentColor"></path></svg></a><a href="https://twitter.com/doris_apache" target="_blank" title="twitter" class="item"><svg xmlns="http://www.w3.org/2000/svg" width="2em" height="2em" viewBox="0 0 32 32" fill="none"><path d="M4.625 4.625H11.2809L27.375 27.375H20.7191L4.625 4.625ZM7.52549 6.10639L21.5236 25.8936H24.4746L10.4764 6.10639H7.52549Z" fill="currentColor"></path><path d="M14.4268 18.4803L6.53447 27.375H4.625L13.5581 17.2525L14.4268 18.4803ZM18.1299 14.3066L26.7203 4.625H24.7017L17.2525 13.0662L18.1299 14.3066Z" fill="currentColor"></path></svg></a><a href="https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-2kl08hzc0-SPJe4VWmL_qzrFd2u2XYQA" title="slack" target="_blank" class="item"><svg width="2em" height="2em" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_125_278)"><path d="M12.5875 16.6906C11.0844 16.6906 9.86562 17.9094 9.86562 19.4125V26.2375C9.86562 26.9594 10.1524 27.6517 10.6628 28.1622C11.1733 28.6726 11.8656 28.9594 12.5875 28.9594C13.3094 28.9594 14.0017 28.6726 14.5122 28.1622C15.0226 27.6517 15.3094 26.9594 15.3094 26.2375V19.4531C15.3094 17.9094 14.0906 16.6906 12.5875 16.6906ZM3 19.4531C3 20.175 3.28677 20.8673 3.79722 21.3778C4.30767 21.8882 4.99999 22.175 5.72187 22.175C6.44376 22.175 7.13608 21.8882 7.64653 21.3778C8.15698 20.8673 8.44375 20.175 8.44375 19.4531V16.7312H5.7625C4.25938 16.6906 3 17.9094 3 19.4531ZM12.5875 3C11.8656 3 11.1733 3.28677 10.6628 3.79722C10.1524 4.30767 9.86562 4.99999 9.86562 5.72187C9.86562 6.44376 10.1524 7.13608 10.6628 7.64653C11.1733 8.15698 11.8656 8.44375 12.5875 8.44375H15.3094V5.72187C15.3094 4.21875 14.0906 3 12.5875 3ZM5.72187 15.3094H12.5469C13.2688 15.3094 13.9611 15.0226 14.4715 14.5122C14.982 14.0017 15.2688 13.3094 15.2688 12.5875C15.2688 11.8656 14.982 11.1733 14.4715 10.6628C13.9611 10.1524 13.2688 9.86562 12.5469 9.86562H5.72187C4.99999 9.86562 4.30767 10.1524 3.79722 10.6628C3.28677 11.1733 3 11.8656 3 12.5875C3 13.3094 3.28677 14.0017 3.79722 14.5122C4.30767 15.0226 4.99999 15.3094 5.72187 15.3094ZM26.2375 9.86562C24.7344 9.86562 23.5156 11.0844 23.5156 12.5875V15.3094H26.2375C26.9594 15.3094 27.6517 15.0226 28.1622 14.5122C28.6726 14.0017 28.9594 13.3094 28.9594 12.5875C28.9594 11.8656 28.6726 11.1733 28.1622 10.6628C27.6517 10.1524 26.9594 9.86562 26.2375 9.86562ZM16.6906 5.72187V12.5875C16.6906 13.3094 16.9774 14.0017 17.4878 14.5122C17.9983 15.0226 18.6906 15.3094 19.4125 15.3094C20.1344 15.3094 20.8267 15.0226 21.3372 14.5122C21.8476 14.0017 22.1344 13.3094 22.1344 12.5875V5.72187C22.1344 4.99999 21.8476 4.30767 21.3372 3.79722C20.8267 3.28677 20.1344 3 19.4125 3C18.6906 3 17.9983 3.28677 17.4878 3.79722C16.9774 4.30767 16.6906 4.99999 16.6906 5.72187ZM22.1344 26.2781C22.1344 24.775 20.9156 23.5562 19.4125 23.5562H16.6906V26.2781C16.6906 27 16.9774 27.6923 17.4878 28.2028C17.9983 28.7132 18.6906 29 19.4125 29C20.1344 29 20.8267 28.7132 21.3372 28.2028C21.8476 27.6923 22.1344 27 22.1344 26.2781ZM26.2781 16.6906H19.4125C18.6906 16.6906 17.9983 16.9774 17.4878 17.4878C16.9774 17.9983 16.6906 18.6906 16.6906 19.4125C16.6906 20.1344 16.9774 20.8267 17.4878 21.3372C17.9983 21.8476 18.6906 22.1344 19.4125 22.1344H26.2375C27.7406 22.1344 28.9594 20.9156 28.9594 19.4125C29 17.9094 27.7812 16.6906 26.2781 16.6906Z" fill="currentColor"></path></g><defs><clipPath id="clip0_125_278"><rect width="26" height="26" fill="currentColor" transform="translate(3 3)"></rect></clipPath></defs></svg></a></div><div class="social"><a href="https://www.youtube.com/@apachedoris/channels" title="youtube" target="_blank" class="item"><svg xmlns="http://www.w3.org/2000/svg" width="2em" height="2em" viewBox="0 0 32 32" fill="none"><path d="M28.5167 7.83429C28.9436 8.25423 29.2532 8.77539 29.4154 9.34742C29.8205 11.5462 30.0159 13.7775 29.999 16.0121C30.0144 18.2382 29.819 20.4609 29.4154 22.6515C29.2532 23.2235 28.9436 23.7446 28.5167 24.1645C28.0898 24.5845 27.5601 24.889 26.9785 25.0486C24.7728 25.625 16.0124 25.625 16.0124 25.625C16.0124 25.625 7.22652 25.625 5.04638 25.0486C4.46489 24.889 3.9351 24.5845 3.5082 24.1645C3.08132 23.7446 2.77176 23.2235 2.60948 22.6515C2.19736 20.4617 1.9934 18.239 2.00025 16.0121C1.9918 13.7767 2.19577 11.5455 2.60948 9.34742C2.77176 8.77539 3.08132 8.25423 3.5082 7.83429C3.9351 7.41436 4.46489 7.10985 5.04638 6.95021C7.25103 6.36354 16.0124 6.37502 16.0124 6.37502C16.0124 6.37502 24.796 6.37502 26.9785 6.95021C27.5601 7.10985 28.0898 7.41436 28.5167 7.83429ZM12.5 21.25L21.25 16.008L12.5 10.75V21.25Z" fill="currentColor"></path></svg></a><a href="https://www.linkedin.com/company/doris-apache/" title="linkedin" target="_blank" class="item"><svg width="2rem" height="2rem" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M4.29925 26.9996H9.66738V11.6781H4.29925V26.9996ZM22.1628 11.1949C19.9409 11.1949 18.7157 11.9388 17.3054 13.7407V11.6777H11.9459V26.9996H17.305V18.6738C17.305 16.9168 18.145 15.1982 20.1535 15.1982C22.162 15.1982 22.6559 16.9164 22.6559 18.632V27H28V18.2902C28 12.2386 24.3854 11.1949 22.1628 11.1949ZM6.99325 4C5.3395 4 4 5.21047 4 6.7046C4 8.19759 5.3395 9.40617 6.99325 9.40617C8.6455 9.40617 9.985 8.19722 9.985 6.7046C9.985 5.21047 8.6455 4 6.99325 4Z" fill="white"></path></svg></a><a href="https://medium.com/@ApacheDoris" title="medium" target="_blank" class="item"><svg width="2em" height="2em" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"><g id="Frame"><path id="Vector" d="M17.7967 16.5385C17.8029 18.53 16.9746 20.4425 15.4937 21.8559C14.0128 23.2693 12.0004 24.0681 9.89836 24.0769C7.79633 24.0681 5.78391 23.2693 4.30302 21.8559C2.82212 20.4425 1.99383 18.53 2.00003 16.5385C1.99383 14.5469 2.82212 12.6344 4.30302 11.221C5.78391 9.80759 7.79633 9.00878 9.89836 9C12.0004 9.00878 14.0128 9.80759 15.4937 11.221C16.9746 12.6344 17.8029 14.5469 17.7967 16.5385ZM26.4533 16.5385C26.4533 20.4514 24.6917 23.6348 22.51 23.6348C20.3283 23.6348 18.555 20.4514 18.555 16.5385C18.555 12.6255 20.3283 9.44214 22.51 9.44214C24.6917 9.44214 26.4533 12.6255 26.4533 16.5385ZM30 16.5385C30 20.0424 29.3817 22.8942 28.6117 22.8942C27.8417 22.8942 27.2233 20.0424 27.2233 16.5385C27.2233 13.0345 27.8417 10.1827 28.6117 10.1827C29.3817 10.1827 30 13.0345 30 16.5385Z" fill="currentColor"></path></g></svg></a><a class="item wechat"><svg width="2em" height="2em" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M20.7578 11.5169C21.0708 11.5169 21.3795 11.5398 21.6851 11.573C20.8524 7.73517 16.7052 4.88306 11.9718 4.88306C6.67951 4.88306 2.34412 8.45283 2.34412 12.9854C2.34412 15.6013 3.78679 17.7498 6.19667 19.4161L5.2339 22.2827L8.59917 20.6122C9.80411 20.8478 10.7698 21.0906 11.9718 21.0906C12.2738 21.0906 12.5728 21.0759 12.8703 21.0523C12.682 20.4159 12.5728 19.7485 12.5728 19.0566C12.5728 14.8947 16.1847 11.5169 20.7578 11.5169ZM15.5822 8.9335C16.3072 8.9335 16.7871 9.40601 16.7871 10.1229C16.7871 10.8369 16.3072 11.3153 15.5822 11.3153C14.8601 11.3153 14.1365 10.8369 14.1365 10.1229C14.1365 9.40601 14.8601 8.9335 15.5822 8.9335ZM8.84429 11.3153C8.12218 11.3153 7.3942 10.8368 7.3942 10.1229C7.3942 9.40597 8.12218 8.93346 8.84429 8.93346C9.56559 8.93346 10.0463 9.40597 10.0463 10.1229C10.0463 10.8369 9.56559 11.3153 8.84429 11.3153ZM29.5453 18.9422C29.5453 15.1332 25.6935 12.0285 21.3677 12.0285C16.7871 12.0285 13.1797 15.1332 13.1797 18.9422C13.1797 22.7567 16.7871 25.8547 21.3677 25.8547C22.326 25.8547 23.2932 25.6169 24.2559 25.3777L26.897 26.8086L26.1726 24.4282C28.1056 22.993 29.5453 21.0906 29.5453 18.9422ZM18.7126 17.7498C18.2335 17.7498 17.7499 17.278 17.7499 16.7966C17.7499 16.3219 18.2335 15.8442 18.7126 15.8442C19.4406 15.8442 19.9176 16.3219 19.9176 16.7966C19.9176 17.278 19.4406 17.7498 18.7126 17.7498ZM24.0079 17.7498C23.5324 17.7498 23.0518 17.278 23.0518 16.7966C23.0518 16.3219 23.5324 15.8442 24.0079 15.8442C24.73 15.8442 25.2128 16.3219 25.2128 16.7966C25.2128 17.278 24.73 17.7498 24.0079 17.7498Z" fill="currentColor"></path></svg><div class="wechat-dropdown"><p class="text-[#4c576c] text-xs">Connect on WeChat</p><img src="https://cdnd.selectdb.com/assets/images/doris-wechat-b949e908a3bc2776d824f79a9100bd4b.png" alt=""></div></a></div></div></div></div><div class="footer__copyright">Copyright © 2024 The Apache Software Foundation,Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0" target="_blank">Apache License, Version 2.0</a>. Apache, Doris, Apache Doris, the Apache feather logo and the Apache Doris logo are trademarks of The Apache Software Foundation.</div></div></div></div> |
| <script src="https://cdnd.selectdb.com/assets/js/runtime~main.eb208fba.js"></script> |
| <script src="https://cdnd.selectdb.com/assets/js/main.fa7fcb85.js"></script> |
| </body> |
| </html> |
