| <!doctype html> |
| <html class="docs-version-0.9.0" lang="cn" dir="ltr"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <meta name="generator" content="Docusaurus v2.0.0-beta.14"> |
| <link rel="alternate" type="application/rss+xml" href="/cn/blog/rss.xml" title="Apache Hudi: User-Facing Analytics RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/cn/blog/atom.xml" title="Apache Hudi: User-Facing Analytics Atom Feed"> |
| <link rel="alternate" type="application/json" href="/cn/blog/feed.json" title="Apache Hudi: User-Facing Analytics JSON Feed"> |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache Hudi" href="/cn/opensearch.xml"> |
| <link rel="alternate" type="application/rss+xml" href="/cn/videos/rss.xml" title="Apache Hudi RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/cn/videos/atom.xml" title="Apache Hudi Atom Feed"> |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Comfortaa|Ubuntu|Roboto|Source+Code+Pro"> |
| <link rel="stylesheet" href="https://at-ui.github.io/feather-font/css/iconfont.css"><title data-react-helmet="true">管理 Hudi Pipelines | Apache Hudi</title><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"><meta data-react-helmet="true" property="og:url" content="https://hudi.apache.org/cn/docs/0.9.0/deployment"><meta data-react-helmet="true" name="docsearch:language" content="cn"><meta data-react-helmet="true" name="docsearch:version" content="0.9.0"><meta data-react-helmet="true" name="docsearch:docusaurus_tag" content="docs-default-0.9.0"><meta data-react-helmet="true" property="og:title" content="管理 Hudi Pipelines | Apache Hudi"><meta data-react-helmet="true" name="description" content="管理员/运维人员可以通过以下方式了解Hudi数据集/管道"><meta data-react-helmet="true" property="og:description" content="管理员/运维人员可以通过以下方式了解Hudi数据集/管道"><meta data-react-helmet="true" name="keywords" content="hudi,administration,operation,devops"><link data-react-helmet="true" rel="icon" href="/cn/assets/images/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://hudi.apache.org/cn/docs/0.9.0/deployment"><link data-react-helmet="true" rel="alternate" href="https://hudi.apache.org/docs/0.9.0/deployment" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://hudi.apache.org/cn/docs/0.9.0/deployment" hreflang="cn"><link data-react-helmet="true" rel="alternate" href="https://hudi.apache.org/docs/0.9.0/deployment" hreflang="x-default"><link data-react-helmet="true" rel="preconnect" href="https://BH4D9OD16A-dsn.algolia.net" crossorigin="anonymous"><link rel="stylesheet" href="/cn/assets/css/styles.052701b1.css"> |
| <link rel="preload" href="/cn/assets/js/runtime~main.e00cff1c.js" as="script"> |
| <link rel="preload" href="/cn/assets/js/main.a490f48b.js" as="script"> |
| </head> |
| <body> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div><a href="#" class="skipToContent_OuoZ">Skip to main content</a></div><div class="announcementBar_axC9" role="banner"><div class="announcementBarContent_6uhP">⭐️ If you like <b>Apache Hudi</b>, give it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/apache/hudi"><b>GitHub!<svg xmlns="http://www.w3.org/2000/svg\" width="16" height="16" fill="currentColor" class="bi bi-github" viewBox="0 -2 16 16"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg></b></a> ⭐</div></div><nav class="navbar navbar--fixed-top navbarWrapper_UIa0"><div class="navbar__inner navbarInnerStyle_FVs7"><img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=8f594acf-9b77-44fb-9475-3e82ead1910c" width="0" height="0" alt=""><img referrerpolicy="no-referrer-when-downgrade" src="https://analytics.apache.org/matomo.php?idsite=47&rec=1" width="0" height="0" alt=""><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle clean-btn" type="button" tabindex="0"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/cn/"><div class="navbar__logo navbarLogo_Bz6n"><img src="/cn/assets/images/hudi.png" alt="Apache Hudi" class="themedImage_TMUO themedImage--light_4Vu1"><img src="/cn/assets/images/hudi.png" alt="Apache Hudi" class="themedImage_TMUO themedImage--dark_uzRr"></div></a><a class="navbar__item navbar__link navbarFontSize_x1wi" href="/cn/docs/overview"><div class="labelWrapperDropdown_Mqbj">Docs</div></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" class="navbar__link downloadLinkDropdownHide_aDP3 navbarFontSize_x1wi"><div class="labelWrapperDropdown_Mqbj">Learn<svg width="10" height="6" viewBox="0 0 10 6" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M8.5 1.25 5 4.75l-3.5-3.5" stroke="#1C1E21" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></div></a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/cn/talks"><div class="labelWrapperDropdown_Mqbj">Talks</div></a></li><li><a class="dropdown__link" href="/cn/videos"><div class="labelWrapperDropdown_Mqbj">Video Guides</div></a></li><li><a class="dropdown__link" href="/cn/docs/faq"><div class="labelWrapperDropdown_Mqbj">FAQ</div></a></li><li><a class="dropdown__link" href="/cn/tech-specs"><div class="labelWrapperDropdown_Mqbj">Tech Specs</div></a></li><li><a class="dropdown__link" href="/cn/tech-specs-1point0"><div class="labelWrapperDropdown_Mqbj">Tech Specs 1.0</div></a></li><li><a href="https://cwiki.apache.org/confluence/display/HUDI" target="_blank" rel="noopener noreferrer" class="dropdown__link"><span class="externalLink_AE3f">Technical Wiki<svg width="20" height="20" viewBox="0 0 26 26" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M16.965 8.745 9.01 16.7M10.561 8.758l6.403-.013-.013 6.403" stroke="#0DB1F9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><rect x="4.5" y="4.5" width="17" height="17" rx="2.5" stroke="#0DB1F9"></rect></svg></span></a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" class="navbar__link downloadLinkDropdownHide_aDP3 navbarFontSize_x1wi"><div class="labelWrapperDropdown_Mqbj">Contribute<svg width="10" height="6" viewBox="0 0 10 6" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M8.5 1.25 5 4.75l-3.5-3.5" stroke="#1C1E21" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></div></a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/cn/contribute/how-to-contribute"><div class="labelWrapperDropdown_Mqbj">How to Contribute</div></a></li><li><a class="dropdown__link" href="/cn/contribute/developer-setup"><div class="labelWrapperDropdown_Mqbj">Developer Setup</div></a></li><li><a class="dropdown__link" href="/cn/contribute/rfc-process"><div class="labelWrapperDropdown_Mqbj">RFC Process</div></a></li><li><a class="dropdown__link" href="/cn/contribute/report-security-issues"><div class="labelWrapperDropdown_Mqbj">Report Security Issues</div></a></li><li><a href="https://issues.apache.org/jira/projects/HUDI/summary" target="_blank" rel="noopener noreferrer" class="dropdown__link"><span class="externalLink_AE3f">Report Issues<svg width="20" height="20" viewBox="0 0 26 26" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M16.965 8.745 9.01 16.7M10.561 8.758l6.403-.013-.013 6.403" stroke="#0DB1F9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><rect x="4.5" y="4.5" width="17" height="17" rx="2.5" stroke="#0DB1F9"></rect></svg></span></a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" class="navbar__link downloadLinkDropdownHide_aDP3 navbarFontSize_x1wi"><div class="labelWrapperDropdown_Mqbj">Community<svg width="10" height="6" viewBox="0 0 10 6" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M8.5 1.25 5 4.75l-3.5-3.5" stroke="#1C1E21" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></div></a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/cn/community/get-involved"><div class="labelWrapperDropdown_Mqbj">Get Involved</div></a></li><li><a class="dropdown__link" href="/cn/community/syncs"><div class="labelWrapperDropdown_Mqbj">Community Syncs</div></a></li><li><a class="dropdown__link" href="/cn/community/office_hours"><div class="labelWrapperDropdown_Mqbj">Office Hours</div></a></li><li><a class="dropdown__link" href="/cn/community/team"><div class="labelWrapperDropdown_Mqbj">Team</div></a></li></ul></div><a class="navbar__item navbar__link navbarFontSize_x1wi" href="/cn/blog"><div class="labelWrapperDropdown_Mqbj">Blog</div></a><a class="navbar__item navbar__link navbarFontSize_x1wi" href="/cn/powered-by"><div class="labelWrapperDropdown_Mqbj">Who's Using</div></a><a class="navbar__item navbar__link navbarFontSize_x1wi" href="/cn/roadmap"><div class="labelWrapperDropdown_Mqbj">Roadmap</div></a><a class="navbar__item navbar__link navbarFontSize_x1wi" href="/cn/releases/download"><div class="labelWrapperDropdown_Mqbj">Download</div></a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link downloadLinkDropdownHide_aDP3" href="/cn/docs/0.9.0/overview"><div class="labelWrapperDropdown_Mqbj">0.9.0<svg width="10" height="6" viewBox="0 0 10 6" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M8.5 1.25 5 4.75l-3.5-3.5" stroke="#1C1E21" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></div></a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/cn/docs/next/deployment"><div class="labelWrapperDropdown_Mqbj">Next</div></a></li><li><a class="dropdown__link" href="/cn/docs/deployment"><div class="labelWrapperDropdown_Mqbj">0.14.1</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.14.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.14.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.13.1/deployment"><div class="labelWrapperDropdown_Mqbj">0.13.1</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.13.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.13.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.12.3/deployment"><div class="labelWrapperDropdown_Mqbj">0.12.3</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.12.2/deployment"><div class="labelWrapperDropdown_Mqbj">0.12.2</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.12.1/deployment"><div class="labelWrapperDropdown_Mqbj">0.12.1</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.12.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.12.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.11.1/deployment"><div class="labelWrapperDropdown_Mqbj">0.11.1</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.11.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.11.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.10.1/deployment"><div class="labelWrapperDropdown_Mqbj">0.10.1</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.10.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.10.0</div></a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/cn/docs/0.9.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.9.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.8.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.8.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.7.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.7.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.6.0/deployment"><div class="labelWrapperDropdown_Mqbj">0.6.0</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.5.3/deployment"><div class="labelWrapperDropdown_Mqbj">0.5.3</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.5.2/deployment"><div class="labelWrapperDropdown_Mqbj">0.5.2</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.5.1/deployment"><div class="labelWrapperDropdown_Mqbj">0.5.1</div></a></li><li><a class="dropdown__link" href="/cn/docs/0.5.0/quick-start-guide"><div class="labelWrapperDropdown_Mqbj">0.5.0</div></a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" class="navbar__link downloadLinkDropdownHide_aDP3"><div class="labelWrapperDropdown_Mqbj"><span><svg viewBox="0 0 20 20" width="20" height="20" aria-hidden="true" class="iconLanguage_zID8"><path fill="currentColor" d="M19.753 10.909c-.624-1.707-2.366-2.726-4.661-2.726-.09 0-.176.002-.262.006l-.016-2.063 3.525-.607c.115-.019.133-.119.109-.231-.023-.111-.167-.883-.188-.976-.027-.131-.102-.127-.207-.109-.104.018-3.25.461-3.25.461l-.013-2.078c-.001-.125-.069-.158-.194-.156l-1.025.016c-.105.002-.164.049-.162.148l.033 2.307s-3.061.527-3.144.543c-.084.014-.17.053-.151.143.019.09.19 1.094.208 1.172.018.08.072.129.188.107l2.924-.504.035 2.018c-1.077.281-1.801.824-2.256 1.303-.768.807-1.207 1.887-1.207 2.963 0 1.586.971 2.529 2.328 2.695 3.162.387 5.119-3.06 5.769-4.715 1.097 1.506.256 4.354-2.094 5.98-.043.029-.098.129-.033.207l.619.756c.08.096.206.059.256.023 2.51-1.73 3.661-4.515 2.869-6.683zm-7.386 3.188c-.966-.121-.944-.914-.944-1.453 0-.773.327-1.58.876-2.156a3.21 3.21 0 011.229-.799l.082 4.277a2.773 2.773 0 01-1.243.131zm2.427-.553l.046-4.109c.084-.004.166-.01.252-.01.773 0 1.494.145 1.885.361.391.217-1.023 2.713-2.183 3.758zm-8.95-7.668a.196.196 0 00-.196-.145h-1.95a.194.194 0 00-.194.144L.008 16.916c-.017.051-.011.076.062.076h1.733c.075 0 .099-.023.114-.072l1.008-3.318h3.496l1.008 3.318c.016.049.039.072.113.072h1.734c.072 0 .078-.025.062-.076-.014-.05-3.083-9.741-3.494-11.04zm-2.618 6.318l1.447-5.25 1.447 5.25H3.226z"></path></svg><span>Chinese</span></span><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#a)"><path d="M14 6.457a6.842 6.842 0 0 0-7-6.02 6.843 6.843 0 0 0-7 6.02v1.085a6.843 6.843 0 0 0 7 6.02 6.843 6.843 0 0 0 7-6.02V6.457Zm-1.094 0h-2.625a9.92 9.92 0 0 0-.376-2.222 6.65 6.65 0 0 0 1.531-.875 5.25 5.25 0 0 1 1.444 3.097h.026Zm-8.032 0a8.479 8.479 0 0 1 .324-1.872 7.376 7.376 0 0 0 3.63 0c.175.61.284 1.239.325 1.872h-4.28Zm4.305 1.085a8.391 8.391 0 0 1-.324 1.873 7.464 7.464 0 0 0-3.658 0 8.479 8.479 0 0 1-.323-1.873h4.305Zm.35-4.375A10.342 10.342 0 0 0 8.75 1.75c.627.194 1.218.49 1.75.875a5.748 5.748 0 0 1-.998.577l.027-.035ZM7.254 1.54A8.75 8.75 0 0 1 8.46 3.552c-.48.11-.97.165-1.461.167-.492-.001-.982-.057-1.461-.167.308-.722.715-1.4 1.207-2.012h.508ZM4.498 3.202a5.748 5.748 0 0 1-.998-.577 6.029 6.029 0 0 1 1.75-.875c-.294.46-.546.947-.753 1.452Zm-1.873.15c.47.358.984.652 1.531.874A9.625 9.625 0 0 0 3.78 6.45H1.155a5.25 5.25 0 0 1 1.47-3.098ZM1.12 7.541h2.625c.038.753.164 1.5.376 2.223a6.649 6.649 0 0 0-1.531.875 5.25 5.25 0 0 1-1.47-3.098Zm3.377 3.255c.207.506.459.992.753 1.453a6.03 6.03 0 0 1-1.75-.875c.312-.226.646-.419.997-.578Zm2.25 1.663a8.594 8.594 0 0 1-1.208-2.013 6.501 6.501 0 0 1 2.922 0 8.54 8.54 0 0 1-1.207 2.013h-.508Zm2.755-1.663c.367.156.716.35 1.042.578a6.338 6.338 0 0 1-1.75.875c.275-.464.512-.95.708-1.453Zm1.873-.148a6.647 6.647 0 0 0-1.531-.875 9.45 9.45 0 0 0 .376-2.223h2.625a5.25 5.25 0 0 1-1.47 3.098Z" fill="#1C1E21"></path></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h14v14H0z"></path></clipPath></defs></svg></div></a><ul class="dropdown__menu"><li><a href="/docs/0.9.0/deployment" target="_self" rel="noopener noreferrer" class="dropdown__link"><div class="labelWrapperDropdown_Mqbj">English</div></a></li><li><a href="/cn/docs/0.9.0/deployment" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active"><div class="labelWrapperDropdown_Mqbj">Chinese</div></a></li></ul></div><a href="https://github.com/apache/hudi" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"><div class="labelWrapperDropdown_Mqbj"></div></a><a href="https://twitter.com/ApacheHudi" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-twitter-link" aria-label="Hudi Twitter Handle"><div class="labelWrapperDropdown_Mqbj"></div></a><a href="https://join.slack.com/t/apache-hudi/shared_invite/zt-2ggm1fub8-_yt4Reu9djwqqVRFC7X49g" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-slack-link" aria-label="Hudi Slack Channel"><div class="labelWrapperDropdown_Mqbj"></div></a><a href="https://www.youtube.com/channel/UCs7AhE0BWaEPZSChrBR-Muw" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-youtube-link" aria-label="Hudi YouTube Channel"><div class="labelWrapperDropdown_Mqbj"></div></a><a href="https://www.linkedin.com/company/apache-hudi/?viewAsMember=true" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-linkedin-link" aria-label="Hudi Linkedin Page"><div class="labelWrapperDropdown_Mqbj"></div></a><div class="searchBox_fBfG"><div role="button" class="searchButton_g9-U" aria-label="Search"><span class="searchText_RI6l">Search</span><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><circle cx="6.864" cy="6.864" r="5.243" stroke="#1C1E21" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></circle><path d="m10.51 10.783 2.056 2.05" stroke="#1C1E21" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></div></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/cn/"><div class="navbar__logo"><img src="/cn/assets/images/hudi.png" alt="Apache Hudi" class="themedImage_TMUO themedImage--light_4Vu1"><img src="/cn/assets/images/hudi.png" alt="Apache Hudi" class="themedImage_TMUO themedImage--dark_uzRr"></div></a><button type="button" class="clean-btn navbar-sidebar__close"><svg viewBox="0 0 15 15" width="21" height="21"><g stroke="var(--ifm-color-emphasis-600)" stroke-width="1.2"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><div class="navbar-sidebar__items"><div class="navbar-sidebar__item menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/cn/docs/overview"><div class="labelWrapperDropdown_Mqbj">Docs</div></a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist"><div class="labelWrapperDropdown_Mqbj">Learn</div></a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist"><div class="labelWrapperDropdown_Mqbj">Contribute</div></a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist"><div class="labelWrapperDropdown_Mqbj">Community</div></a></li><li class="menu__list-item"><a class="menu__link" href="/cn/blog"><div class="labelWrapperDropdown_Mqbj">Blog</div></a></li><li class="menu__list-item"><a class="menu__link" href="/cn/powered-by"><div class="labelWrapperDropdown_Mqbj">Who's Using</div></a></li><li class="menu__list-item"><a class="menu__link" href="/cn/roadmap"><div class="labelWrapperDropdown_Mqbj">Roadmap</div></a></li><li class="menu__list-item"><a class="menu__link" href="/cn/releases/download"><div class="labelWrapperDropdown_Mqbj">Download</div></a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist"><div class="labelWrapperDropdown_Mqbj">Versions</div></a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist"><div class="labelWrapperDropdown_Mqbj"><span><svg viewBox="0 0 20 20" width="20" height="20" aria-hidden="true" class="iconLanguage_zID8"><path fill="currentColor" d="M19.753 10.909c-.624-1.707-2.366-2.726-4.661-2.726-.09 0-.176.002-.262.006l-.016-2.063 3.525-.607c.115-.019.133-.119.109-.231-.023-.111-.167-.883-.188-.976-.027-.131-.102-.127-.207-.109-.104.018-3.25.461-3.25.461l-.013-2.078c-.001-.125-.069-.158-.194-.156l-1.025.016c-.105.002-.164.049-.162.148l.033 2.307s-3.061.527-3.144.543c-.084.014-.17.053-.151.143.019.09.19 1.094.208 1.172.018.08.072.129.188.107l2.924-.504.035 2.018c-1.077.281-1.801.824-2.256 1.303-.768.807-1.207 1.887-1.207 2.963 0 1.586.971 2.529 2.328 2.695 3.162.387 5.119-3.06 5.769-4.715 1.097 1.506.256 4.354-2.094 5.98-.043.029-.098.129-.033.207l.619.756c.08.096.206.059.256.023 2.51-1.73 3.661-4.515 2.869-6.683zm-7.386 3.188c-.966-.121-.944-.914-.944-1.453 0-.773.327-1.58.876-2.156a3.21 3.21 0 011.229-.799l.082 4.277a2.773 2.773 0 01-1.243.131zm2.427-.553l.046-4.109c.084-.004.166-.01.252-.01.773 0 1.494.145 1.885.361.391.217-1.023 2.713-2.183 3.758zm-8.95-7.668a.196.196 0 00-.196-.145h-1.95a.194.194 0 00-.194.144L.008 16.916c-.017.051-.011.076.062.076h1.733c.075 0 .099-.023.114-.072l1.008-3.318h3.496l1.008 3.318c.016.049.039.072.113.072h1.734c.072 0 .078-.025.062-.076-.014-.05-3.083-9.741-3.494-11.04zm-2.618 6.318l1.447-5.25 1.447 5.25H3.226z"></path></svg><span>Languages</span></span></div></a></li><li class="menu__list-item"><a href="https://github.com/apache/hudi" target="_blank" rel="noopener noreferrer" class="menu__link header-github-link" aria-label="GitHub repository"><div class="labelWrapperDropdown_Mqbj"></div></a></li><li class="menu__list-item"><a href="https://twitter.com/ApacheHudi" target="_blank" rel="noopener noreferrer" class="menu__link header-twitter-link" aria-label="Hudi Twitter Handle"><div class="labelWrapperDropdown_Mqbj"></div></a></li><li class="menu__list-item"><a href="https://join.slack.com/t/apache-hudi/shared_invite/zt-2ggm1fub8-_yt4Reu9djwqqVRFC7X49g" target="_blank" rel="noopener noreferrer" class="menu__link header-slack-link" aria-label="Hudi Slack Channel"><div class="labelWrapperDropdown_Mqbj"></div></a></li><li class="menu__list-item"><a href="https://www.youtube.com/channel/UCs7AhE0BWaEPZSChrBR-Muw" target="_blank" rel="noopener noreferrer" class="menu__link header-youtube-link" aria-label="Hudi YouTube Channel"><div class="labelWrapperDropdown_Mqbj"></div></a></li><li class="menu__list-item"><a href="https://www.linkedin.com/company/apache-hudi/?viewAsMember=true" target="_blank" rel="noopener noreferrer" class="menu__link header-linkedin-link" aria-label="Hudi Linkedin Page"><div class="labelWrapperDropdown_Mqbj"></div></a></li></ul></div><div class="navbar-sidebar__item menu"><button type="button" class="clean-btn navbar-sidebar__back">← Back to main menu</button></div></div></div></nav><div class="main-wrapper docs-wrapper docs-doc-page"><div class="docPage_GMj9"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_i9tI" type="button"></button><aside class="docSidebarContainer_k0Pq"><div class="sidebar_a3j0"><nav class="menu thin-scrollbar menu_cyFh menuWithAnnouncementBar_+O1J"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/overview">概念</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist hasHref_TwRn" href="/cn/docs/0.9.0/quick-start-guide">Quick Start</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/cn/docs/0.9.0/quick-start-guide">Spark Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/cn/docs/0.9.0/flink-quick-start-guide">Flink 指南</a></li></ul></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/use_cases">使用案例</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/writing_data">写入 Hudi 数据集</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/schema_evolution">Schema Evolution</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/concurrency_control">Concurrency Control</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/querying_data">查询 Hudi 数据集</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/configurations">配置</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/cn/docs/0.9.0/performance">性能</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/cn/docs/0.9.0/deployment">管理 Hudi Pipelines</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist hasHref_TwRn" href="/cn/docs/0.9.0/cloud">Storage Configurations</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist hasHref_TwRn" href="/cn/docs/0.9.0/docker_demo">Resources</a></div></li></ul></nav></div></aside><main class="docMainContainer_Q970"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_zHA2"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache Hudi<!-- --> <b>0.9.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/cn/docs/deployment">latest version</a></b> (<!-- -->0.14.1<!-- -->).</div></div><div class="docItemContainer_oiyr"><article><span class="theme-doc-version-badge badge badge--secondary">Version: <!-- -->0.9.0</span><div class="tocCollapsible_aw-L theme-doc-toc-mobile tocMobile_Tx6Y"><button type="button" class="clean-btn tocCollapsibleButton_zr6a">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>管理 Hudi Pipelines</h1></header><p>管理员/运维人员可以通过以下方式了解Hudi数据集/管道</p><ul><li><a href="#admin-cli">通过Admin CLI进行管理</a></li><li><a href="#metrics">Graphite指标</a></li><li><a href="#spark-ui">Hudi应用程序的Spark UI</a></li></ul><p>本节简要介绍了每一种方法，并提供了有关<a href="#troubleshooting">故障排除</a>的一些常规指南</p><h2 class="anchor anchorWithStickyNavbar_y2LR" id="admin-cli">Admin CLI<a class="hash-link" href="#admin-cli" title="Direct link to heading"></a></h2><p>一旦构建了hudi，就可以通过<code>cd hudi-cli && ./hudi-cli.sh</code>启动shell。 |
| 一个hudi数据集位于DFS上的<strong>basePath</strong>位置，我们需要该位置才能连接到Hudi数据集。 |
| Hudi库使用.hoodie子文件夹跟踪所有元数据，从而有效地在内部管理该数据集。</p><p>初始化hudi表，可使用如下命令。</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">18/09/06 15:56:52 INFO annotation.AutowiredAnnotationBeanPostProcessor: JSR-330 'javax.inject.Inject' annotation found and supported for autowiring</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">============================================</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* _ _ _ _ *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* | | | | | | (_) *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* | |__| | __| | - *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* | __ || | / _` | || *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* | | | || || (_| | || *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* |_| |_|\___/ \____/ || *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* *</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">============================================</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Welcome to Hoodie CLI. Please type help if you are looking for help.</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">hudi->create --path /user/hive/warehouse/table1 --tableName hoodie_table_1 --tableType COPY_ON_WRITE</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">.....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">18/09/06 15:57:15 INFO table.HoodieTableMetaClient: Finished Loading Table of type COPY_ON_WRITE from ...</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>使用desc命令可以查看hudi表的描述信息:</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:hoodie_table_1->desc</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">18/09/06 15:57:19 INFO timeline.HoodieActiveTimeline: Loaded instants []</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> _________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Property | Value |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |========================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | basePath | ... |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | metaPath | ... |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | fileSystem | hdfs |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | hoodie.table.name | hoodie_table_1 |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | hoodie.table.type | COPY_ON_WRITE |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | hoodie.archivelog.folder| |</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>以下是连接到包含uber trips的Hudi数据集的示例命令。</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->connect --path /app/uber/trips</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">16/10/05 23:20:37 INFO model.HoodieTableMetadata: Attempting to load the commits under /app/uber/trips/.hoodie with suffix .commit</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">16/10/05 23:20:37 INFO model.HoodieTableMetadata: Attempting to load the commits under /app/uber/trips/.hoodie with suffix .inflight</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">16/10/05 23:20:37 INFO model.HoodieTableMetadata: All commits :HoodieCommits{commitList=[20161002045850, 20161002052915, 20161002055918, 20161002065317, 20161002075932, 20161002082904, 20161002085949, 20161002092936, 20161002105903, 20161002112938, 20161002123005, 20161002133002, 20161002155940, 20161002165924, 20161002172907, 20161002175905, 20161002190016, 20161002192954, 20161002195925, 20161002205935, 20161002215928, 20161002222938, 20161002225915, 20161002232906, 20161003003028, 20161003005958, 20161003012936, 20161003022924, 20161003025859, 20161003032854, 20161003042930, 20161003052911, 20161003055907, 20161003062946, 20161003065927, 20161003075924, 20161003082926, 20161003085925, 20161003092909, 20161003100010, 20161003102913, 20161003105850, 20161003112910, 20161003115851, 20161003122929, 20161003132931, 20161003142952, 20161003145856, 20161003152953, 20161003155912, 20161003162922, 20161003165852, 20161003172923, 20161003175923, 20161003195931, 20161003210118, 20161003212919, 20161003215928, 20161003223000, 20161003225858, 20161004003042, 20161004011345, 20161004015235, 20161004022234, 20161004063001, 20161004072402, 20161004074436, 20161004080224, 20161004082928, 20161004085857, 20161004105922, 20161004122927, 20161004142929, 20161004163026, 20161004175925, 20161004194411, 20161004203202, 20161004211210, 20161004214115, 20161004220437, 20161004223020, 20161004225321, 20161004231431, 20161004233643, 20161005010227, 20161005015927, 20161005022911, 20161005032958, 20161005035939, 20161005052904, 20161005070028, 20161005074429, 20161005081318, 20161005083455, 20161005085921, 20161005092901, 20161005095936, 20161005120158, 20161005123418, 20161005125911, 20161005133107, 20161005155908, 20161005163517, 20161005165855, 20161005180127, 20161005184226, 20161005191051, 20161005193234, 20161005203112, 20161005205920, 20161005212949, 20161005223034, 20161005225920]}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Metadata for table trips loaded</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips-></span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>连接到数据集后，便可使用许多其他命令。该shell程序具有上下文自动完成帮助(按TAB键)，下面是所有命令的列表，本节中对其中的一些命令进行了详细示例。</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->help</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* ! - Allows execution of operating system (OS) commands</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* // - Inline comment markers (start of line only)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* ; - Inline comment markers (start of line only)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* addpartitionmeta - Add partition metadata to a dataset, if not present</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* clear - Clears the console</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* cls - Clears the console</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commit rollback - Rollback a commit</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commits compare - Compare commits with another Hoodie dataset</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commit showfiles - Show file level details of a commit</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commit showpartitions - Show partition level details of a commit</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commits refresh - Refresh the commits</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commits show - Show the commits</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* commits sync - Compare commits with another Hoodie dataset</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* connect - Connect to a hoodie dataset</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* date - Displays the local date and time</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* exit - Exits the shell</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* help - List all commands usage</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* quit - Exits the shell</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* records deduplicate - De-duplicate a partition path contains duplicates & produce repaired files to replace with</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* script - Parses the specified resource file and executes its commands</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* stats filesizes - File Sizes. Display summary stats on sizes of files</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* stats wa - Write Amplification. Ratio of how many records were upserted to how many records were actually written</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* sync validate - Validate the sync by counting the number of records</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* system properties - Shows the shell's properties</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* utils loadClass - Load a class</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* version - Displays shell version</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips-></span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="检查提交">检查提交<a class="hash-link" href="#检查提交" title="Direct link to heading"></a></h3><p>在Hudi中，更新或插入一批记录的任务被称为<strong>提交</strong>。提交可提供基本的原子性保证，即只有提交的数据可用于查询。 |
| 每个提交都有一个单调递增的字符串/数字，称为<strong>提交编号</strong>。通常，这是我们开始提交的时间。</p><p>查看有关最近10次提交的一些基本信息，</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->commits show --sortBy "Total Bytes Written" --desc true --limit 10</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ________________________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | CommitTime | Total Bytes Written| Total Files Added| Total Files Updated| Total Partitions Written| Total Records Written| Total Update Records Written| Total Errors|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |=======================================================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips-></span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>在每次写入开始时，Hudi还将.inflight提交写入.hoodie文件夹。您可以使用那里的时间戳来估计正在进行的提交已经花费的时间</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">$ hdfs dfs -ls /app/uber/trips/.hoodie/*.inflight</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">-rw-r--r-- 3 vinoth supergroup 321984 2016-10-05 23:18 /app/uber/trips/.hoodie/20161005225920.inflight</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="深入到特定的提交">深入到特定的提交<a class="hash-link" href="#深入到特定的提交" title="Direct link to heading"></a></h3><p>了解写入如何分散到特定分区，</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->commit showpartitions --commit 20161005165855 --sortBy "Total Bytes Written" --desc true --limit 10</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> __________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Partition Path| Total Files Added| Total Files Updated| Total Records Inserted| Total Records Updated| Total Bytes Written| Total Errors|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |=========================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>如果您需要文件级粒度，我们可以执行以下操作</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->commit showfiles --commit 20161005165855 --sortBy "Partition Path"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Partition Path| File ID | Previous Commit| Total Records Updated| Total Records Written| Total Bytes Written| Total Errors|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |=======================================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="文件系统视图">文件系统视图<a class="hash-link" href="#文件系统视图" title="Direct link to heading"></a></h3><p>Hudi将每个分区视为文件组的集合，每个文件组包含按提交顺序排列的文件切片列表(请参阅概念)。以下命令允许用户查看数据集的文件切片。</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain"> hoodie:stock_ticks_mor->show fsview all</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta File Size| Delta Files |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |==============================================================================================================================================================================================================================================================================================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> hoodie:stock_ticks_mor->show fsview latest --partitionPath "2018/08/31"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ......</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> __________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta Size| Delta Size - compaction scheduled| Delta Size - compaction unscheduled| Delta To Base Ratio - compaction scheduled| Delta To Base Ratio - compaction unscheduled| Delta Files - compaction scheduled | Delta Files - compaction unscheduled|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | 20.8 KB | 0.0 B | 0.0 B | 0.0 B | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]| [] |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> hoodie:stock_ticks_mor-></span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="统计信息">统计信息<a class="hash-link" href="#统计信息" title="Direct link to heading"></a></h3><p>由于Hudi直接管理DFS数据集的文件大小，这些信息会帮助你全面了解Hudi的运行状况</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->stats filesizes --partitionPath 2016/09/01 --sortBy "95th" --desc true --limit 10</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | CommitTime | Min | 10th | 50th | avg | 95th | Max | NumFiles| StdDev |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |===============================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | <COMMIT_ID> | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 2 | 2.3 KB |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>如果Hudi写入花费的时间更长，那么可以通过观察写放大指标来发现任何异常</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->stats wa</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> __________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | CommitTime | Total Upserted| Total Written| Write Amplifiation Factor|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |=========================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ....</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="归档的提交">归档的提交<a class="hash-link" href="#归档的提交" title="Direct link to heading"></a></h3><p>为了限制DFS上.commit文件的增长量，Hudi将较旧的.commit文件(适当考虑清理策略)归档到commits.archived文件中。 |
| 这是一个序列文件，其包含commitNumber => json的映射，及有关提交的原始信息(上面已很好地汇总了相同的信息)。</p><h3 class="anchor anchorWithStickyNavbar_y2LR" id="压缩">压缩<a class="hash-link" href="#压缩" title="Direct link to heading"></a></h3><p>要了解压缩和写程序之间的时滞，请使用以下命令列出所有待处理的压缩。</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->compactions show all</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ___________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Compaction Instant Time| State | Total FileIds to be Compacted|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |==================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | <INSTANT_1> | REQUESTED| 35 |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | <INSTANT_2> | INFLIGHT | 27 |</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>要检查特定的压缩计划，请使用</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->compaction show --instant <INSTANT_1></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> _________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | Partition Path| File Id | Base Instant | Data File Path | Total Delta Files| getMetrics |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |================================================================================================================================================================================================================================================</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | 2018/07/17 | <UUID> | <INSTANT_1> | viewfs://ns-default/.../../UUID_<INSTANT>.parquet | 1 | {TOTAL_LOG_FILES=1.0, TOTAL_IO_READ_MB=1230.0, TOTAL_LOG_FILES_SIZE=2.51255751E8, TOTAL_IO_WRITE_MB=991.0, TOTAL_IO_MB=2221.0}|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>要手动调度或运行压缩，请使用以下命令。该命令使用spark启动器执行压缩操作。 |
| 注意：确保没有其他应用程序正在同时调度此数据集的压缩</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->help compaction schedule</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Keyword: compaction schedule</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Description: Schedule Compaction</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: sparkMemory</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Spark executor memory</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: false</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '1G'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* compaction schedule - Schedule Compaction</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->help compaction run</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Keyword: compaction run</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Description: Run Compaction for given instant time</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: tableName</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Table name</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: true</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: parallelism</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Parallelism for hoodie compaction</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: true</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: schemaFilePath</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Path for Avro schema file</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: true</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: sparkMemory</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Spark executor memory</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: true</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: retry</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Number of retries</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: true</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Keyword: compactionInstant</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Help: Base path for the target hoodie dataset</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Mandatory: true</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if specified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> Default if unspecified: '__NULL__'</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">* compaction run - Run Compaction for given instant time</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="验证压缩">验证压缩<a class="hash-link" href="#验证压缩" title="Direct link to heading"></a></h3><p>验证压缩计划：检查压缩所需的所有文件是否都存在且有效</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:stock_ticks_mor->compaction validate --instant 20181005222611</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> COMPACTION PLAN VALID</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> ___________________________________________________________________________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |==========================================================================================================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | true | |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:stock_ticks_mor->compaction validate --instant 20181005222601</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> COMPACTION PLAN INVALID</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error |</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> |=====================================================================================================================================================================================================================================================================================================|</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | false| All log files specified in compaction operation is not present. Missing .... |</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="注意">注意<a class="hash-link" href="#注意" title="Direct link to heading"></a></h3><p>必须在其他写入/摄取程序没有运行的情况下执行以下命令。</p><p>有时，有必要从压缩计划中删除fileId以便加快或取消压缩操作。 |
| 压缩计划之后在此文件上发生的所有新日志文件都将被安全地重命名以便进行保留。Hudi提供以下CLI来支持</p><h3 class="anchor anchorWithStickyNavbar_y2LR" id="取消调度压缩">取消调度压缩<a class="hash-link" href="#取消调度压缩" title="Direct link to heading"></a></h3><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->compaction unscheduleFileId --fileId <FileUUID></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">No File renames needed to unschedule file from pending compaction. Operation successful.</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><p>在其他情况下，需要撤销整个压缩计划。以下CLI支持此功能</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:trips->compaction unschedule --instant <compactionInstant></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">.....</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">No File renames needed to unschedule pending compaction. Operation successful.</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h3 class="anchor anchorWithStickyNavbar_y2LR" id="修复压缩">修复压缩<a class="hash-link" href="#修复压缩" title="Direct link to heading"></a></h3><p>上面的压缩取消调度操作有时可能会部分失败(例如：DFS暂时不可用)。 |
| 如果发生部分故障，则压缩操作可能与文件切片的状态不一致。 |
| 当您运行<code>压缩验证</code>时，您会注意到无效的压缩操作(如果有的话)。 |
| 在这种情况下，修复命令将立即执行，它将重新排列文件切片，以使文件不丢失，并且文件切片与压缩计划一致</p><div class="codeBlockContainer_J+bg language-java theme-code-block"><div class="codeBlockContent_csEI java"><pre tabindex="0" class="prism-code language-java codeBlock_rtdJ thin-scrollbar" style="color:#F8F8F2;background-color:#282A36"><code class="codeBlockLines_1zSZ"><span class="token-line" style="color:#F8F8F2"><span class="token plain">hoodie:stock_ticks_mor->compaction repair --instant 20181005222611</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">......</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Compaction successfully repaired</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">.....</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_M3SB clean-btn">Copy</button></div></div><h2 class="anchor anchorWithStickyNavbar_y2LR" id="metrics"><a href="/cn/docs/configurations#%E6%8C%87%E6%A0%87%E9%85%8D%E7%BD%AE">指标</a><a class="hash-link" href="#metrics" title="Direct link to heading"></a></h2><p>为Hudi Client配置正确的数据集名称和指标环境后，它将生成以下graphite指标，以帮助调试hudi数据集</p><ul><li><strong>提交持续时间</strong> - 这是成功提交一批记录所花费的时间</li><li><strong>回滚持续时间</strong> - 同样，撤消失败的提交所剩余的部分数据所花费的时间(每次写入失败后都会自动发生)</li><li><strong>文件级别指标</strong> - 显示每次提交中新增、版本、删除(清除)的文件数量</li><li><strong>记录级别指标</strong> - 每次提交插入/更新的记录总数</li><li><strong>分区级别指标</strong> - 更新的分区数量(对于了解提交持续时间的突然峰值非常有用)</li></ul><p>然后可以将这些指标绘制在grafana等标准工具上。以下是提交持续时间图表示例。</p><figure><img class="docimage" src="/cn/assets/images/hudi_commit_duration-64b7b65fc946ab2d6b69ffdf6f5bb9b0.png" alt="hudi_commit_duration.png"></figure><h2 class="anchor anchorWithStickyNavbar_y2LR" id="troubleshooting">故障排除<a class="hash-link" href="#troubleshooting" title="Direct link to heading"></a></h2><p>以下部分通常有助于调试Hudi故障。以下元数据已被添加到每条记录中，可以通过标准Hadoop SQL引擎(Hive/Presto/Spark)检索，来更容易地诊断问题的严重性。</p><ul><li><strong>_hoodie_record_key</strong> - 作为每个DFS分区内的主键，是所有更新/插入的基础</li><li><strong>_hoodie_commit_time</strong> - 该记录上次的提交</li><li><strong>_hoodie_file_name</strong> - 包含记录的实际文件名(对检查重复非常有用)</li><li><strong>_hoodie_partition_path</strong> - basePath的路径，该路径标识包含此记录的分区</li></ul><p>请注意，到目前为止，Hudi假定应用程序为给定的recordKey传递相同的确定性分区路径。即仅在每个分区内保证recordKey(主键)的唯一性。</p><h3 class="anchor anchorWithStickyNavbar_y2LR" id="缺失记录">缺失记录<a class="hash-link" href="#缺失记录" title="Direct link to heading"></a></h3><p>请在可能写入记录的窗口中，使用上面的admin命令检查是否存在任何写入错误。 |
| 如果确实发现错误，那么记录实际上不是由Hudi写入的，而是交还给应用程序来决定如何处理。</p><h3 class="anchor anchorWithStickyNavbar_y2LR" id="重复">重复<a class="hash-link" href="#重复" title="Direct link to heading"></a></h3><p>首先，请确保访问Hudi数据集的查询是<a href="/cn/docs/0.9.0/querying_data">没有问题的</a>，并之后确认的确有重复。</p><ul><li>如果确认，请使用上面的元数据字段来标识包含记录的物理文件和分区文件。</li><li>如果重复的记录存在于不同分区路径下的文件，则意味着您的应用程序正在为同一recordKey生成不同的分区路径，请修复您的应用程序.</li><li>如果重复的记录存在于同一分区路径下的多个文件，请使用邮件列表汇报这个问题。这不应该发生。您可以使用<code>records deduplicate</code>命令修复数据。</li></ul><h3 class="anchor anchorWithStickyNavbar_y2LR" id="spark-ui">Spark故障<a class="hash-link" href="#spark-ui" title="Direct link to heading"></a></h3><p>典型的upsert() DAG如下所示。请注意，Hudi客户端会缓存中间的RDD，以智能地并调整文件大小和Spark并行度。 |
| 另外，由于还显示了探针作业，Spark UI显示了两次sortByKey，但它只是一个排序。</p><figure><img class="docimage" src="/cn/assets/images/hudi_upsert_dag-3b2d81de8560fad7af112e40a2f8c437.png" alt="hudi_upsert_dag.png"></figure><p>概括地说，有两个步骤</p><p><strong>索引查找以标识要更改的文件</strong></p><ul><li>Job 1 : 触发输入数据读取，转换为HoodieRecord对象，然后根据输入记录拿到目标分区路径。</li><li>Job 2 : 加载我们需要检查的文件名集。</li><li>Job 3 & 4 : 通过联合上面1和2中的RDD，智能调整spark join并行度，然后进行实际查找。</li><li>Job 5 : 生成带有位置的recordKeys作为标记的RDD。</li></ul><p><strong>执行数据的实际写入</strong></p><ul><li>Job 6 : 将记录与recordKey(位置)进行懒惰连接，以提供最终的HoodieRecord集，现在它包含每条记录的文件/分区路径信息(如果插入，则为null)。然后还要再次分析工作负载以确定文件的大小。</li><li>Job 7 : 实际写入数据(更新 + 插入 + 插入转为更新以保持文件大小)</li></ul><p>根据异常源(Hudi/Spark)，上述关于DAG的信息可用于查明实际问题。最常遇到的故障是由YARN/DFS临时故障引起的。 |
| 将来，将在项目中添加更复杂的调试/管理UI，以帮助自动进行某些调试。</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/hudi/tree/asf-site/website/versioned_docs/version-0.9.0/deployment.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_mS5F" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_mt2f"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/cn/docs/0.9.0/performance"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">性能</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/cn/docs/0.9.0/cloud"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">云储存</div></a></div></nav></div></div><div class="col col--3"><div class="tableOfContents_vrFS thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#admin-cli" class="table-of-contents__link toc-highlight">Admin CLI</a><ul><li><a href="#检查提交" class="table-of-contents__link toc-highlight">检查提交</a></li><li><a href="#深入到特定的提交" class="table-of-contents__link toc-highlight">深入到特定的提交</a></li><li><a href="#文件系统视图" class="table-of-contents__link toc-highlight">文件系统视图</a></li><li><a href="#统计信息" class="table-of-contents__link toc-highlight">统计信息</a></li><li><a href="#归档的提交" class="table-of-contents__link toc-highlight">归档的提交</a></li><li><a href="#压缩" class="table-of-contents__link toc-highlight">压缩</a></li><li><a href="#验证压缩" class="table-of-contents__link toc-highlight">验证压缩</a></li><li><a href="#注意" class="table-of-contents__link toc-highlight">注意</a></li><li><a href="#取消调度压缩" class="table-of-contents__link toc-highlight">取消调度压缩</a></li><li><a href="#修复压缩" class="table-of-contents__link toc-highlight">修复压缩</a></li></ul></li><li><a href="#metrics" class="table-of-contents__link toc-highlight">指标</a></li><li><a href="#troubleshooting" class="table-of-contents__link toc-highlight">故障排除</a><ul><li><a href="#缺失记录" class="table-of-contents__link toc-highlight">缺失记录</a></li><li><a href="#重复" class="table-of-contents__link toc-highlight">重复</a></li><li><a href="#spark-ui" class="table-of-contents__link toc-highlight">Spark故障</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">About</div><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/cn/blog/2021/07/21/streaming-data-lake-platform">Our Vision</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/concepts">Concepts</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/community/team">Team</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/releases/release-0.14.1">Releases</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/releases/download">Download</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/powered-by">Who's Using</a></li></ul></div><div class="col footer__col"><div class="footer__title">Learn</div><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/cn/docs/quick-start-guide">Quick Start</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/docker_demo">Docker Demo</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/blog">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/talks">Talks</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/videos">Video Guides</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/faq">FAQ</a></li><li class="footer__item"><a href="https://cwiki.apache.org/confluence/display/HUDI" target="_blank" rel="noopener noreferrer" class="footer__link-item"><span>Technical Wiki<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li></ul></div><div class="col footer__col"><div class="footer__title">Hudi On Cloud</div><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/cn/docs/s3_hoodie">AWS</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/gcs_hoodie">Google Cloud</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/oss_hoodie">Alibaba Cloud</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/azure_hoodie">Microsoft Azure</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/cos_hoodie">Tencent Cloud</a></li><li class="footer__item"><a class="footer__link-item" href="/cn/docs/ibm_cos_hoodie">IBM Cloud</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/cn/community/get-involved">Get Involved</a></li><li class="footer__item"><a href="https://join.slack.com/t/apache-hudi/shared_invite/zt-2ggm1fub8-_yt4Reu9djwqqVRFC7X49g" target="_blank" rel="noopener noreferrer" class="footer__link-item"><span>Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li class="footer__item"><a href="https://github.com/apache/hudi" target="_blank" rel="noopener noreferrer" class="footer__link-item"><span>GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li class="footer__item"><a href="https://twitter.com/ApacheHudi" target="_blank" rel="noopener noreferrer" class="footer__link-item"><span>Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li class="footer__item"><a href="https://www.youtube.com/channel/UCs7AhE0BWaEPZSChrBR-Muw" target="_blank" rel="noopener noreferrer" class="footer__link-item"><span>YouTube<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li class="footer__item"><a href="https://www.linkedin.com/company/apache-hudi/?viewAsMember=true" target="_blank" rel="noopener noreferrer" class="footer__link-item"><span>Linkedin<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_wgqa"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></span></a></li><li class="footer__item"><a href="mailto:dev-subscribe@hudi.apache.org?Subject=SubscribeToHudi" target="_blank" rel="noopener noreferrer" class="footer__link-item">Mailing List</a></li></ul></div><div class="col footer__col"><div class="footer__title">Apache</div><ul class="footer__items"><li class="footer__item"><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks</a></li><li class="footer__item"><a href="https://www.apache.org/licenses" target="_blank" rel="noopener noreferrer" class="footer__link-item">License</a></li><li class="footer__item"><a href="https://www.apache.org/security" target="_blank" rel="noopener noreferrer" class="footer__link-item">Security</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship</a></li><li class="footer__item"><a href="https://www.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation</a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a href="https://hudi.apache.org/" target="_blank" rel="noopener noreferrer" class="footerLogoLink_SRtH"><img src="/cn/assets/images/logo-big.png" alt="Apache Hudi™" class="themedImage_TMUO themedImage--light_4Vu1 footer__logo"><img src="/cn/assets/images/logo-big.png" alt="Apache Hudi™" class="themedImage_TMUO themedImage--dark_uzRr footer__logo"></a></div><div class="footer__copyright">Copyright © 2021 <a href="https://apache.org">The Apache Software Foundation</a>, Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0"> Apache License, Version 2.0</a>. |
| Hudi, Apache and the Apache feather logo are trademarks of The Apache Software Foundation. <a href="/docs/privacy">Privacy Policy</a></div></div></div></footer></div> |
| <script src="/cn/assets/js/runtime~main.e00cff1c.js"></script> |
| <script src="/cn/assets/js/main.a490f48b.js"></script> |
| </body> |
| </html> |
