blob: 7d6fea5985f0c0826e44324029f5533cbed927ad [file] [log] [blame]
<!doctype html>
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-0.6.0 plugin-docs plugin-id-default docs-doc-id-ecosystem/kaldi/RunningDistributedThchs30KaldiJobs">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="generator" content="Docusaurus v2.0.0-beta.18">
<title data-rh="true">RunningDistributedThchs30KaldiJobs | Apache Submarine</title><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://submarine.apache.org//docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="0.6.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-0.6.0"><meta data-rh="true" name="docsearch:version" content="0.6.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-0.6.0"><meta data-rh="true" property="og:title" content="RunningDistributedThchs30KaldiJobs | Apache Submarine"><meta data-rh="true" name="description" content="&lt;!--"><meta data-rh="true" property="og:description" content="&lt;!--"><link data-rh="true" rel="icon" href="/img/submarine.ico"><link data-rh="true" rel="canonical" href="https://submarine.apache.org//docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs"><link data-rh="true" rel="alternate" href="https://submarine.apache.org//docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs" hreflang="en"><link data-rh="true" rel="alternate" href="https://submarine.apache.org//zh-cn/docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs" hreflang="zh-cn"><link data-rh="true" rel="alternate" href="https://submarine.apache.org//docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs" hreflang="x-default"><link rel="stylesheet" href="/assets/css/styles.80258812.css">
<link rel="preload" href="/assets/js/runtime~main.9d177e25.js" as="script">
<link rel="preload" href="/assets/js/main.7cd2eed3.js" as="script">
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
<div role="region"><a href="#" class="skipToContent_ZgBM">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle clean-btn" type="button" tabindex="0"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_W2Cr themedImage--light_TfLj"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_W2Cr themedImage--dark_oUvU"></div><b class="navbar__title">Apache Submarine</b></a><a class="navbar__item navbar__link" href="/docs/0.6.0/gettingStarted/quickstart">Docs</a><a class="navbar__item navbar__link" href="/docs/0.6.0/api/environment">API</a><a class="navbar__item navbar__link" href="/docs/0.6.0/download">Download</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><span><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_dNtB"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg><span>English</span></span></a><ul class="dropdown__menu"><li><a href="/docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active">English</a></li><li><a href="/zh-cn/docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs" target="_self" rel="noopener noreferrer" class="dropdown__link">δΈ­ζ–‡</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/0.6.0/gettingStarted/quickstart">0.6.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/gettingStarted/quickstart">master πŸƒ</a></li><li><a class="dropdown__link" href="/docs/gettingStarted/quickstart">0.8.0</a></li><li><a class="dropdown__link" href="/docs/0.7.0/gettingStarted/quickstart">0.7.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs">0.6.0</a></li><li><a class="dropdown__link" href="/versions">All versions</a></li></ul></div><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="toggle_S7eR colorModeToggle_vKtC"><button class="clean-btn toggleButton_rCf9 toggleButtonDisabled_Pu9x" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_v35p"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_nQuB"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Search" aria-label="Search" class="navbar__search-input search-bar"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div class="main-wrapper"><div class="docPage_P2Lg"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_RiI4" type="button"></button><main class="docMainContainer_TCnq docMainContainerEnhanced_WDCb"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_DM6M"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache Submarine<!-- --> <b>0.6.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/gettingStarted/quickstart">latest version</a></b> (<!-- -->0.8.0<!-- -->).</div></div><div class="docItemContainer_vinB"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.6.0</span><div class="tocCollapsible_jdIR theme-doc-toc-mobile tocMobile_TmEX"><button type="button" class="clean-btn tocCollapsibleButton_Fzxq">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>RunningDistributedThchs30KaldiJobs</h1></header><h1>Thchs30 Kaldi Example With YARN Service</h1><h2 class="anchor anchorWithStickyNavbar_mojV" id="prepare-data-for-training">Prepare data for training<a class="hash-link" href="#prepare-data-for-training" title="Direct link to heading">​</a></h2><p>Thchs30 is a common benchmark in machine learning for speech data and transcripts. Below example is based on Thchs30 dataset.</p><p>1) download gz file:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">THCHS30_PATH=/data/hdfs1/nfs/aisearch/kaldi/thchs30</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">mkdir $THCHS30_PATH/data &amp;&amp; cd $THCHS30_PATH/data</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">wget http://www.openslr.org/resources/18/data_thchs30.tgz</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">wget http://www.openslr.org/resources/18/test-noise.tgz</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">wget http://www.openslr.org/resources/18/resource.tgz</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>2) Checkout <a href="https://github.com/apache/submarine.git" target="_blank" rel="noopener noreferrer">https://github.com/apache/submarine.git</a>:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/apache/submarine.git</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>3) Go to <code>submarine/docker/ecosystem/</code></p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cp -r ./kaldi/sge $THCHS30_PATH/sge</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>4) optional,Modify <code>/opt/kaldi/egs/thchs30/s5/cmd.sh</code> in the Container,This queue is used by default</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">export train_cmd=&quot;queue.pl -q all.q&quot;</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p><strong>Warning:</strong></p><p>Please note that YARN service doesn&#x27;t allow multiple services with the same name, so please run following command</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">yarn application -destroy &lt;service-name&gt;</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>to delete services if you want to reuse the same service name.</p><h2 class="anchor anchorWithStickyNavbar_mojV" id="prepare-docker-images">Prepare Docker images<a class="hash-link" href="#prepare-docker-images" title="Direct link to heading">​</a></h2><p>Refer to <a href="/docs/0.6.0/ecosystem/kaldi/WriteDockerfileKaldi">Write Dockerfile</a> to build a Docker image or use prebuilt one:</p><ul><li>hadoopsubmarine/kaldi-latest-gpu-base:0.0.1</li></ul><h2 class="anchor anchorWithStickyNavbar_mojV" id="run-kaldi-jobs">Run Kaldi jobs<a class="hash-link" href="#run-kaldi-jobs" title="Direct link to heading">​</a></h2><h3 class="anchor anchorWithStickyNavbar_mojV" id="run-distributed-training">Run distributed training<a class="hash-link" href="#run-distributed-training" title="Direct link to heading">​</a></h3><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Change the variables according to your needs</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">SUBMARINE_VERSION=3.3.0-SNAPSHOT</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">WORKER_NUM=2</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">SGE_CFG_PATH=/cfg</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">THCHS30_PATH=/data/hdfs1/nfs/aisearch/kaldi/thchs30</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">DOCKER_HADOOP_HDFS_HOME=/app/${SUBMARINE_VERSION}</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"># Dependent on registrydns, you must fill in &lt; your RegistryDNSIP&gt; in resolv.conf</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">yarn jar /usr/local/matrix/share/hadoop/yarn/${SUBMARINE_VERSION}.jar \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">job run --name kaldi-thchs30-distributed \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--env DOCKER_HADOOP_HDFS_HOME=$DOCKER_HADOOP_HDFS_HOME \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--env YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK=calico-network \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--env PYTHONUNBUFFERED=&quot;0&quot; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--env TZ=&quot;Asia/Shanghai&quot; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--env YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS=${THCHS30_PATH}/sge/resolv.conf:/etc/resolv.conf,\</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">${THCHS30_PATH}/sge/passwd:/etc/passwd:rw,\</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">${THCHS30_PATH}/sge/group:/etc/group:rw,\</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">${THCHS30_PATH}/sge:$SGE_CFG_PATH,\</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">${THCHS30_PATH}/data:/opt/kaldi/egs/thchs30,\</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">${THCHS30_PATH}/mul/s5:/opt/kaldi/egs/mul-thchs30/s5 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--input_path /opt/kaldi/egs/thchs30/data \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--docker_image hadoopsubmarine/kaldi-latest-gpu-base:0.0.1 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--num_workers $WORKER_NUM \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--worker_resources memory=64G,vcores=32,gpu=1 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--worker_launch_cmd &quot;sudo mkdir -p /opt/kaldi/egs/mul-thchs30/s5 &amp;&amp; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">sudo cp /opt/kaldi/egs/thchs30/s5/* /opt/kaldi/egs/mul-thchs30/s5 -r &amp;&amp; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">cluster_user=`whoami` domain_suffix=&quot;ml.com&quot; &amp;&amp; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">cd /cfg &amp;&amp; bash sge_run.sh $WORKER_NUM $SGE_CFG_PATH &amp;&amp; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">if [ $(echo $HOST_NAME |grep &quot;^master-&quot;) ] then sleep 2m &amp;&amp; cd /opt/kaldi/egs/mul-thchs30/s5 &amp;&amp; ./run.sh fi&quot; \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">--verbose</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>Explanations:</p><ul><li><code>&gt;1</code> num_workers indicates it is a distributed training.</li><li>Parameters / resources / Docker image of parameter server can be specified separately. For many cases, parameter server doesn&#x27;t require GPU.We don&#x27;t need parameter server here</li></ul><p>For the meaning of the individual parameters, see the <a href="/docs/0.6.0/adminDocs/yarn/">QuickStart</a> page!</p><p><em>Outputs of distributed training</em></p><p>Sample output of master:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading package lists...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Building dependency tree...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading state information...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">The following additional packages will be installed:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> bsd-mailx cpio gridengine-common ifupdown iproute2 isc-dhcp-client</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> isc-dhcp-common libatm1 libdns-export162 libisc-export160 liblockfile-bin</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> liblockfile1 libmnl0 libxmuu1 libxtables11 ncurses-term netbase</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> openssh-client openssh-server openssh-sftp-server postfix python3-chardet</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> python3-pkg-resources python3-requests python3-six python3-urllib3</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ssh-import-id ssl-cert tcsh xauth</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Suggested packages:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> libarchive1 gridengine-qmon ppp rdnssd iproute2-doc resolvconf avahi-autoipd</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> isc-dhcp-client-ddns apparmor ssh-askpass libpam-ssh keychain monkeysphere</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> rssh molly-guard ufw procmail postfix-mysql postfix-pgsql postfix-ldap</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> postfix-pcre sasl2-bin libsasl2-modules dovecot-common postfix-cdb</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> postfix-doc python3-setuptools python3-ndg-httpsclient python3-openssl</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> python3-pyasn1 openssl-blacklist</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">The following NEW packages will be installed:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> bsd-mailx cpio gridengine-client gridengine-common gridengine-exec</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> gridengine-master ifupdown iproute2 isc-dhcp-client isc-dhcp-common libatm1</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> libdns-export162 libisc-export160 liblockfile-bin liblockfile1 libmnl0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> libxmuu1 libxtables11 ncurses-term netbase openssh-client openssh-server</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> openssh-sftp-server postfix python3-chardet python3-pkg-resources</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> python3-requests python3-six python3-urllib3 ssh-import-id ssl-cert tcsh</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> xauth</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">0 upgraded, 33 newly installed, 0 to remove and 30 not upgraded.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Need to get 12.1 MB of archives.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">After this operation, 65.8 MB of additional disk space will be used.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:1 http://archive.ubuntu.com/ubuntu xenial/main amd64 libatm1 amd64 1:2.5.1-1.5 [24.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:2 http://archive.ubuntu.com/ubuntu xenial/main amd64 libmnl0 amd64 1.0.3-5 [12.0 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:3 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile-bin amd64 1.09-6ubuntu1 [10.8 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:4 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile1 amd64 1.09-6ubuntu1 [8056 B]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:5 http://archive.ubuntu.com/ubuntu xenial/main amd64 cpio amd64 2.11+dfsg-5ubuntu1 [74.8 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:6 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 iproute2 amd64 4.3.0-1ubuntu3.16.04.5 [523 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:7 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 ifupdown amd64 0.8.10ubuntu1.4 [54.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:8 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libisc-export160 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [153 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:9 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libdns-export162 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [665 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:10 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-client amd64 4.3.3-5ubuntu12.10 [224 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:11 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-common amd64 4.3.3-5ubuntu12.10 [105 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:12 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxtables11 amd64 1.6.0-2ubuntu3 [27.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:13 http://archive.ubuntu.com/ubuntu xenial/main amd64 netbase all 5.3 [12.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:14 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxmuu1 amd64 2:1.1.2-2 [9674 B]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:15 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-client amd64 1:7.2p2-4ubuntu2.8 [590 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:16 http://archive.ubuntu.com/ubuntu xenial/main amd64 xauth amd64 1:1.0.9-1ubuntu2 [22.7 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:17 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssl-cert all 1.0.37 [16.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:18 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 postfix amd64 3.1.0-3ubuntu0.3 [1152 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:19 http://archive.ubuntu.com/ubuntu xenial/main amd64 bsd-mailx amd64 8.1.2-0.20160123cvs-2 [63.7 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:20 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-common all 6.2u5-7.4 [156 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:21 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-client amd64 6.2u5-7.4 [3394 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:22 http://archive.ubuntu.com/ubuntu xenial/universe amd64 tcsh amd64 6.18.01-5 [410 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:23 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-exec amd64 6.2u5-7.4 [990 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:24 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-master amd64 6.2u5-7.4 [2429 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:25 http://archive.ubuntu.com/ubuntu xenial/main amd64 ncurses-term all 6.0+20160213-1ubuntu1 [249 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:26 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-sftp-server amd64 1:7.2p2-4ubuntu2.8 [38.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:27 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-server amd64 1:7.2p2-4ubuntu2.8 [335 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:28 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-pkg-resources all 20.7.0-1 [79.0 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:29 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-chardet all 2.3.0-2 [96.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:30 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-six all 1.10.0-3 [11.0 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:31 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-urllib3 all 1.13.1-2ubuntu0.16.04.3 [58.5 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:32 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-requests all 2.9.1-3ubuntu0.1 [55.8 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:33 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssh-import-id all 5.5-0ubuntu1 [10.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Fetched 12.1 MB in 0s (15.0 MB/s)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libatm1:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 5%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 10%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 15%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 20%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 25%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 30%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 35%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 40%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 45%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 50%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 55%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 60%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 65%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 70%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 75%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 80%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 85%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 90%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 95%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 100%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 21398 files and directories currently installed.)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libatm1_1%3a2.5.1-1.5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libatm1:amd64 (1:2.5.1-1.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libmnl0:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libmnl0_1.0.3-5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libmnl0:amd64 (1.0.3-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package liblockfile-bin.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../liblockfile-bin_1.09-6ubuntu1_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking liblockfile-bin (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package liblockfile1:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../liblockfile1_1.09-6ubuntu1_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking liblockfile1:amd64 (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package cpio.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../cpio_2.11+dfsg-5ubuntu1_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking cpio (2.11+dfsg-5ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package iproute2.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../iproute2_4.3.0-1ubuntu3.16.04.5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking iproute2 (4.3.0-1ubuntu3.16.04.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ifupdown.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ifupdown_0.8.10ubuntu1.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ifupdown (0.8.10ubuntu1.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libisc-export160.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libisc-export160_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libdns-export162.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libdns-export162_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package isc-dhcp-client.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../isc-dhcp-client_4.3.3-5ubuntu12.10_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking isc-dhcp-client (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package isc-dhcp-common.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../isc-dhcp-common_4.3.3-5ubuntu12.10_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking isc-dhcp-common (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libxtables11:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libxtables11_1.6.0-2ubuntu3_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libxtables11:amd64 (1.6.0-2ubuntu3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package netbase.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../archives/netbase_5.3_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking netbase (5.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libxmuu1:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libxmuu1_2%3a1.1.2-2_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libxmuu1:amd64 (2:1.1.2-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package openssh-client.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../openssh-client_1%3a7.2p2-4ubuntu2.8_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking openssh-client (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package xauth.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../xauth_1%3a1.0.9-1ubuntu2_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking xauth (1:1.0.9-1ubuntu2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ssl-cert.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ssl-cert_1.0.37_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ssl-cert (1.0.37) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package postfix.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../postfix_3.1.0-3ubuntu0.3_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking postfix (3.1.0-3ubuntu0.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package bsd-mailx.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../bsd-mailx_8.1.2-0.20160123cvs-2_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking bsd-mailx (8.1.2-0.20160123cvs-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-common.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-common_6.2u5-7.4_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-common (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-client.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-client_6.2u5-7.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-client (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package tcsh.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../tcsh_6.18.01-5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking tcsh (6.18.01-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-exec.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-exec_6.2u5-7.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-exec (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-master.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-master_6.2u5-7.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-master (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ncurses-term.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ncurses-term_6.0+20160213-1ubuntu1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ncurses-term (6.0+20160213-1ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package openssh-sftp-server.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../openssh-sftp-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking openssh-sftp-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package openssh-server.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../openssh-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking openssh-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-pkg-resources.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-pkg-resources_20.7.0-1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-pkg-resources (20.7.0-1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-chardet.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-chardet_2.3.0-2_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-chardet (2.3.0-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-six.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-six_1.10.0-3_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-six (1.10.0-3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-urllib3.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-urllib3_1.13.1-2ubuntu0.16.04.3_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-requests.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-requests_2.9.1-3ubuntu0.1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-requests (2.9.1-3ubuntu0.1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ssh-import-id.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ssh-import-id_5.5-0ubuntu1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ssh-import-id (5.5-0ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for systemd (229-4ubuntu21.22) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for libc-bin (2.23-0ubuntu11) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libatm1:amd64 (1:2.5.1-1.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libmnl0:amd64 (1.0.3-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up liblockfile-bin (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up liblockfile1:amd64 (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up cpio (2.11+dfsg-5ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">update-alternatives: using /bin/mt-gnu to provide /bin/mt (mt) in auto mode</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up iproute2 (4.3.0-1ubuntu3.16.04.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ifupdown (0.8.10ubuntu1.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating /etc/network/interfaces.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up isc-dhcp-client (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up isc-dhcp-common (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libxtables11:amd64 (1.6.0-2ubuntu3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up netbase (5.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libxmuu1:amd64 (2:1.1.2-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up openssh-client (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up xauth (1:1.0.9-1ubuntu2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ssl-cert (1.0.37) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up postfix (3.1.0-3ubuntu0.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating /etc/postfix/dynamicmaps.cf</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting myhostname: master-0.XXX</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting alias maps</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting alias database</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">changing /etc/mailname to master-0.XXX</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting myorigin</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting destinations: $myhostname, master-0.XXX, localhost.XXX, , localhost</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting relayhost:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting mynetworks: 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting mailbox_size_limit: 0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting recipient_delimiter: +</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting inet_interfaces: all</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting inet_protocols: all</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/etc/aliases does not exist, creating it.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">WARNING: /etc/aliases exists, but does not have a root alias.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Postfix is now set up with a default configuration. If you need to make</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">changes, edit</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/etc/postfix/main.cf (and others) as needed. To view Postfix configuration</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">values, see postconf(1).</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">After modifying main.cf, be sure to run &#x27;/etc/init.d/postfix reload&#x27;.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Running newaliases</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of restart.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up bsd-mailx (8.1.2-0.20160123cvs-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">update-alternatives: using /usr/bin/bsd-mailx to provide /usr/bin/mailx (mailx) in auto mode</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-common (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating config file /etc/default/gridengine with new version</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-client (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up tcsh (6.18.01-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">update-alternatives: using /bin/tcsh to provide /bin/csh (csh) in auto mode</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-exec (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of start.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-master (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">su: Authentication failure</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Ignored)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Initializing cluster with the following parameters:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> =&gt; SGE_ROOT: /var/lib/gridengine</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> =&gt; SGE_CELL: default</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> =&gt; Spool directory: /var/spool/gridengine/spooldb</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> =&gt; Initial manager user: sgeadmin</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Initializing spool (/var/spool/gridengine/spooldb)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Initializing global configuration based on /usr/share/gridengine/default-configuration</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Initializing complexes based on /usr/share/gridengine/centry</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Initializing usersets based on /usr/share/gridengine/usersets</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Adding user sgeadmin as a manager</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Cluster creation complete</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of start.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ncurses-term (6.0+20160213-1ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up openssh-sftp-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up openssh-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 RSA key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">2048 SHA256:hfQpES1aS4cjF8AOCIParZR6342vdwutoyITru0wtuE root@master-0.XXX (RSA)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 DSA key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">1024 SHA256:gOsPMVgwXBHJzixN/gtJAG+hVCHqw8t7Fhy4nsx8od0 root@master-0.XXX (DSA)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 ECDSA key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">256 SHA256:3D5SNniUb4z+/BuqXheFgG+DfjsxXqTT/zwWAqdX4jM root@master-0.XXX (ECDSA)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 ED25519 key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">256 SHA256:SwyeV9iSqOW4TKLi4Wvc0zD8lWtupHCJpDu8oWBwbfU root@master-0.XXX (ED25519)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of start.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-pkg-resources (20.7.0-1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-chardet (2.3.0-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-six (1.10.0-3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-requests (2.9.1-3ubuntu0.1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ssh-import-id (5.5-0ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for libc-bin (2.23-0ubuntu11) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for systemd (229-4ubuntu21.22) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading package lists...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Building dependency tree...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading state information...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">0 upgraded, 0 newly installed, 0 to remove and 30 not upgraded.</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>cat $SGE_CFG_PATH/setcfg.log</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">finish master</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">add worker node worker-0.XXX</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>Sample output of worker:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">please wait</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading package lists...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Building dependency tree...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading state information...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">The following additional packages will be installed:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> bsd-mailx cpio gridengine-common ifupdown iproute2 isc-dhcp-client</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> isc-dhcp-common libatm1 libdns-export162 libisc-export160 liblockfile-bin</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> liblockfile1 libmnl0 libxmuu1 libxtables11 ncurses-term netbase</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> openssh-client openssh-server openssh-sftp-server postfix python3-chardet</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> python3-pkg-resources python3-requests python3-six python3-urllib3</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ssh-import-id ssl-cert tcsh xauth</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Suggested packages:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> libarchive1 gridengine-qmon ppp rdnssd iproute2-doc resolvconf avahi-autoipd</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> isc-dhcp-client-ddns apparmor ssh-askpass libpam-ssh keychain monkeysphere</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> rssh molly-guard ufw procmail postfix-mysql postfix-pgsql postfix-ldap</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> postfix-pcre sasl2-bin libsasl2-modules dovecot-common postfix-cdb</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> postfix-doc python3-setuptools python3-ndg-httpsclient python3-openssl</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> python3-pyasn1 openssl-blacklist</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">The following NEW packages will be installed:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> bsd-mailx cpio gridengine-client gridengine-common gridengine-exec ifupdown</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> iproute2 isc-dhcp-client isc-dhcp-common libatm1 libdns-export162</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> libisc-export160 liblockfile-bin liblockfile1 libmnl0 libxmuu1 libxtables11</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> ncurses-term netbase openssh-client openssh-server openssh-sftp-server</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> postfix python3-chardet python3-pkg-resources python3-requests python3-six</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"> python3-urllib3 ssh-import-id ssl-cert tcsh xauth</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">0 upgraded, 32 newly installed, 0 to remove and 30 not upgraded.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Need to get 9633 kB of archives.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">After this operation, 51.2 MB of additional disk space will be used.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:1 http://archive.ubuntu.com/ubuntu xenial/main amd64 libatm1 amd64 1:2.5.1-1.5 [24.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:2 http://archive.ubuntu.com/ubuntu xenial/main amd64 libmnl0 amd64 1.0.3-5 [12.0 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:3 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile-bin amd64 1.09-6ubuntu1 [10.8 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:4 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile1 amd64 1.09-6ubuntu1 [8056 B]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:5 http://archive.ubuntu.com/ubuntu xenial/main amd64 cpio amd64 2.11+dfsg-5ubuntu1 [74.8 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:6 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 iproute2 amd64 4.3.0-1ubuntu3.16.04.5 [523 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:7 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 ifupdown amd64 0.8.10ubuntu1.4 [54.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:8 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libisc-export160 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [153 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:9 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libdns-export162 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [665 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:10 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-client amd64 4.3.3-5ubuntu12.10 [224 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:11 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-common amd64 4.3.3-5ubuntu12.10 [105 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:12 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxtables11 amd64 1.6.0-2ubuntu3 [27.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:13 http://archive.ubuntu.com/ubuntu xenial/main amd64 netbase all 5.3 [12.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:14 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxmuu1 amd64 2:1.1.2-2 [9674 B]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:15 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-client amd64 1:7.2p2-4ubuntu2.8 [590 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:16 http://archive.ubuntu.com/ubuntu xenial/main amd64 xauth amd64 1:1.0.9-1ubuntu2 [22.7 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:17 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssl-cert all 1.0.37 [16.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:18 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 postfix amd64 3.1.0-3ubuntu0.3 [1152 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:19 http://archive.ubuntu.com/ubuntu xenial/main amd64 bsd-mailx amd64 8.1.2-0.20160123cvs-2 [63.7 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:20 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-common all 6.2u5-7.4 [156 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:21 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-client amd64 6.2u5-7.4 [3394 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:22 http://archive.ubuntu.com/ubuntu xenial/universe amd64 tcsh amd64 6.18.01-5 [410 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:23 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-exec amd64 6.2u5-7.4 [990 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:24 http://archive.ubuntu.com/ubuntu xenial/main amd64 ncurses-term all 6.0+20160213-1ubuntu1 [249 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:25 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-sftp-server amd64 1:7.2p2-4ubuntu2.8 [38.9 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:26 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-server amd64 1:7.2p2-4ubuntu2.8 [335 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:27 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-pkg-resources all 20.7.0-1 [79.0 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:28 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-chardet all 2.3.0-2 [96.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:29 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-six all 1.10.0-3 [11.0 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:30 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-urllib3 all 1.13.1-2ubuntu0.16.04.3 [58.5 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:31 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-requests all 2.9.1-3ubuntu0.1 [55.8 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Get:32 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssh-import-id all 5.5-0ubuntu1 [10.2 kB]</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Fetched 9633 kB in 2s (4496 kB/s)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libatm1:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 5%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 10%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 15%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 20%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 25%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 30%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 35%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 40%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 45%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 50%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 55%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 60%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 65%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 70%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 75%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 80%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 85%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 90%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 95%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 100%</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">(Reading database ... 21398 files and directories currently installed.)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libatm1_1%3a2.5.1-1.5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libatm1:amd64 (1:2.5.1-1.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libmnl0:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libmnl0_1.0.3-5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libmnl0:amd64 (1.0.3-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package liblockfile-bin.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../liblockfile-bin_1.09-6ubuntu1_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking liblockfile-bin (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package liblockfile1:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../liblockfile1_1.09-6ubuntu1_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking liblockfile1:amd64 (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package cpio.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../cpio_2.11+dfsg-5ubuntu1_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking cpio (2.11+dfsg-5ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package iproute2.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../iproute2_4.3.0-1ubuntu3.16.04.5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking iproute2 (4.3.0-1ubuntu3.16.04.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ifupdown.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ifupdown_0.8.10ubuntu1.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ifupdown (0.8.10ubuntu1.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libisc-export160.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libisc-export160_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libdns-export162.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libdns-export162_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package isc-dhcp-client.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../isc-dhcp-client_4.3.3-5ubuntu12.10_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking isc-dhcp-client (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package isc-dhcp-common.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../isc-dhcp-common_4.3.3-5ubuntu12.10_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking isc-dhcp-common (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libxtables11:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libxtables11_1.6.0-2ubuntu3_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libxtables11:amd64 (1.6.0-2ubuntu3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package netbase.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../archives/netbase_5.3_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking netbase (5.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package libxmuu1:amd64.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../libxmuu1_2%3a1.1.2-2_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking libxmuu1:amd64 (2:1.1.2-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package openssh-client.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../openssh-client_1%3a7.2p2-4ubuntu2.8_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking openssh-client (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package xauth.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../xauth_1%3a1.0.9-1ubuntu2_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking xauth (1:1.0.9-1ubuntu2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ssl-cert.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ssl-cert_1.0.37_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ssl-cert (1.0.37) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package postfix.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../postfix_3.1.0-3ubuntu0.3_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking postfix (3.1.0-3ubuntu0.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package bsd-mailx.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../bsd-mailx_8.1.2-0.20160123cvs-2_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking bsd-mailx (8.1.2-0.20160123cvs-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-common.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-common_6.2u5-7.4_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-common (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-client.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-client_6.2u5-7.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-client (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package tcsh.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../tcsh_6.18.01-5_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking tcsh (6.18.01-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package gridengine-exec.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../gridengine-exec_6.2u5-7.4_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking gridengine-exec (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ncurses-term.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ncurses-term_6.0+20160213-1ubuntu1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ncurses-term (6.0+20160213-1ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package openssh-sftp-server.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../openssh-sftp-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking openssh-sftp-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package openssh-server.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../openssh-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking openssh-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-pkg-resources.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-pkg-resources_20.7.0-1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-pkg-resources (20.7.0-1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-chardet.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-chardet_2.3.0-2_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-chardet (2.3.0-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-six.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-six_1.10.0-3_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-six (1.10.0-3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-urllib3.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-urllib3_1.13.1-2ubuntu0.16.04.3_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package python3-requests.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../python3-requests_2.9.1-3ubuntu0.1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking python3-requests (2.9.1-3ubuntu0.1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Selecting previously unselected package ssh-import-id.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Preparing to unpack .../ssh-import-id_5.5-0ubuntu1_all.deb ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Unpacking ssh-import-id (5.5-0ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for systemd (229-4ubuntu21.22) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for libc-bin (2.23-0ubuntu11) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libatm1:amd64 (1:2.5.1-1.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libmnl0:amd64 (1.0.3-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up liblockfile-bin (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up liblockfile1:amd64 (1.09-6ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up cpio (2.11+dfsg-5ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">update-alternatives: using /bin/mt-gnu to provide /bin/mt (mt) in auto mode</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up iproute2 (4.3.0-1ubuntu3.16.04.5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ifupdown (0.8.10ubuntu1.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating /etc/network/interfaces.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up isc-dhcp-client (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up isc-dhcp-common (4.3.3-5ubuntu12.10) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libxtables11:amd64 (1.6.0-2ubuntu3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up netbase (5.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up libxmuu1:amd64 (2:1.1.2-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up openssh-client (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up xauth (1:1.0.9-1ubuntu2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ssl-cert (1.0.37) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up postfix (3.1.0-3ubuntu0.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating /etc/postfix/dynamicmaps.cf</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting myhostname: worker-0.XXX</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting alias maps</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting alias database</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">changing /etc/mailname to worker-0.XXX</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting myorigin</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting destinations: $myhostname, worker-0.XXX, localhost.XXX, , localhost</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting relayhost:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting mynetworks: 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting mailbox_size_limit: 0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting recipient_delimiter: +</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting inet_interfaces: all</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">setting inet_protocols: all</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/etc/aliases does not exist, creating it.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">WARNING: /etc/aliases exists, but does not have a root alias.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Postfix is now set up with a default configuration. If you need to make</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">changes, edit</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">/etc/postfix/main.cf (and others) as needed. To view Postfix configuration</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">values, see postconf(1).</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">After modifying main.cf, be sure to run &#x27;/etc/init.d/postfix reload&#x27;.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Running newaliases</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of restart.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up bsd-mailx (8.1.2-0.20160123cvs-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">update-alternatives: using /usr/bin/bsd-mailx to provide /usr/bin/mailx (mailx) in auto mode</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-common (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating config file /etc/default/gridengine with new version</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-client (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up tcsh (6.18.01-5) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">update-alternatives: using /bin/tcsh to provide /bin/csh (csh) in auto mode</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up gridengine-exec (6.2u5-7.4) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of start.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ncurses-term (6.0+20160213-1ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up openssh-sftp-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up openssh-server (1:7.2p2-4ubuntu2.8) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 RSA key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">2048 SHA256:ok/TxzwtF5W8I55sDxrt4Agy4fuWn39BiSovvDObhVE root@worker-0.XXX (RSA)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 DSA key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">1024 SHA256:4y48kVYt3mS3q1KgZzEoYMnS/2d/tA8TJUK5uNSaxZY root@worker-0.XXX (DSA)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 ECDSA key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">256 SHA256:4D7zm4cD2IbDnHoXnzcIo3FISbvOW8eOstGBNf1/bvo root@worker-0.XXX (ECDSA)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Creating SSH2 ED25519 key; this may take some time ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">256 SHA256:/HrA3xiZiH5CZkXwtcfE6GwcMM+hEhZzTdFHxj4PzDg root@worker-0.XXX (ED25519)</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: could not determine current runlevel</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">invoke-rc.d: policy-rc.d denied execution of start.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-pkg-resources (20.7.0-1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-chardet (2.3.0-2) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-six (1.10.0-3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up python3-requests (2.9.1-3ubuntu0.1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Setting up ssh-import-id (5.5-0ubuntu1) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for libc-bin (2.23-0ubuntu11) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Processing triggers for systemd (229-4ubuntu21.22) ...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading package lists...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Building dependency tree...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Reading state information...</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">0 upgraded, 0 newly installed, 0 to remove and 30 not upgraded.</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>cat $SGE_CFG_PATH/setcfg.log</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">please wait</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Start SGE for worker is finished</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">done for worker-0.XXX worker.</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>Sample output of sge:
<img loading="lazy" alt="alt text" src="/assets/images/sge_cluster-b9ac6485a770b5f8a031b29f6f430810.png" title="SGE for multiple jobs" width="1658" height="572" class="img_E7b_"></p><p><img loading="lazy" alt="alt text" src="/assets/images/sge_stat-29ad9dfe485895eb54eea5baad709e15.png" title="SGE for user multiple jobs" width="1408" height="714" class="img_E7b_"></p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_dcUD" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_foO9"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div><div class="col col--3"><div class="tableOfContents_cNA8 thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-data-for-training" class="table-of-contents__link toc-highlight">Prepare data for training</a></li><li><a href="#prepare-docker-images" class="table-of-contents__link toc-highlight">Prepare Docker images</a></li><li><a href="#run-kaldi-jobs" class="table-of-contents__link toc-highlight">Run Kaldi jobs</a><ul><li><a href="#run-distributed-training" class="table-of-contents__link toc-highlight">Run distributed training</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/docs/gettingStarted/quickstart">Getting Started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/environment">API docs</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items"><li class="footer__item"><a href="https://stackoverflow.com/questions/tagged/apache-submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Stack Overflow<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://s.apache.org/slack-invite" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items"><li class="footer__item"><a href="https://medium.com/@apache.submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footerLogoLink_gHmE"><img src="https://hadoop.apache.org/asf_logo_wide.png" alt="Apache Open Source Logo" class="themedImage_W2Cr themedImage--light_TfLj footer__logo"><img src="https://hadoop.apache.org/asf_logo_wide.png" alt="Apache Open Source Logo" class="themedImage_W2Cr themedImage--dark_oUvU footer__logo"></a></div><div class="footer__copyright">Apache Submarine, Submarine, Apache, the Apache feather logo, and the Apache Submarine project logo are
either registered trademarks or trademarks of the Apache Software Foundation in the United States and other
countries.<br> Copyright Β© 2023 Apache Submarine is Apache2 Licensed software.</div></div></div></footer></div>
<script src="/assets/js/runtime~main.9d177e25.js"></script>
<script src="/assets/js/main.7cd2eed3.js"></script>
</body>
</html>