<!doctype html>
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-0.6.0 plugin-docs plugin-id-default docs-doc-id-userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/README">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="generator" content="Docusaurus v2.0.0-beta.18">
<title data-rh="true">README | Apache Submarine</title><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://submarine.apache.org//docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="0.6.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-0.6.0"><meta data-rh="true" name="docsearch:version" content="0.6.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-0.6.0"><meta data-rh="true" property="og:title" content="README | Apache Submarine"><meta data-rh="true" name="description" content="&lt;!--"><meta data-rh="true" property="og:description" content="&lt;!--"><link data-rh="true" rel="icon" href="/img/submarine.ico"><link data-rh="true" rel="canonical" href="https://submarine.apache.org//docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/"><link data-rh="true" rel="alternate" href="https://submarine.apache.org//docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/" hreflang="en"><link data-rh="true" rel="alternate" href="https://submarine.apache.org//zh-cn/docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/" hreflang="zh-cn"><link data-rh="true" rel="alternate" href="https://submarine.apache.org//docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/" hreflang="x-default"><link rel="stylesheet" href="/assets/css/styles.80258812.css">
<link rel="preload" href="/assets/js/runtime~main.89d436a2.js" as="script">
<link rel="preload" href="/assets/js/main.b3d57bf8.js" as="script">
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
<div role="region"><a href="#" class="skipToContent_ZgBM">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle clean-btn" type="button" tabindex="0"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_W2Cr themedImage--light_TfLj"><img src="/img/icons/128.png" alt="Apache Submarine Site Logo" class="themedImage_W2Cr themedImage--dark_oUvU"></div><b class="navbar__title">Apache Submarine</b></a><a class="navbar__item navbar__link" href="/docs/0.6.0/gettingStarted/quickstart">Docs</a><a class="navbar__item navbar__link" href="/docs/0.6.0/api/environment">API</a><a class="navbar__item navbar__link" href="/docs/0.6.0/download">Download</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><span><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_dNtB"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg><span>English</span></span></a><ul class="dropdown__menu"><li><a href="/docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active">English</a></li><li><a href="/zh-cn/docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/" target="_self" rel="noopener noreferrer" class="dropdown__link">中文</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/0.6.0/gettingStarted/quickstart">0.6.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/gettingStarted/quickstart">master 🏃</a></li><li><a class="dropdown__link" href="/docs/gettingStarted/quickstart">0.7.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/">0.6.0</a></li><li><a class="dropdown__link" href="/versions">All versions</a></li></ul></div><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="toggle_S7eR colorModeToggle_vKtC"><button class="clean-btn toggleButton_rCf9 toggleButtonDisabled_Pu9x" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_v35p"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_nQuB"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="navbar__search"><span aria-label="expand searchbar" role="button" class="search-icon" tabindex="0"></span><input type="search" id="search_input_react" placeholder="Search" aria-label="Search" class="navbar__search-input search-bar"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div class="main-wrapper"><div class="docPage_P2Lg"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_RiI4" type="button"></button><main class="docMainContainer_TCnq docMainContainerEnhanced_WDCb"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_DM6M"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache Submarine<!-- --> <b>0.6.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/gettingStarted/quickstart">latest version</a></b> (<!-- -->0.7.0<!-- -->).</div></div><div class="docItemContainer_vinB"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.6.0</span><div class="tocCollapsible_jdIR theme-doc-toc-mobile tocMobile_TmEX"><button type="button" class="clean-btn tocCollapsibleButton_Fzxq">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>README</h1></header><p>(Copied from <a href="https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator" target="_blank" rel="noopener noreferrer">https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator</a>)</p><p>CIFAR-10 is a common benchmark in machine learning for image recognition.</p><p><a href="http://www.cs.toronto.edu/~kriz/cifar.html" target="_blank" rel="noopener noreferrer">http://www.cs.toronto.edu/~kriz/cifar.html</a></p><p>Code in this directory focuses on how to use TensorFlow Estimators to train and
evaluate a CIFAR-10 ResNet model on:</p><ul><li>A single host with one CPU;</li><li>A single host with multiple GPUs;</li><li>Multiple hosts with CPU or multiple GPUs;</li></ul><p>Before trying to run the model we highly encourage you to read all the README.</p><h2 class="anchor anchorWithStickyNavbar_mojV" id="prerequisite">Prerequisite<a class="hash-link" href="#prerequisite" title="Direct link to heading">​</a></h2><ol><li><p><a href="https://www.tensorflow.org/install/" target="_blank" rel="noopener noreferrer">Install</a> TensorFlow version 1.2.1 or
later.</p></li><li><p>Download the CIFAR-10 dataset and generate TFRecord files using the provided
script.  The script and associated command below will download the CIFAR-10
dataset and then generate a TFRecord for the training, validation, and
evaluation datasets.</p></li></ol><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python generate_cifar10_tfrecords.py --data-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token variable" style="color:rgb(191, 199, 213)">${</span><span class="token variable environment constant" style="color:rgb(130, 170, 255)">PWD</span><span class="token variable" style="color:rgb(191, 199, 213)">}</span><span class="token plain">/cifar-10-data</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>After running the command above, you should see the following files in the
--data-dir (<code>ls -R cifar-10-data</code>):</p><ul><li>train.tfrecords</li><li>validation.tfrecords</li><li>eval.tfrecords</li></ul><h2 class="anchor anchorWithStickyNavbar_mojV" id="training-on-a-single-machine-with-gpus-or-cpu">Training on a single machine with GPUs or CPU<a class="hash-link" href="#training-on-a-single-machine-with-gpus-or-cpu" title="Direct link to heading">​</a></h2><p>Run the training on CPU only. After training, it runs the evaluation.</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python cifar10_main.py --data-dir=${PWD}/cifar-10-data \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --job-dir=/tmp/cifar10 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --num-gpus=0 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --train-steps=1000</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>Run the model on 2 GPUs using CPU as parameter server. After training, it runs
the evaluation.</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python cifar10_main.py --data-dir=${PWD}/cifar-10-data \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --job-dir=/tmp/cifar10 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --num-gpus=2 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --train-steps=1000</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>Run the model on 2 GPUs using GPU as parameter server.
It will run an experiment, which for local setting basically means it will run
stop training
a couple of times to perform evaluation.</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python cifar10_main.py --data-dir=${PWD}/cifar-10-data \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --job-dir=/tmp/cifar10 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --variable-strategy GPU \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --num-gpus=2 \</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>There are more command line flags to play with; run
<code>python cifar10_main.py --help</code> for details.</p><h2 class="anchor anchorWithStickyNavbar_mojV" id="run-distributed-training">Run distributed training<a class="hash-link" href="#run-distributed-training" title="Direct link to heading">​</a></h2><h3 class="anchor anchorWithStickyNavbar_mojV" id="optional-running-on-google-cloud-machine-learning-engine">(Optional) Running on Google Cloud Machine Learning Engine<a class="hash-link" href="#optional-running-on-google-cloud-machine-learning-engine" title="Direct link to heading">​</a></h3><p>This example can be run on Google Cloud Machine Learning Engine (ML Engine),
which will configure the environment and take care of running workers,
parameters servers, and masters in a fault tolerant way.</p><p>To install the command line tool, and set up a project and billing, see the
quickstart <a href="https://cloud.google.com/ml-engine/docs/quickstarts/command-line" target="_blank" rel="noopener noreferrer">here</a>.</p><p>You&#x27;ll also need a Google Cloud Storage bucket for the data. If you followed the
instructions above, you can just run:</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">MY_BUCKET=gs://&lt;my-bucket-name&gt;</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">gsutil cp -r ${PWD}/cifar-10-data $MY_BUCKET/</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>Then run the following command from the <code>tutorials/image</code> directory of this
repository (the parent directory of this README):</p><div class="codeBlockContainer_I0IT theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-text codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">gcloud ml-engine jobs submit training cifarmultigpu \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --runtime-version 1.2 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --job-dir=$MY_BUCKET/model_dirs/cifarmultigpu \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --config cifar10_estimator/cmle_config.yaml \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --package-path cifar10_estimator/ \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --module-name cifar10_estimator.cifar10_main \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    -- \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --data-dir=$MY_BUCKET/cifar-10-data \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --num-gpus=4 \</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    --train-steps=1000</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><h3 class="anchor anchorWithStickyNavbar_mojV" id="set-tf_config">Set TF_CONFIG<a class="hash-link" href="#set-tf_config" title="Direct link to heading">​</a></h3><p>Considering that you already have multiple hosts configured, all you need is a
<code>TF_CONFIG</code> environment variable on each host. You can set up the hosts manually
or check <a href="https://github.com/tensorflow/ecosystem" target="_blank" rel="noopener noreferrer">tensorflow/ecosystem</a> for
instructions about how to set up a Cluster.</p><p>The <code>TF_CONFIG</code> will be used by the <code>RunConfig</code> to know the existing hosts and
their task: <code>master</code>, <code>ps</code> or <code>worker</code>.</p><p>Here&#x27;s an example of <code>TF_CONFIG</code>.</p><div class="codeBlockContainer_I0IT language-python theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-python codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cluster </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;master&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;master-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">           </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;ps&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;ps-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">           </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;worker&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;worker-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">TF_CONFIG </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cluster&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> cluster</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">   </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;task&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;type&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> master</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;index&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">   </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;model_dir&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;gs://&lt;bucket_path&gt;/&lt;dir_path&gt;&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">   </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;environment&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cloud&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p><em>Cluster</em></p><p>A cluster spec, which is basically a dictionary that describes all of the tasks
in the cluster. More about it <a href="https://www.tensorflow.org/deploy/distributed" target="_blank" rel="noopener noreferrer">here</a>.</p><p>In this cluster spec we are defining a cluster with 1 master, 1 ps and 1 worker.</p><ul><li><p><code>ps</code>: saves the parameters among all workers. All workers can
read/write/update the parameters for model via ps. As some models are
extremely large the parameters are shared among the ps (each ps stores a
subset).</p></li><li><p><code>worker</code>: does the training.</p></li><li><p><code>master</code>: basically a special worker, it does training, but also restores and
saves checkpoints and do evaluation.</p></li></ul><p><em>Task</em></p><p>The Task defines what is the role of the current node, for this example the node
is the master on index 0 on the cluster spec, the task will be different for
each node. An example of the <code>TF_CONFIG</code> for a worker would be:</p><div class="codeBlockContainer_I0IT language-python theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-python codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">cluster </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;master&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;master-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">           </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;ps&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;ps-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">           </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;worker&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;worker-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">TF_CONFIG </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cluster&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> cluster</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">   </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;task&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;type&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> worker</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;index&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">   </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;model_dir&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;gs://&lt;bucket_path&gt;/&lt;dir_path&gt;&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">   </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;environment&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cloud&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p><em>Model_dir</em></p><p>This is the path where the master will save the checkpoints, graph and
TensorBoard files. For a multi host environment you may want to use a
Distributed File System, Google Storage and DFS are supported.</p><p><em>Environment</em></p><p>By the default environment is <em>local</em>, for a distributed setting we need to
change it to <em>cloud</em>.</p><h3 class="anchor anchorWithStickyNavbar_mojV" id="running-script">Running script<a class="hash-link" href="#running-script" title="Direct link to heading">​</a></h3><p>Once you have a <code>TF_CONFIG</code> configured properly on each host you&#x27;re ready to run
on distributed settings.</p><h4 class="anchor anchorWithStickyNavbar_mojV" id="master">Master<a class="hash-link" href="#master" title="Direct link to heading">​</a></h4><p>Run this on master:
Runs an Experiment in sync mode on 4 GPUs using CPU as parameter server for
40000 steps. It will run evaluation a couple of times during training. The
num_workers argument is used only to update the learning rate correctly. Make
sure the model_dir is the same as defined on the TF_CONFIG.</p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python cifar10_main.py --data-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">gs://path/cifar-10-data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --job-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">gs://path/model_dir/ </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --num-gpus</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --train-steps</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">40000</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --sync </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --num-workers</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p><em>Output:</em></p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Using model_dir </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> TF_CONFIG: gs://path/model_dir/</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Using config: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_checkpoints_secs&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">600</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_num_ps_replicas&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_keep_checkpoint_max&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_task_type&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;master&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_is_chief&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> True, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_cluster_spec&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">&lt;</span><span class="token plain">tensorflow.python.training.server_lib.ClusterSpec object at 0x7fd16fb2be1</span><span class="token operator file-descriptor important" style="color:rgb(137, 221, 255)">0</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_model_dir&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;gs://path/model_dir/&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_checkpoints_steps&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> None, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_keep_checkpoint_every_n_hours&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">10000</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_session_config&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> intra_op_parallelism_threads: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">allow_soft_placement: </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_tf_random_seed&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> None, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_environment&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cloud&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_num_worker_replicas&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_task_id&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_summary_steps&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">100</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_tf_config&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  per_process_gpu_memory_fraction: </span><span class="token number" style="color:rgb(247, 140, 108)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_evaluation_master&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_master&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;grpc://master-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:26.496208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Found device </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain"> with properties:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">name: Tesla K80</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">major: </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"> minor: </span><span class="token number" style="color:rgb(247, 140, 108)">7</span><span class="token plain"> memoryClockRate </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">GHz</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.8235</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">pciBusID 0000:00:04.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Total memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.17GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Free memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.09GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:26.775660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Found device </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> with properties:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">name: Tesla K80</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">major: </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"> minor: </span><span class="token number" style="color:rgb(247, 140, 108)">7</span><span class="token plain"> memoryClockRate </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">GHz</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.8235</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">pciBusID 0000:00:05.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Total memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.17GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Free memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.10GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:29.675171: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Started server with target: grpc://localhost:8000</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:SyncReplicasV2: </span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">replicas_to_aggregate</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">;</span><span class="token plain"> </span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">total_num_replicas</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Create CheckpointSaverHook.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Restoring parameters from gs://path/model_dir/model.ckpt-0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:37.560775: I tensorflow/core/distributed_runtime/master_session.cc:999</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Start master session 156fcb55fe6648d6 with config:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">intra_op_parallelism_threads: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  per_process_gpu_memory_fraction: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">allow_soft_placement: </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Saving checkpoints </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> into gs://path/model_dir/model.ckpt.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1.20682</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1.20682</span><span class="token plain">, learning_rate </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:SyncReplicasV2: </span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">replicas_to_aggregate</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">;</span><span class="token plain"> </span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">total_num_replicas</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Starting evaluation at </span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01-20:00:14</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:04.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745949: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:05.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:06.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745964: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:3</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:07.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745969: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:4</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:08.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745975: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:5</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">5</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:09.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:6</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">6</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:0a.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain">:00:15.745997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Creating TensorFlow device </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">/gpu:7</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">device: </span><span class="token number" style="color:rgb(247, 140, 108)">7</span><span class="token plain">, name: Tesla K80, pci bus id: 0000:00:0b.0</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Restoring parameters from gs://path/model_dir/model.ckpt-10023</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">5</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">6</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">7</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">9</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">12</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">13</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Evaluation </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token number" style="color:rgb(247, 140, 108)">100</span><span class="token plain">/100</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Finished evaluation at </span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01-20:00:31</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Saving dict </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> global step </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">: accuracy </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.0994</span><span class="token plain">, global_step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">630.425</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><h4 class="anchor anchorWithStickyNavbar_mojV" id="worker">Worker<a class="hash-link" href="#worker" title="Direct link to heading">​</a></h4><p>Run this on worker:
Runs an Experiment in sync mode on 4 GPUs using CPU as parameter server for
40000 steps. It will run evaluation a couple of times during training. Make sure
the model_dir is the same as defined on the TF_CONFIG.</p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python cifar10_main.py --data-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">gs://path/cifar-10-data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --job-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">gs://path/model_dir/ </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --num-gpus</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">4</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --train-steps</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">40000</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">\</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">                       --sync</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p><em>Output:</em></p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Using model_dir </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> TF_CONFIG: gs://path/model_dir/</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Using config: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_checkpoints_secs&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">600</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_num_ps_replicas&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_keep_checkpoint_max&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_task_type&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;worker&#x27;</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_is_chief&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> False, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_cluster_spec&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token operator" style="color:rgb(137, 221, 255)">&lt;</span><span class="token plain">tensorflow.python.training.server_lib.ClusterSpec object at 0x7f6918438e1</span><span class="token operator file-descriptor important" style="color:rgb(137, 221, 255)">0</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_model_dir&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;gs://&lt;path&gt;/model_dir/&#x27;</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_checkpoints_steps&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> None, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_keep_checkpoint_every_n_hours&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">10000</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_session_config&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> intra_op_parallelism_threads: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">allow_soft_placement: </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_tf_random_seed&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> None, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_environment&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cloud&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_num_worker_replicas&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">,</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_task_id&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_summary_steps&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">100</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_tf_config&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  per_process_gpu_memory_fraction: </span><span class="token number" style="color:rgb(247, 140, 108)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:26.496208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Found device </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain"> with properties:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">name: Tesla K80</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">major: </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"> minor: </span><span class="token number" style="color:rgb(247, 140, 108)">7</span><span class="token plain"> memoryClockRate </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">GHz</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.8235</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">pciBusID 0000:00:04.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Total memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.17GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Free memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.09GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:26.775660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Found device </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"> with properties:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">name: Tesla K80</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">major: </span><span class="token number" style="color:rgb(247, 140, 108)">3</span><span class="token plain"> minor: </span><span class="token number" style="color:rgb(247, 140, 108)">7</span><span class="token plain"> memoryClockRate </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">GHz</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.8235</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">pciBusID 0000:00:05.0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Total memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.17GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">Free memory: </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token plain">.10GiB</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-08-01 </span><span class="token number" style="color:rgb(247, 140, 108)">19</span><span class="token plain">:59:29.675171: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Started server with target: grpc://localhost:8000</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">16</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">32</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token plain">, </span><span class="token number" style="color:rgb(247, 140, 108)">8</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">64</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">?, </span><span class="token number" style="color:rgb(247, 140, 108)">11</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:SyncReplicasV2: </span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">replicas_to_aggregate</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token punctuation" style="color:rgb(199, 146, 234)">;</span><span class="token plain"> </span><span class="token assign-left variable" style="color:rgb(191, 199, 213)">total_num_replicas</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token number" style="color:rgb(247, 140, 108)">2</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Create CheckpointSaverHook.</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-07-31 </span><span class="token number" style="color:rgb(247, 140, 108)">22</span><span class="token plain">:38:04.629150: I</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">tensorflow/core/distributed_runtime/master.cc:209</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> CreateSession still waiting</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> response from worker: /job:master/replica:0/task:0</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-07-31 </span><span class="token number" style="color:rgb(247, 140, 108)">22</span><span class="token plain">:38:09.263492: I</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">tensorflow/core/distributed_runtime/master_session.cc:999</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Start master</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">session cc58f93b1e259b0c with config:</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">intra_op_parallelism_threads: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">per_process_gpu_memory_fraction: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">allow_soft_placement: </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5.82382</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5.82382</span><span class="token plain">, learning_rate </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.8</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">1116.92</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1116.92</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">1233.73</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">1377.83</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">20</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">1485.43</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2509.3</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">30</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">1680.27</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2770.39</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">40</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">1825.38</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2788.78</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">50</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">1929.32</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2697.27</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">60</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2015.17</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2749.05</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">70</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">37.6272</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">79</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">19.554</span><span class="token plain"> sec</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">37.6272</span><span class="token plain">, learning_rate </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0.8</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">19.554</span><span class="token plain"> sec</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2074.92</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2618.36</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">80</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2132.71</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2744.13</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">90</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2183.38</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2777.21</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2224.4</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2739.03</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">110</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2240.28</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2431.26</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">120</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2272.12</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2739.32</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">130</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2300.68</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2750.03</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">140</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2325.81</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2745.63</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">150</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2347.14</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2721.53</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">160</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Average examples/sec: </span><span class="token number" style="color:rgb(247, 140, 108)">2367.74</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">2754.54</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">170</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:loss </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">27.8453</span><span class="token plain">, step </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">179</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token number" style="color:rgb(247, 140, 108)">18.893</span><span class="token plain"> sec</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">..</span><span class="token plain">.</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><h4 class="anchor anchorWithStickyNavbar_mojV" id="ps">PS<a class="hash-link" href="#ps" title="Direct link to heading">​</a></h4><p>Run this on ps:
The ps will not do training so most of the arguments won&#x27;t affect the execution</p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">python cifar10_main.py --job-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">gs://path/model_dir/</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p><em>Output:</em></p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Using model_dir </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> TF_CONFIG: gs://path/model_dir/</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">INFO:tensorflow:Using config: </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_checkpoints_secs&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">600</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_num_ps_replicas&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_keep_checkpoint_max&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">5</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_task_type&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;ps&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_is_chief&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> False, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_cluster_spec&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">&lt;</span><span class="token plain">tensorflow.python.training.server_lib.ClusterSpec object at 0x7f48f1addf9</span><span class="token operator file-descriptor important" style="color:rgb(137, 221, 255)">0</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_model_dir&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;gs://path/model_dir/&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_checkpoints_steps&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> None, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_keep_checkpoint_every_n_hours&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">10000</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_session_config&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> intra_op_parallelism_threads: </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">allow_soft_placement: </span><span class="token boolean" style="color:rgb(255, 88, 116)">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_tf_random_seed&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> None, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_environment&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;cloud&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_num_worker_replicas&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_task_id&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_save_summary_steps&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(247, 140, 108)">100</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_tf_config&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> gpu_options </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">  per_process_gpu_memory_fraction: </span><span class="token number" style="color:rgb(247, 140, 108)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_evaluation_master&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;&#x27;</span><span class="token plain">, </span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;_master&#x27;</span><span class="token builtin class-name" style="color:rgb(255, 203, 107)">:</span><span class="token plain"> u</span><span class="token string" style="color:rgb(195, 232, 141)">&#x27;grpc://master-ip:8000&#x27;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-07-31 </span><span class="token number" style="color:rgb(247, 140, 108)">22</span><span class="token plain">:54:58.928088: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Initialize GrpcChannelCache </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> job master -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> master-ip:8000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-07-31 </span><span class="token number" style="color:rgb(247, 140, 108)">22</span><span class="token plain">:54:58.928153: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Initialize GrpcChannelCache </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> job </span><span class="token function" style="color:rgb(130, 170, 255)">ps</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> localhost:8000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-07-31 </span><span class="token number" style="color:rgb(247, 140, 108)">22</span><span class="token plain">:54:58.928160: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Initialize GrpcChannelCache </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> job worker -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">{</span><span class="token number" style="color:rgb(247, 140, 108)">0</span><span class="token plain"> -</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"> worker-ip:8000</span><span class="token punctuation" style="color:rgb(199, 146, 234)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token number" style="color:rgb(247, 140, 108)">2017</span><span class="token plain">-07-31 </span><span class="token number" style="color:rgb(247, 140, 108)">22</span><span class="token plain">:54:58.929873: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> Started server with target: grpc://localhost:8000</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><h2 class="anchor anchorWithStickyNavbar_mojV" id="visualizing-results-with-tensorboard">Visualizing results with TensorBoard<a class="hash-link" href="#visualizing-results-with-tensorboard" title="Direct link to heading">​</a></h2><p>When using Estimators you can also visualize your data in TensorBoard, with no
changes in your code. You can use TensorBoard to visualize your TensorFlow
graph, plot quantitative metrics about the execution of your graph, and show
additional data like images that pass through it.</p><p>You&#x27;ll see something similar to this if you &quot;point&quot; TensorBoard to the
<code>job dir</code> parameter you used to train or evaluate your model.</p><p>Check TensorBoard during training or after it. Just point TensorBoard to the
model_dir you chose on the previous step.</p><div class="codeBlockContainer_I0IT language-shell theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-shell codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">tensorboard --log-dir</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token string" style="color:rgb(195, 232, 141)">&quot;&lt;job dir&gt;&quot;</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><h2 class="anchor anchorWithStickyNavbar_mojV" id="warnings">Warnings<a class="hash-link" href="#warnings" title="Direct link to heading">​</a></h2><p>When running <code>cifar10_main.py</code> with <code>--sync</code> argument you may see an error
similar to:</p><div class="codeBlockContainer_I0IT language-python theme-code-block"><div class="codeBlockContent_wNvx" style="color:#bfc7d5;background-color:#292d3e"><pre tabindex="0" class="prism-code language-python codeBlock_jd64 thin-scrollbar"><code class="codeBlockLines_mRuA"><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;cifar10_main.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">538</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">&lt;</span><span class="token plain">module</span><span class="token operator" style="color:rgb(137, 221, 255)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    tf</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">app</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">48</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> run</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    _sys</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">exit</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">main</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">_sys</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">argv</span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token number" style="color:rgb(247, 140, 108)">1</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">+</span><span class="token plain"> flags_passthrough</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;cifar10_main.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">518</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> main</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    hooks</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> run_config</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">config</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">210</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> run</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token keyword" style="font-style:italic">return</span><span class="token plain"> _execute_schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">experiment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> schedule</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">47</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> _execute_schedule</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token keyword" style="font-style:italic">return</span><span class="token plain"> task</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">501</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> train_and_evaluate</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    hooks</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">_eval_hooks</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">681</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> _call_evaluate</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    hooks</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">hooks</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">292</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> evaluate</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    name</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">638</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> _evaluate_model</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    features</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> labels</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> model_fn_lib</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">ModeKeys</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">EVAL</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">545</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> _call_model_fn</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    features</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> labels</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">**</span><span class="token plain">kwargs</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;cifar10_main.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">331</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> _resnet_model_fn</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    gradvars</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> global_step</span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain">tf</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">get_global_step</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/sync_replicas_optimizer.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">252</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> apply_gradients</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    variables</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">global_variables</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">170</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> wrapped</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token keyword" style="font-style:italic">return</span><span class="token plain"> _add_should_use_warning</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">fn</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token operator" style="color:rgb(137, 221, 255)">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token operator" style="color:rgb(137, 221, 255)">**</span><span class="token plain">kwargs</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">139</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> _add_should_use_warning</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    wrapped </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> TFShouldUseWarningWrapper</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">File </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py&quot;</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> line </span><span class="token number" style="color:rgb(247, 140, 108)">96</span><span class="token punctuation" style="color:rgb(199, 146, 234)">,</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> __init__</span><br></span><span class="token-line" style="color:#bfc7d5"><span class="token plain">    stack </span><span class="token operator" style="color:rgb(137, 221, 255)">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(199, 146, 234)">[</span><span class="token plain">s</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">strip</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token plain"> </span><span class="token keyword" style="font-style:italic">for</span><span class="token plain"> s </span><span class="token keyword" style="font-style:italic">in</span><span class="token plain"> traceback</span><span class="token punctuation" style="color:rgb(199, 146, 234)">.</span><span class="token plain">format_stack</span><span class="token punctuation" style="color:rgb(199, 146, 234)">(</span><span class="token punctuation" style="color:rgb(199, 146, 234)">)</span><span class="token punctuation" style="color:rgb(199, 146, 234)">]</span><br></span></code></pre><button type="button" aria-label="Copy code to clipboard" title="Copy" class="copyButton_eDfN clean-btn"><span class="copyButtonIcons_W9eQ" aria-hidden="true"><svg class="copyButtonIcon_XEyF" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_i9w9" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div><p>This should not affect your training, and should be fixed on the next releases.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/submarine/edit/master/website/versioned_docs/version-0.6.0/userDocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/README.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_dcUD" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_foO9"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div><div class="col col--3"><div class="tableOfContents_cNA8 thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisite" class="table-of-contents__link toc-highlight">Prerequisite</a></li><li><a href="#training-on-a-single-machine-with-gpus-or-cpu" class="table-of-contents__link toc-highlight">Training on a single machine with GPUs or CPU</a></li><li><a href="#run-distributed-training" class="table-of-contents__link toc-highlight">Run distributed training</a><ul><li><a href="#optional-running-on-google-cloud-machine-learning-engine" class="table-of-contents__link toc-highlight">(Optional) Running on Google Cloud Machine Learning Engine</a></li><li><a href="#set-tf_config" class="table-of-contents__link toc-highlight">Set TF_CONFIG</a></li><li><a href="#running-script" class="table-of-contents__link toc-highlight">Running script</a></li></ul></li><li><a href="#visualizing-results-with-tensorboard" class="table-of-contents__link toc-highlight">Visualizing results with TensorBoard</a></li><li><a href="#warnings" class="table-of-contents__link toc-highlight">Warnings</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items"><li class="footer__item"><a class="footer__link-item" href="/docs/gettingStarted/quickstart">Getting Started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/environment">API docs</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items"><li class="footer__item"><a href="https://stackoverflow.com/questions/tagged/apache-submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Stack Overflow<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://s.apache.org/slack-invite" target="_blank" rel="noopener noreferrer" class="footer__link-item">Slack<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items"><li class="footer__item"><a href="https://medium.com/@apache.submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">Blog</a></li><li class="footer__item"><a href="https://github.com/apache/submarine" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_I5OW"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footerLogoLink_gHmE"><img src="https://hadoop.apache.org/asf_logo_wide.png" alt="Apache Open Source Logo" class="themedImage_W2Cr themedImage--light_TfLj footer__logo"><img src="https://hadoop.apache.org/asf_logo_wide.png" alt="Apache Open Source Logo" class="themedImage_W2Cr themedImage--dark_oUvU footer__logo"></a></div><div class="footer__copyright">Apache Submarine, Submarine, Apache, the Apache feather logo, and the Apache Submarine project logo are
       either registered trademarks or trademarks of the Apache Software Foundation in the United States and other
        countries.<br> Copyright © 2023 Apache Submarine is Apache2 Licensed software.</div></div></div></footer></div>
<script src="/assets/js/runtime~main.89d436a2.js"></script>
<script src="/assets/js/main.b3d57bf8.js"></script>
</body>
</html>