| |
| |
| |
| <!DOCTYPE html> |
| <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <title>Benchmark for Distributed training — incubator-singa 2.0.0 documentation</title> |
| |
| |
| |
| |
| |
| |
| |
| |
| <script type="text/javascript" src="../_static/js/modernizr.min.js"></script> |
| |
| |
| <script type="text/javascript"> |
| var DOCUMENTATION_OPTIONS = { |
| URL_ROOT:'../', |
| VERSION:'2.0.0', |
| LANGUAGE:'None', |
| COLLAPSE_INDEX:false, |
| FILE_SUFFIX:'.html', |
| HAS_SOURCE: true, |
| SOURCELINK_SUFFIX: '.txt' |
| }; |
| </script> |
| <script type="text/javascript" src="../_static/jquery.js"></script> |
| <script type="text/javascript" src="../_static/underscore.js"></script> |
| <script type="text/javascript" src="../_static/doctools.js"></script> |
| |
| <script type="text/javascript" src="../_static/js/theme.js"></script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> |
| <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Model Zoo" href="model_zoo/index.html" /> |
| <link rel="prev" title="ONNX" href="onnx.html" /> |
| <link href="../_static/style.css" rel="stylesheet" type="text/css"> |
| <!--link href="../_static/fontawesome-all.min.css" rel="stylesheet" type="text/css"--> |
| <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css" integrity="sha384-DNOHZ68U8hZfKXOrtjWvjxusGo9WQnrNx2sqG0tfsghAvtVlRW3tvkXWZh58N9jp" crossorigin="anonymous"> |
| <style> |
| .fa:hover { |
| opacity: 0.7; |
| } |
| .fab:hover { |
| opacity: 0.7; |
| } |
| </style> |
| |
| </head> |
| |
| <body class="wy-body-for-nav"> |
| |
| |
| <div class="wy-grid-for-nav"> |
| |
| <nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
| <div class="wy-side-scroll"> |
| <div class="wy-side-nav-search" > |
| |
| |
| |
| <a href="../index.html" class="icon icon-home"> incubator-singa |
| |
| |
| |
| |
| <img src="../_static/singa.png" class="logo" alt="Logo"/> |
| |
| </a> |
| |
| |
| |
| |
| <div class="version"> |
| latest |
| </div> |
| |
| |
| |
| |
| <div role="search"> |
| <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> |
| <input type="text" name="q" placeholder="Search docs" /> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| </form> |
| </div> |
| |
| |
| </div> |
| |
| <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
| |
| |
| |
| |
| |
| |
| <ul class="current"> |
| <li class="toctree-l1 current"><a class="reference internal" href="index.html">Documentation</a><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="installation.html">Installation</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="software_stack.html">Software Stack</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="device.html">Device</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="tensor.html">Tensor</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="autograd.html">Autograd in Singa</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="onnx.html">ONNX</a></li> |
| <li class="toctree-l2 current"><a class="current reference internal" href="#">Benchmark for Distributed training</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="model_zoo/index.html">Model Zoo</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="../downloads.html">Download SINGA</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> |
| </ul> |
| <p class="caption"><span class="caption-text">Development</span></p> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="../develop/contribute-code.html">How to Contribute Code</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../develop/contribute-docs.html">How to Contribute to Documentation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../develop/how-to-release.html">How to prepare a release</a></li> |
| </ul> |
| <p class="caption"><span class="caption-text">Community</span></p> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="../community/source-repository.html">Source Repository</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../community/mail-lists.html">Project Mailing Lists</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../community/issue-tracking.html">Issue Tracking</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../community/team-list.html">The SINGA Team</a></li> |
| </ul> |
| |
| |
| |
| </div> |
| </div> |
| </nav> |
| |
| <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
| |
| |
| <nav class="wy-nav-top" aria-label="top navigation"> |
| |
| <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| <a href="../index.html">incubator-singa</a> |
| |
| </nav> |
| |
| |
| <div class="wy-nav-content"> |
| |
| <div class="rst-content"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div role="navigation" aria-label="breadcrumbs navigation"> |
| |
| <ul class="wy-breadcrumbs"> |
| |
| <li><a href="../index.html">Docs</a> »</li> |
| |
| <li><a href="index.html">Documentation</a> »</li> |
| |
| <li>Benchmark for Distributed training</li> |
| |
| |
| <li class="wy-breadcrumbs-aside"> |
| |
| |
| |
| </li> |
| |
| </ul> |
| |
| |
| <hr/> |
| </div> |
| <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
| <div itemprop="articleBody"> |
| |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| --><div class="section" id="benchmark-for-distributed-training"> |
| <h1>Benchmark for Distributed training<a class="headerlink" href="#benchmark-for-distributed-training" title="Permalink to this headline">¶</a></h1> |
| <p>Workload: we use a deep convolutional neural network, <a class="reference external" href="https://github.com/apache/incubator-singa/blob/master/examples/autograd/resnet.py">ResNet-50</a> as the application. ResNet-50 is has 50 convolution layers for image classification. It requires 3.8 GFLOPs to pass a single image (of size 224x224) through the network. The input image size is 224x224.</p> |
| <p>Hardware: we use p2.8xlarge instances from AWS, each of which has 8 Nvidia Tesla K80 GPUs, 96 GB GPU memory in total, 32 vCPU, 488 GB main memory, 10 Gbps network bandwidth.</p> |
| <p>Metric: we measure the time per iteration for different number of workers to evaluate the scalability of SINGA. The batch size is fixed to be 32 per GPU. Synchronous training scheme is applied. As a result, the effective batch size is $32N$, where N is the number of GPUs. We compare with a popular open source system which uses the parameter server topology. The first GPU is selected as the server.</p> |
| <img src="../_static/images/benchmark.png" align="center" width="500px"/> |
| <br/> |
| <span><strong>Scalability test. Bars are for the throughput; lines are for the communication cost.</strong></span></div> |
| |
| |
| </div> |
| |
| </div> |
| <footer> |
| |
| <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> |
| |
| <a href="model_zoo/index.html" class="btn btn-neutral float-right" title="Model Zoo" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> |
| |
| |
| <a href="onnx.html" class="btn btn-neutral float-left" title="ONNX" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> |
| |
| </div> |
| |
| |
| <hr/> |
| |
| <div role="contentinfo"> |
| <p> |
| © Copyright 2019 The Apache Software Foundation. All rights reserved. Apache SINGA, Apache, the Apache feather logo, and the Apache SINGA project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners. |
| |
| </p> |
| </div> |
| Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
| |
| </footer> |
| |
| </div> |
| </div> |
| |
| </section> |
| |
| </div> |
| |
| |
| |
| <script type="text/javascript"> |
| jQuery(function () { |
| SphinxRtdTheme.Navigation.enable(true); |
| }); |
| </script> |
| |
| |
| |
| |
| |
| |
| <div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> |
| <span class="rst-current-version" data-toggle="rst-current-version"> |
| <span class="fa fa-book"> incubator-singa </span> |
| v: latest |
| <span class="fa fa-caret-down"></span> |
| </span> |
| <div class="rst-other-versions"> |
| <dl> |
| <dt>Languages</dt> |
| <dd><a href=".././index.html">English</a></dd> |
| <dd><a href=".././zh/index.html">中文</a></dd> |
| </dl> |
| <dl> |
| <dt>Versions</dt> |
| <dd><a href="http://singa.apache.org/v0.3.0/">0.3</a></dd> |
| <dd><a href="http://singa.apache.org/v1.1.0/">1.1</a></dd> |
| </dl> |
| |
| </div> |
| <a href="http://incubator.apache.org/"> <img src= "../_static/apache.png" style="background-color:white;"> </a> |
| |
| <a href="http://www.apache.org" style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Foundation</a> |
| <a href="http://www.apache.org/events/current-event" style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Events</a> |
| <a href="http://www.apache.org/foundation/thanks.html" style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Thanks</a> |
| <a href="http://www.apache.org/foundation/sponsorship.html" style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Sponsorship</a> |
| <a href="http://www.apache.org/licenses/" style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">License</a> |
| <br> |
| <a href="https://github.com/apache/incubator-singa" class="fa fa-github" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| <a href="https://aws.amazon.com/marketplace/seller-profile?id=5bcac385-12c4-4802-aec7-351e09b77b4c" class="fab fa-aws" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| <a href="https://hub.docker.com/r/apache/singa/" class="fab fa-docker" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| <a href="https://www.linkedin.com/groups/13550034" class="fa fa-linkedin" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| <a href="https://twitter.com/ApacheSinga" class="fa fa-twitter" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| <a href="https://www.facebook.com/Apache-SINGA-347284219056544/" class="fa fa-facebook" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| <a href="https://www.researchgate.net/project/Apache-SINGA" class="fab fa-researchgate" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> |
| |
| </div> |
| |
| <a href="https://github.com/apache/incubator-singa"> |
| <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" |
| src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png" |
| alt="Fork me on GitHub"> |
| </a> |
| |
| |
| |
| |
| </body> |
| </html> |
