versions/master/how_to/model_parallel_lstm.html - mxnet-test - Git at Google

 <!DOCTYPE html>

 <html lang="en">
 <head>
 <meta charset="utf-8"/>
 <meta content="IE=edge" http-equiv="X-UA-Compatible"/>
 <meta content="width=device-width, initial-scale=1" name="viewport"/>
 <title>Training with Multiple GPUs Using Model Parallelism — mxnet  documentation</title>
 <link crossorigin="anonymous" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" rel="stylesheet"/>
 <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css" rel="stylesheet"/>
 <link href="../_static/basic.css" rel="stylesheet" type="text/css">
 <link href="../_static/pygments.css" rel="stylesheet" type="text/css">
 <link href="../_static/mxnet.css" rel="stylesheet" type="text/css"/>
 <script type="text/javascript">
       var DOCUMENTATION_OPTIONS = {
         URL_ROOT:    '../',
         VERSION:     '',
         COLLAPSE_INDEX: false,
         FILE_SUFFIX: '.html',
         HAS_SOURCE:  true,
         SOURCELINK_SUFFIX: ''
       };
     </script>
 <script src="../_static/jquery-1.11.1.js" type="text/javascript"></script>
 <script src="../_static/underscore.js" type="text/javascript"></script>
 <script src="../_static/searchtools_custom.js" type="text/javascript"></script>
 <script src="../_static/doctools.js" type="text/javascript"></script>
 <script src="../_static/selectlang.js" type="text/javascript"></script>
 <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" type="text/javascript"></script>
 <script type="text/javascript"> jQuery(function() { Search.loadIndex("/searchindex.js"); Search.init();}); </script>
 <script>
       (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new
       Date();a=s.createElement(o),
       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
       })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

       ga('create', 'UA-96378503-1', 'auto');
       ga('send', 'pageview');

     </script>
 <!-- -->
 <!-- <script type="text/javascript" src="../_static/jquery.js"></script> -->
 <!-- -->
 <!-- <script type="text/javascript" src="../_static/underscore.js"></script> -->
 <!-- -->
 <!-- <script type="text/javascript" src="../_static/doctools.js"></script> -->
 <!-- -->
 <!-- <script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> -->
 <!-- -->
 <link href="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet-icon.png" rel="icon" type="image/png"/>
 </link></link></head>
 <body role="document"><!-- Previous Navbar Layout
 <div class="navbar navbar-default navbar-fixed-top">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
         <span class="sr-only">Toggle navigation</span>
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </button>
       <a href="../" class="navbar-brand">
         <img src="http://data.mxnet.io/theme/mxnet.png">
       </a>
     </div>
     <div id="navbar" class="navbar-collapse collapse">
       <ul id="navbar" class="navbar navbar-left">

         <li> <a href="../get_started/index.html">Get Started</a> </li>

         <li> <a href="../tutorials/index.html">Tutorials</a> </li>

         <li> <a href="../how_to/index.html">How To</a> </li>


         <li class="dropdown">
           <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">Packages <span class="caret"></span></a>
           <ul class="dropdown-menu">

             <li><a href="../packages/python/index.html">
                 Python
             </a></li>

             <li><a href="../packages/r/index.html">
                 R
             </a></li>

             <li><a href="../packages/julia/index.html">
                 Julia
             </a></li>

             <li><a href="../packages/c++/index.html">
                 C++
             </a></li>

             <li><a href="../packages/scala/index.html">
                 Scala
             </a></li>

             <li><a href="../packages/perl/index.html">
                 Perl
             </a></li>

           </ul>
         </li>

         <li> <a href="../system/index.html">System</a> </li>
         <li>
 <form class="" role="search" action="../search.html" method="get" autocomplete="off">
   <div class="form-group inner-addon left-addon">
     <i class="glyphicon glyphicon-search"></i>
     <input type="text" name="q" class="form-control" placeholder="Search">
   </div>
   <input type="hidden" name="check_keywords" value="yes" />
   <input type="hidden" name="area" value="default" />

 </form> </li>
       </ul>
       <ul id="navbar" class="navbar navbar-right">
         <li> <a href="../index.html"><span class="flag-icon flag-icon-us"></span></a> </li>
         <li> <a href="..//zh/index.html"><span class="flag-icon flag-icon-cn"></span></a> </li>
       </ul>
     </div>
   </div>
 </div>
 Previous Navbar Layout End -->
 <div class="navbar navbar-fixed-top">
 <div class="container" id="navContainer">
 <div class="innder" id="header-inner">
 <h1 id="logo-wrap">
 <a href="../" id="logo"><img src="http://data.mxnet.io/theme/mxnet.png"/></a>
 </h1>
 <nav class="nav-bar" id="main-nav">
 <a class="main-nav-link" href="../get_started/install.html">Install</a>
 <a class="main-nav-link" href="../tutorials/index.html">Tutorials</a>
 <a class="main-nav-link" href="../how_to/index.html">How To</a>
 <span id="dropdown-menu-position-anchor">
 <a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">API <span class="caret"></span></a>
 <ul class="dropdown-menu" id="package-dropdown-menu">
 <li><a class="main-nav-link" href="../api/python/index.html">Python</a></li>
 <li><a class="main-nav-link" href="../api/scala/index.html">Scala</a></li>
 <li><a class="main-nav-link" href="../api/r/index.html">R</a></li>
 <li><a class="main-nav-link" href="../api/julia/index.html">Julia</a></li>
 <li><a class="main-nav-link" href="../api/c++/index.html">C++</a></li>
 <li><a class="main-nav-link" href="../api/perl/index.html">Perl</a></li>
 </ul>
 </span>
 <a class="main-nav-link" href="../architecture/index.html">Architecture</a>
 <!-- <a class="main-nav-link" href="../community/index.html">Community</a> -->
 <a class="main-nav-link" href="https://github.com/dmlc/mxnet">Github</a>
 <span id="dropdown-menu-position-anchor-version" style="position: relative"><a href="#" class="main-nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">Versions(master)<span class="caret"></span></a><ul id="package-dropdown-menu" class="dropdown-menu"><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/>v0.10.14</a></li><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/versions/0.10/index.html>0.10</a></li><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/versions/master/index.html>master</a></li></ul></span></nav>
 <script> function getRootPath(){ return "../" } </script>
 <div class="burgerIcon dropdown">
 <a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button">☰</a>
 <ul class="dropdown-menu dropdown-menu-right" id="burgerMenu">
 <li><a href="../get_started/install.html">Install</a></li>
 <li><a href="../tutorials/index.html">Tutorials</a></li>
 <li><a href="../how_to/index.html">How To</a></li>
 <li class="dropdown-submenu">
 <a href="#" tabindex="-1">API</a>
 <ul class="dropdown-menu">
 <li><a href="../api/python/index.html" tabindex="-1">Python</a>
 </li>
 <li><a href="../api/scala/index.html" tabindex="-1">Scala</a>
 </li>
 <li><a href="../api/r/index.html" tabindex="-1">R</a>
 </li>
 <li><a href="../api/julia/index.html" tabindex="-1">Julia</a>
 </li>
 <li><a href="../api/c++/index.html" tabindex="-1">C++</a>
 </li>
 <li><a href="../api/perl/index.html" tabindex="-1">Perl</a>
 </li>
 </ul>
 </li>
 <li><a href="../architecture/index.html">Architecture</a></li>
 <li><a class="main-nav-link" href="https://github.com/dmlc/mxnet">Github</a></li>
 <li id="dropdown-menu-position-anchor-version-mobile" class="dropdown-submenu" style="position: relative"><a href="#" tabindex="-1">Versions(master)</a><ul class="dropdown-menu"><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/>v0.10.14</a></li><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/versions/0.10/index.html>0.10</a></li><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/versions/master/index.html>master</a></li></ul></li></ul>
 </div>
 <div class="plusIcon dropdown">
 <a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button"><span aria-hidden="true" class="glyphicon glyphicon-plus"></span></a>
 <ul class="dropdown-menu dropdown-menu-right" id="plusMenu"></ul>
 </div>
 <div id="search-input-wrap">
 <form action="../search.html" autocomplete="off" class="" method="get" role="search">
 <div class="form-group inner-addon left-addon">
 <i class="glyphicon glyphicon-search"></i>
 <input class="form-control" name="q" placeholder="Search" type="text"/>
 </div>
 <input name="check_keywords" type="hidden" value="yes">
 <input name="area" type="hidden" value="default"/>
 </input></form>
 <div id="search-preview"></div>
 </div>
 <div id="searchIcon">
 <span aria-hidden="true" class="glyphicon glyphicon-search"></span>
 </div>
 <!-- <div id="lang-select-wrap"> -->
 <!--   <label id="lang-select-label"> -->
 <!--     <\!-- <i class="fa fa-globe"></i> -\-> -->
 <!--     <span></span> -->
 <!--   </label> -->
 <!--   <select id="lang-select"> -->
 <!--     <option value="en">Eng</option> -->
 <!--     <option value="zh">中文</option> -->
 <!--   </select> -->
 <!-- </div> -->
 <!--     <a id="mobile-nav-toggle">
         <span class="mobile-nav-toggle-bar"></span>
         <span class="mobile-nav-toggle-bar"></span>
         <span class="mobile-nav-toggle-bar"></span>
       </a> -->
 </div>
 </div>
 </div>
 <div class="container">
 <div class="row">
 <div aria-label="main navigation" class="sphinxsidebar leftsidebar" role="navigation">
 <div class="sphinxsidebarwrapper">
 <ul>
 <li class="toctree-l1"><a class="reference internal" href="../api/python/index.html">Python Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/r/index.html">R Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/julia/index.html">Julia Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/c++/index.html">C++ Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/scala/index.html">Scala Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/perl/index.html">Perl Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="index.html">HowTo Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../architecture/index.html">System Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../tutorials/index.html">Tutorials</a></li>
 </ul>
 </div>
 </div>
 <div class="content">
 <div class="section" id="training-with-multiple-gpus-using-model-parallelism">
 <span id="training-with-multiple-gpus-using-model-parallelism"></span><h1>Training with Multiple GPUs Using Model Parallelism<a class="headerlink" href="#training-with-multiple-gpus-using-model-parallelism" title="Permalink to this headline">¶</a></h1>
 <p>Training deep learning models can be resource intensive.
 Even with a powerful GPU, some models can take days or weeks to train.
 Large long short-term memory (LSTM) recurrent neural networks
 can be especially slow to train,
 with each layer, at each time step, requiring eight matrix multiplications.
 Fortunately, given cloud services like AWS,
 machine learning practitioners often  have access
 to multiple machines and multiple GPUs.
 One key strength of <em>MXNet</em> is its ability to leverage
 powerful heterogeneous hardware environments to achieve significant speedups.</p>
 <p>There are two primary ways that we can spread a workload across multiple devices.
 In a previous document, <a class="reference internal" href="multi_devices.html"><em>we addressed data parallelism</em></a>,
 an approach in which examples within a batch are divvied among the available devices.
 With data parallelism, each device stores a complete copy of the model.
 Here, we explore <em>model parallelism</em>, a different approach.
 Instead of splitting the batch among the devices, we partition the model itself.
 Most commonly, we achieve model parallelism by assigning the parameters (and computation)
 of different layers of the network to different devices.</p>
 <p>In particular, we will focus on LSTM recurrent networks.
 LSTMS are powerful sequence models, that have proven especially useful
 for <a class="reference external" href="https://arxiv.org/pdf/1409.0473.pdf">natural language translation</a>, <a class="reference external" href="https://arxiv.org/abs/1512.02595">speech recognition</a>,
 and working with <a class="reference external" href="https://arxiv.org/abs/1511.03677">time series data</a>.
 For a general high-level introduction to LSTMs,
 see the excellent <a class="reference external" href="http://colah.github.io/posts/2015-08-Understanding-LSTMs/">tutorial</a> by Christopher Olah. For a working example of LSTM training with model parallelism,
 see <a class="reference external" href="https://github.com/dmlc/mxnet/blob/master/example/model-parallel-lstm/lstm.py">example/model-parallelism-lstm/</a>.</p>
 <div class="section" id="model-parallelism-using-multiple-gpus-as-a-pipeline">
 <span id="model-parallelism-using-multiple-gpus-as-a-pipeline"></span><h2>Model Parallelism: Using Multiple GPUs As a Pipeline<a class="headerlink" href="#model-parallelism-using-multiple-gpus-as-a-pipeline" title="Permalink to this headline">¶</a></h2>
 <p>Model parallelism in deep learning was first proposed
 for the <em>extraordinarily large</em> convolutional layer in GoogleNet.
 From this implementation, we take the idea of placing each layer on a separate GPU.
 Using model parallelism in such a layer-wise fashion
 provides the benefit that no GPU has to maintain all of the model parameters in memory.</p>
 <p><img alt="screen shot 2016-05-06 at 10 13 16 pm" src="https://cloud.githubusercontent.com/assets/5545640/15089697/d6f4fca0-13d7-11e6-9331-7f94fcc7b4c6.png" width="517"/></p>
 <p>In the preceding figure, each LSTM layer is assigned to a different GPU.
 After GPU 1 finishes computing layer 1 for the first sentence, it passes its output to GPU 2.
 At the same time, GPU 1 fetches the next sentence and starts training.
 This differs significantly from data parallelism.
 Here, there is no contention to update the shared model at the end of each iteration,
 and most of the communication happens when passing intermediate results between GPUs.</p>
 <p>In the current implementation, the layers are defined in <a class="reference external" href="https://github.com/dmlc/mxnet/blob/master/example/model-parallel-lstm/lstm.py">lstm_unroll()</a>.</p>
 </div>
 <div class="section" id="workload-partitioning">
 <span id="workload-partitioning"></span><h2>Workload Partitioning<a class="headerlink" href="#workload-partitioning" title="Permalink to this headline">¶</a></h2>
 <p>Implementing model parallelism requires knowledge of the training task.
 Here are some general heuristics that we find useful:</p>
 <ul class="simple">
 <li>To minimize communication time, place neighboring layers on the same GPUs.</li>
 <li>Be careful to balance the workload between GPUs.</li>
 <li>Remember that different kinds of layers have different computation-memory properties.</li>
 </ul>
 <p><img alt="screen shot 2016-05-07 at 1 51 02 am" src="https://cloud.githubusercontent.com/assets/5545640/15090455/37a30ab0-13f6-11e6-863b-efe2b10ec2e6.png" width="449"/></p>
 <p>Let’s take a quick look at the two pipelines in the preceding diagram.
 They both have eight layers with a decoder and an encoder layer.
 Based on our first principle, it’s unwise to place all neighboring layers on separate GPUs.
 We also want to balance the workload across GPUs.
 Although the LSTM layers consume less memory than the decoder/encoder layers, they consume more computation time because of the dependency of the unrolled LSTM.
 Thus, the partition on the left will be faster than the one on the right
 because the workload is more evenly distributed.</p>
 <p>Currently, the layer partition is implemented in <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L187">lstm.py</a> and configured in <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L187">lstm_ptb.py</a> using the <code class="docutils literal"><span class="pre">group2ctx</span></code> option.</p>
 </div>
 <div class="section" id="apply-bucketing-to-model-parallelism">
 <span id="apply-bucketing-to-model-parallelism"></span><h2>Apply Bucketing to Model Parallelism<a class="headerlink" href="#apply-bucketing-to-model-parallelism" title="Permalink to this headline">¶</a></h2>
 <p>To achieve model parallelism while using bucketing,
 you need to unroll an LSTM model for each bucket
 to obtain an executor for each.
 For details about how the model is bound, see <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L154">lstm.py</a>.</p>
 <p>On the other hand, because model parallelism partitions the model/layers,
 the input data has to be transformed/transposed to the agreed shape.
 For more details, see <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L154">bucket_io</a>.</p>
 </div>
 </div>
 <div class="container">
 <div class="footer">
 <p> © 2015-2017 DMLC. All rights reserved. </p>
 </div>
 </div>
 </div>
 <div aria-label="main navigation" class="sphinxsidebar rightsidebar" role="navigation">
 <div class="sphinxsidebarwrapper">
 <h3><a href="../index.html">Table Of Contents</a></h3>
 <ul>
 <li><a class="reference internal" href="#">Training with Multiple GPUs Using Model Parallelism</a><ul>
 <li><a class="reference internal" href="#model-parallelism-using-multiple-gpus-as-a-pipeline">Model Parallelism: Using Multiple GPUs As a Pipeline</a></li>
 <li><a class="reference internal" href="#workload-partitioning">Workload Partitioning</a></li>
 <li><a class="reference internal" href="#apply-bucketing-to-model-parallelism">Apply Bucketing to Model Parallelism</a></li>
 </ul>
 </li>
 </ul>
 </div>
 </div>
 </div> <!-- pagename != index -->
 <script crossorigin="anonymous" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>
 <script src="../_static/js/sidebar.js" type="text/javascript"></script>
 <script src="../_static/js/search.js" type="text/javascript"></script>
 <script src="../_static/js/navbar.js" type="text/javascript"></script>
 <script src="../_static/js/clipboard.min.js" type="text/javascript"></script>
 <script src="../_static/js/copycode.js" type="text/javascript"></script>
 <script type="text/javascript">
         $('body').ready(function () {
             $('body').css('visibility', 'visible');
         });
     </script>
 </div></body>
 </html>
	<!DOCTYPE html>

	<html lang="en">
	<head>
	<meta charset="utf-8"/>
	<meta content="IE=edge" http-equiv="X-UA-Compatible"/>
	<meta content="width=device-width, initial-scale=1" name="viewport"/>
	<title>Training with Multiple GPUs Using Model Parallelism — mxnet documentation</title>
	<link crossorigin="anonymous" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" rel="stylesheet"/>
	<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css" rel="stylesheet"/>
	<link href="../_static/basic.css" rel="stylesheet" type="text/css">
	<link href="../_static/pygments.css" rel="stylesheet" type="text/css">
	<link href="../_static/mxnet.css" rel="stylesheet" type="text/css"/>
	<script type="text/javascript">
	var DOCUMENTATION_OPTIONS = {
	URL_ROOT: '../',
	VERSION: '',
	COLLAPSE_INDEX: false,
	FILE_SUFFIX: '.html',
	HAS_SOURCE: true,
	SOURCELINK_SUFFIX: ''
	};
	</script>
	<script src="../_static/jquery-1.11.1.js" type="text/javascript"></script>
	<script src="../_static/underscore.js" type="text/javascript"></script>
	<script src="../_static/searchtools_custom.js" type="text/javascript"></script>
	<script src="../_static/doctools.js" type="text/javascript"></script>
	<script src="../_static/selectlang.js" type="text/javascript"></script>
	<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" type="text/javascript"></script>
	<script type="text/javascript"> jQuery(function() { Search.loadIndex("/searchindex.js"); Search.init();}); </script>
	<script>
	(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]\|\|function(){
	(i[r].q=i[r].q\|\|[]).push(arguments)},i[r].l=1*new
	Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

	ga('create', 'UA-96378503-1', 'auto');
	ga('send', 'pageview');

	</script>
	<!-- -->
	<!-- <script type="text/javascript" src="../_static/jquery.js"></script> -->
	<!-- -->
	<!-- <script type="text/javascript" src="../_static/underscore.js"></script> -->
	<!-- -->
	<!-- <script type="text/javascript" src="../_static/doctools.js"></script> -->
	<!-- -->
	<!-- <script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> -->
	<!-- -->
	<link href="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet-icon.png" rel="icon" type="image/png"/>
	</link></link></head>
	<body role="document"><!-- Previous Navbar Layout
	<div class="navbar navbar-default navbar-fixed-top">
	<div class="container">
	<div class="navbar-header">
	<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
	<span class="sr-only">Toggle navigation</span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	</button>
	<a href="../" class="navbar-brand">
	<img src="http://data.mxnet.io/theme/mxnet.png">
	</a>
	</div>
	<div id="navbar" class="navbar-collapse collapse">
	<ul id="navbar" class="navbar navbar-left">

	<li> <a href="../get_started/index.html">Get Started</a> </li>

	<li> <a href="../tutorials/index.html">Tutorials</a> </li>

	<li> <a href="../how_to/index.html">How To</a> </li>


	<li class="dropdown">
	<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">Packages <span class="caret"></span></a>
	<ul class="dropdown-menu">

	<li><a href="../packages/python/index.html">
	Python
	</a></li>

	<li><a href="../packages/r/index.html">
	R
	</a></li>

	<li><a href="../packages/julia/index.html">
	Julia
	</a></li>

	<li><a href="../packages/c++/index.html">
	C++
	</a></li>

	<li><a href="../packages/scala/index.html">
	Scala
	</a></li>

	<li><a href="../packages/perl/index.html">
	Perl
	</a></li>

	</ul>
	</li>

	<li> <a href="../system/index.html">System</a> </li>
	<li>
	<form class="" role="search" action="../search.html" method="get" autocomplete="off">
	<div class="form-group inner-addon left-addon">
	<i class="glyphicon glyphicon-search"></i>
	<input type="text" name="q" class="form-control" placeholder="Search">
	</div>
	<input type="hidden" name="check_keywords" value="yes" />
	<input type="hidden" name="area" value="default" />

	</form> </li>
	</ul>
	<ul id="navbar" class="navbar navbar-right">
	<li> <a href="../index.html"><span class="flag-icon flag-icon-us"></span></a> </li>
	<li> <a href="..//zh/index.html"><span class="flag-icon flag-icon-cn"></span></a> </li>
	</ul>
	</div>
	</div>
	</div>
	Previous Navbar Layout End -->
	<div class="navbar navbar-fixed-top">
	<div class="container" id="navContainer">
	<div class="innder" id="header-inner">
	<h1 id="logo-wrap">
	<a href="../" id="logo"><img src="http://data.mxnet.io/theme/mxnet.png"/></a>
	</h1>
	<nav class="nav-bar" id="main-nav">
	<a class="main-nav-link" href="../get_started/install.html">Install</a>
	<a class="main-nav-link" href="../tutorials/index.html">Tutorials</a>
	<a class="main-nav-link" href="../how_to/index.html">How To</a>
	<span id="dropdown-menu-position-anchor">
	<a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">API <span class="caret"></span></a>
	<ul class="dropdown-menu" id="package-dropdown-menu">
	<li><a class="main-nav-link" href="../api/python/index.html">Python</a></li>
	<li><a class="main-nav-link" href="../api/scala/index.html">Scala</a></li>
	<li><a class="main-nav-link" href="../api/r/index.html">R</a></li>
	<li><a class="main-nav-link" href="../api/julia/index.html">Julia</a></li>
	<li><a class="main-nav-link" href="../api/c++/index.html">C++</a></li>
	<li><a class="main-nav-link" href="../api/perl/index.html">Perl</a></li>
	</ul>
	</span>
	<a class="main-nav-link" href="../architecture/index.html">Architecture</a>
	<!-- <a class="main-nav-link" href="../community/index.html">Community</a> -->
	<a class="main-nav-link" href="https://github.com/dmlc/mxnet">Github</a>
	<span id="dropdown-menu-position-anchor-version" style="position: relative"><a href="#" class="main-nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">Versions(master)<span class="caret"></span></a><ul id="package-dropdown-menu" class="dropdown-menu"><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/>v0.10.14</a></li><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/versions/0.10/index.html>0.10</a></li><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/versions/master/index.html>master</a></li></ul></span></nav>
	<script> function getRootPath(){ return "../" } </script>
	<div class="burgerIcon dropdown">
	<a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button">☰</a>
	<ul class="dropdown-menu dropdown-menu-right" id="burgerMenu">
	<li><a href="../get_started/install.html">Install</a></li>
	<li><a href="../tutorials/index.html">Tutorials</a></li>
	<li><a href="../how_to/index.html">How To</a></li>
	<li class="dropdown-submenu">
	<a href="#" tabindex="-1">API</a>
	<ul class="dropdown-menu">
	<li><a href="../api/python/index.html" tabindex="-1">Python</a>
	</li>
	<li><a href="../api/scala/index.html" tabindex="-1">Scala</a>
	</li>
	<li><a href="../api/r/index.html" tabindex="-1">R</a>
	</li>
	<li><a href="../api/julia/index.html" tabindex="-1">Julia</a>
	</li>
	<li><a href="../api/c++/index.html" tabindex="-1">C++</a>
	</li>
	<li><a href="../api/perl/index.html" tabindex="-1">Perl</a>
	</li>
	</ul>
	</li>
	<li><a href="../architecture/index.html">Architecture</a></li>
	<li><a class="main-nav-link" href="https://github.com/dmlc/mxnet">Github</a></li>
	<li id="dropdown-menu-position-anchor-version-mobile" class="dropdown-submenu" style="position: relative"><a href="#" tabindex="-1">Versions(master)</a><ul class="dropdown-menu"><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/>v0.10.14</a></li><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/versions/0.10/index.html>0.10</a></li><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/versions/master/index.html>master</a></li></ul></li></ul>
	</div>
	<div class="plusIcon dropdown">
	<a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button"><span aria-hidden="true" class="glyphicon glyphicon-plus"></span></a>
	<ul class="dropdown-menu dropdown-menu-right" id="plusMenu"></ul>
	</div>
	<div id="search-input-wrap">
	<form action="../search.html" autocomplete="off" class="" method="get" role="search">
	<div class="form-group inner-addon left-addon">
	<i class="glyphicon glyphicon-search"></i>
	<input class="form-control" name="q" placeholder="Search" type="text"/>
	</div>
	<input name="check_keywords" type="hidden" value="yes">
	<input name="area" type="hidden" value="default"/>
	</input></form>
	<div id="search-preview"></div>
	</div>
	<div id="searchIcon">
	<span aria-hidden="true" class="glyphicon glyphicon-search"></span>
	</div>
	<!-- <div id="lang-select-wrap"> -->
	<!-- <label id="lang-select-label"> -->
	<!-- <\!-- <i class="fa fa-globe"></i> -\-> -->
	<!-- <span></span> -->
	<!-- </label> -->
	<!-- <select id="lang-select"> -->
	<!-- <option value="en">Eng</option> -->
	<!-- <option value="zh">中文</option> -->
	<!-- </select> -->
	<!-- </div> -->
	<!-- <a id="mobile-nav-toggle">
	<span class="mobile-nav-toggle-bar"></span>
	<span class="mobile-nav-toggle-bar"></span>
	<span class="mobile-nav-toggle-bar"></span>
	</a> -->
	</div>
	</div>
	</div>
	<div class="container">
	<div class="row">
	<div aria-label="main navigation" class="sphinxsidebar leftsidebar" role="navigation">
	<div class="sphinxsidebarwrapper">
	<ul>
	<li class="toctree-l1"><a class="reference internal" href="../api/python/index.html">Python Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../api/r/index.html">R Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../api/julia/index.html">Julia Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../api/c++/index.html">C++ Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../api/scala/index.html">Scala Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../api/perl/index.html">Perl Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="index.html">HowTo Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../architecture/index.html">System Documents</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../tutorials/index.html">Tutorials</a></li>
	</ul>
	</div>
	</div>
	<div class="content">
	<div class="section" id="training-with-multiple-gpus-using-model-parallelism">
	<span id="training-with-multiple-gpus-using-model-parallelism"></span><h1>Training with Multiple GPUs Using Model Parallelism<a class="headerlink" href="#training-with-multiple-gpus-using-model-parallelism" title="Permalink to this headline">¶</a></h1>
	<p>Training deep learning models can be resource intensive.
	Even with a powerful GPU, some models can take days or weeks to train.
	Large long short-term memory (LSTM) recurrent neural networks
	can be especially slow to train,
	with each layer, at each time step, requiring eight matrix multiplications.
	Fortunately, given cloud services like AWS,
	machine learning practitioners often have access
	to multiple machines and multiple GPUs.
	One key strength of <em>MXNet</em> is its ability to leverage
	powerful heterogeneous hardware environments to achieve significant speedups.</p>
	<p>There are two primary ways that we can spread a workload across multiple devices.
	In a previous document, <a class="reference internal" href="multi_devices.html"><em>we addressed data parallelism</em></a>,
	an approach in which examples within a batch are divvied among the available devices.
	With data parallelism, each device stores a complete copy of the model.
	Here, we explore <em>model parallelism</em>, a different approach.
	Instead of splitting the batch among the devices, we partition the model itself.
	Most commonly, we achieve model parallelism by assigning the parameters (and computation)
	of different layers of the network to different devices.</p>
	<p>In particular, we will focus on LSTM recurrent networks.
	LSTMS are powerful sequence models, that have proven especially useful
	for <a class="reference external" href="https://arxiv.org/pdf/1409.0473.pdf">natural language translation</a>, <a class="reference external" href="https://arxiv.org/abs/1512.02595">speech recognition</a>,
	and working with <a class="reference external" href="https://arxiv.org/abs/1511.03677">time series data</a>.
	For a general high-level introduction to LSTMs,
	see the excellent <a class="reference external" href="http://colah.github.io/posts/2015-08-Understanding-LSTMs/">tutorial</a> by Christopher Olah. For a working example of LSTM training with model parallelism,
	see <a class="reference external" href="https://github.com/dmlc/mxnet/blob/master/example/model-parallel-lstm/lstm.py">example/model-parallelism-lstm/</a>.</p>
	<div class="section" id="model-parallelism-using-multiple-gpus-as-a-pipeline">
	<span id="model-parallelism-using-multiple-gpus-as-a-pipeline"></span><h2>Model Parallelism: Using Multiple GPUs As a Pipeline<a class="headerlink" href="#model-parallelism-using-multiple-gpus-as-a-pipeline" title="Permalink to this headline">¶</a></h2>
	<p>Model parallelism in deep learning was first proposed
	for the <em>extraordinarily large</em> convolutional layer in GoogleNet.
	From this implementation, we take the idea of placing each layer on a separate GPU.
	Using model parallelism in such a layer-wise fashion
	provides the benefit that no GPU has to maintain all of the model parameters in memory.</p>
	<p><img alt="screen shot 2016-05-06 at 10 13 16 pm" src="https://cloud.githubusercontent.com/assets/5545640/15089697/d6f4fca0-13d7-11e6-9331-7f94fcc7b4c6.png" width="517"/></p>
	<p>In the preceding figure, each LSTM layer is assigned to a different GPU.
	After GPU 1 finishes computing layer 1 for the first sentence, it passes its output to GPU 2.
	At the same time, GPU 1 fetches the next sentence and starts training.
	This differs significantly from data parallelism.
	Here, there is no contention to update the shared model at the end of each iteration,
	and most of the communication happens when passing intermediate results between GPUs.</p>
	<p>In the current implementation, the layers are defined in <a class="reference external" href="https://github.com/dmlc/mxnet/blob/master/example/model-parallel-lstm/lstm.py">lstm_unroll()</a>.</p>
	</div>
	<div class="section" id="workload-partitioning">
	<span id="workload-partitioning"></span><h2>Workload Partitioning<a class="headerlink" href="#workload-partitioning" title="Permalink to this headline">¶</a></h2>
	<p>Implementing model parallelism requires knowledge of the training task.
	Here are some general heuristics that we find useful:</p>
	<ul class="simple">
	<li>To minimize communication time, place neighboring layers on the same GPUs.</li>
	<li>Be careful to balance the workload between GPUs.</li>
	<li>Remember that different kinds of layers have different computation-memory properties.</li>
	</ul>
	<p><img alt="screen shot 2016-05-07 at 1 51 02 am" src="https://cloud.githubusercontent.com/assets/5545640/15090455/37a30ab0-13f6-11e6-863b-efe2b10ec2e6.png" width="449"/></p>
	<p>Let’s take a quick look at the two pipelines in the preceding diagram.
	They both have eight layers with a decoder and an encoder layer.
	Based on our first principle, it’s unwise to place all neighboring layers on separate GPUs.
	We also want to balance the workload across GPUs.
	Although the LSTM layers consume less memory than the decoder/encoder layers, they consume more computation time because of the dependency of the unrolled LSTM.
	Thus, the partition on the left will be faster than the one on the right
	because the workload is more evenly distributed.</p>
	<p>Currently, the layer partition is implemented in <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L187">lstm.py</a> and configured in <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L187">lstm_ptb.py</a> using the <code class="docutils literal"><span class="pre">group2ctx</span></code> option.</p>
	</div>
	<div class="section" id="apply-bucketing-to-model-parallelism">
	<span id="apply-bucketing-to-model-parallelism"></span><h2>Apply Bucketing to Model Parallelism<a class="headerlink" href="#apply-bucketing-to-model-parallelism" title="Permalink to this headline">¶</a></h2>
	<p>To achieve model parallelism while using bucketing,
	you need to unroll an LSTM model for each bucket
	to obtain an executor for each.
	For details about how the model is bound, see <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L154">lstm.py</a>.</p>
	<p>On the other hand, because model parallelism partitions the model/layers,
	the input data has to be transformed/transposed to the agreed shape.
	For more details, see <a class="reference external" href="https://github.com/eric-haibin-lin/mxnet/blob/master/example/model-parallel-lstm/lstm.py#L154">bucket_io</a>.</p>
	</div>
	</div>
	<div class="container">
	<div class="footer">
	<p> © 2015-2017 DMLC. All rights reserved. </p>
	</div>
	</div>
	</div>
	<div aria-label="main navigation" class="sphinxsidebar rightsidebar" role="navigation">
	<div class="sphinxsidebarwrapper">
	<h3><a href="../index.html">Table Of Contents</a></h3>
	<ul>
	<li><a class="reference internal" href="#">Training with Multiple GPUs Using Model Parallelism</a><ul>
	<li><a class="reference internal" href="#model-parallelism-using-multiple-gpus-as-a-pipeline">Model Parallelism: Using Multiple GPUs As a Pipeline</a></li>
	<li><a class="reference internal" href="#workload-partitioning">Workload Partitioning</a></li>
	<li><a class="reference internal" href="#apply-bucketing-to-model-parallelism">Apply Bucketing to Model Parallelism</a></li>
	</ul>
	</li>
	</ul>
	</div>
	</div>
	</div> <!-- pagename != index -->
	<script crossorigin="anonymous" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>
	<script src="../_static/js/sidebar.js" type="text/javascript"></script>
	<script src="../_static/js/search.js" type="text/javascript"></script>
	<script src="../_static/js/navbar.js" type="text/javascript"></script>
	<script src="../_static/js/clipboard.min.js" type="text/javascript"></script>
	<script src="../_static/js/copycode.js" type="text/javascript"></script>
	<script type="text/javascript">
	$('body').ready(function () {
	$('body').css('visibility', 'visible');
	});
	</script>
	</div></body>
	</html>