versions/master/architecture/rnn_interface.html - mxnet-test - Git at Google

 <!DOCTYPE html>

 <html lang="en">
 <head>
 <meta charset="utf-8"/>
 <meta content="IE=edge" http-equiv="X-UA-Compatible"/>
 <meta content="width=device-width, initial-scale=1" name="viewport"/>
 <title>Survey of Existing Interfaces and Implementations — mxnet  documentation</title>
 <link crossorigin="anonymous" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" rel="stylesheet"/>
 <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css" rel="stylesheet"/>
 <link href="../_static/basic.css" rel="stylesheet" type="text/css">
 <link href="../_static/pygments.css" rel="stylesheet" type="text/css">
 <link href="../_static/mxnet.css" rel="stylesheet" type="text/css"/>
 <script type="text/javascript">
       var DOCUMENTATION_OPTIONS = {
         URL_ROOT:    '../',
         VERSION:     '',
         COLLAPSE_INDEX: false,
         FILE_SUFFIX: '.html',
         HAS_SOURCE:  true,
         SOURCELINK_SUFFIX: ''
       };
     </script>
 <script src="../_static/jquery-1.11.1.js" type="text/javascript"></script>
 <script src="../_static/underscore.js" type="text/javascript"></script>
 <script src="../_static/searchtools_custom.js" type="text/javascript"></script>
 <script src="../_static/doctools.js" type="text/javascript"></script>
 <script src="../_static/selectlang.js" type="text/javascript"></script>
 <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML" type="text/javascript"></script>
 <script type="text/javascript"> jQuery(function() { Search.loadIndex("/searchindex.js"); Search.init();}); </script>
 <script>
       (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new
       Date();a=s.createElement(o),
       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
       })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

       ga('create', 'UA-96378503-1', 'auto');
       ga('send', 'pageview');

     </script>
 <!-- -->
 <!-- <script type="text/javascript" src="../_static/jquery.js"></script> -->
 <!-- -->
 <!-- <script type="text/javascript" src="../_static/underscore.js"></script> -->
 <!-- -->
 <!-- <script type="text/javascript" src="../_static/doctools.js"></script> -->
 <!-- -->
 <!-- <script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> -->
 <!-- -->
 <link href="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet-icon.png" rel="icon" type="image/png"/>
 </link></link></head>
 <body role="document"><!-- Previous Navbar Layout
 <div class="navbar navbar-default navbar-fixed-top">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
         <span class="sr-only">Toggle navigation</span>
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </button>
       <a href="../" class="navbar-brand">
         <img src="http://data.mxnet.io/theme/mxnet.png">
       </a>
     </div>
     <div id="navbar" class="navbar-collapse collapse">
       <ul id="navbar" class="navbar navbar-left">

         <li> <a href="../get_started/index.html">Get Started</a> </li>

         <li> <a href="../tutorials/index.html">Tutorials</a> </li>

         <li> <a href="../how_to/index.html">How To</a> </li>


         <li class="dropdown">
           <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">Packages <span class="caret"></span></a>
           <ul class="dropdown-menu">

             <li><a href="../packages/python/index.html">
                 Python
             </a></li>

             <li><a href="../packages/r/index.html">
                 R
             </a></li>

             <li><a href="../packages/julia/index.html">
                 Julia
             </a></li>

             <li><a href="../packages/c++/index.html">
                 C++
             </a></li>

             <li><a href="../packages/scala/index.html">
                 Scala
             </a></li>

             <li><a href="../packages/perl/index.html">
                 Perl
             </a></li>

           </ul>
         </li>

         <li> <a href="../system/index.html">System</a> </li>
         <li>
 <form class="" role="search" action="../search.html" method="get" autocomplete="off">
   <div class="form-group inner-addon left-addon">
     <i class="glyphicon glyphicon-search"></i>
     <input type="text" name="q" class="form-control" placeholder="Search">
   </div>
   <input type="hidden" name="check_keywords" value="yes" />
   <input type="hidden" name="area" value="default" />

 </form> </li>
       </ul>
       <ul id="navbar" class="navbar navbar-right">
         <li> <a href="../index.html"><span class="flag-icon flag-icon-us"></span></a> </li>
         <li> <a href="..//zh/index.html"><span class="flag-icon flag-icon-cn"></span></a> </li>
       </ul>
     </div>
   </div>
 </div>
 Previous Navbar Layout End -->
 <div class="navbar navbar-fixed-top">
 <div class="container" id="navContainer">
 <div class="innder" id="header-inner">
 <h1 id="logo-wrap">
 <a href="../" id="logo"><img src="http://data.mxnet.io/theme/mxnet.png"/></a>
 </h1>
 <nav class="nav-bar" id="main-nav">
 <a class="main-nav-link" href="../get_started/install.html">Install</a>
 <a class="main-nav-link" href="../tutorials/index.html">Tutorials</a>
 <a class="main-nav-link" href="../how_to/index.html">How To</a>
 <span id="dropdown-menu-position-anchor">
 <a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">API <span class="caret"></span></a>
 <ul class="dropdown-menu" id="package-dropdown-menu">
 <li><a class="main-nav-link" href="../api/python/index.html">Python</a></li>
 <li><a class="main-nav-link" href="../api/scala/index.html">Scala</a></li>
 <li><a class="main-nav-link" href="../api/r/index.html">R</a></li>
 <li><a class="main-nav-link" href="../api/julia/index.html">Julia</a></li>
 <li><a class="main-nav-link" href="../api/c++/index.html">C++</a></li>
 <li><a class="main-nav-link" href="../api/perl/index.html">Perl</a></li>
 </ul>
 </span>
 <a class="main-nav-link" href="../architecture/index.html">Architecture</a>
 <!-- <a class="main-nav-link" href="../community/index.html">Community</a> -->
 <a class="main-nav-link" href="https://github.com/dmlc/mxnet">Github</a>
 <span id="dropdown-menu-position-anchor-version" style="position: relative"><a href="#" class="main-nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">Versions(master)<span class="caret"></span></a><ul id="package-dropdown-menu" class="dropdown-menu"><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/>v0.10.14</a></li><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/versions/0.10/index.html>0.10</a></li><li><a class="main-nav-link" href=http://mxnet.incubator.apache.org/test/versions/master/index.html>master</a></li></ul></span></nav>
 <script> function getRootPath(){ return "../" } </script>
 <div class="burgerIcon dropdown">
 <a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button">☰</a>
 <ul class="dropdown-menu dropdown-menu-right" id="burgerMenu">
 <li><a href="../get_started/install.html">Install</a></li>
 <li><a href="../tutorials/index.html">Tutorials</a></li>
 <li><a href="../how_to/index.html">How To</a></li>
 <li class="dropdown-submenu">
 <a href="#" tabindex="-1">API</a>
 <ul class="dropdown-menu">
 <li><a href="../api/python/index.html" tabindex="-1">Python</a>
 </li>
 <li><a href="../api/scala/index.html" tabindex="-1">Scala</a>
 </li>
 <li><a href="../api/r/index.html" tabindex="-1">R</a>
 </li>
 <li><a href="../api/julia/index.html" tabindex="-1">Julia</a>
 </li>
 <li><a href="../api/c++/index.html" tabindex="-1">C++</a>
 </li>
 <li><a href="../api/perl/index.html" tabindex="-1">Perl</a>
 </li>
 </ul>
 </li>
 <li><a href="../architecture/index.html">Architecture</a></li>
 <li><a class="main-nav-link" href="https://github.com/dmlc/mxnet">Github</a></li>
 <li id="dropdown-menu-position-anchor-version-mobile" class="dropdown-submenu" style="position: relative"><a href="#" tabindex="-1">Versions(master)</a><ul class="dropdown-menu"><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/>v0.10.14</a></li><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/versions/0.10/index.html>0.10</a></li><li><a tabindex="-1" href=http://mxnet.incubator.apache.org/test/versions/master/index.html>master</a></li></ul></li></ul>
 </div>
 <div class="plusIcon dropdown">
 <a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button"><span aria-hidden="true" class="glyphicon glyphicon-plus"></span></a>
 <ul class="dropdown-menu dropdown-menu-right" id="plusMenu"></ul>
 </div>
 <div id="search-input-wrap">
 <form action="../search.html" autocomplete="off" class="" method="get" role="search">
 <div class="form-group inner-addon left-addon">
 <i class="glyphicon glyphicon-search"></i>
 <input class="form-control" name="q" placeholder="Search" type="text"/>
 </div>
 <input name="check_keywords" type="hidden" value="yes">
 <input name="area" type="hidden" value="default"/>
 </input></form>
 <div id="search-preview"></div>
 </div>
 <div id="searchIcon">
 <span aria-hidden="true" class="glyphicon glyphicon-search"></span>
 </div>
 <!-- <div id="lang-select-wrap"> -->
 <!--   <label id="lang-select-label"> -->
 <!--     <\!-- <i class="fa fa-globe"></i> -\-> -->
 <!--     <span></span> -->
 <!--   </label> -->
 <!--   <select id="lang-select"> -->
 <!--     <option value="en">Eng</option> -->
 <!--     <option value="zh">中文</option> -->
 <!--   </select> -->
 <!-- </div> -->
 <!--     <a id="mobile-nav-toggle">
         <span class="mobile-nav-toggle-bar"></span>
         <span class="mobile-nav-toggle-bar"></span>
         <span class="mobile-nav-toggle-bar"></span>
       </a> -->
 </div>
 </div>
 </div>
 <div class="container">
 <div class="row">
 <div aria-label="main navigation" class="sphinxsidebar leftsidebar" role="navigation">
 <div class="sphinxsidebarwrapper">
 <ul>
 <li class="toctree-l1"><a class="reference internal" href="../api/python/index.html">Python Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/r/index.html">R Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/julia/index.html">Julia Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/c++/index.html">C++ Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/scala/index.html">Scala Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api/perl/index.html">Perl Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../how_to/index.html">HowTo Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="index.html">System Documents</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../tutorials/index.html">Tutorials</a></li>
 </ul>
 </div>
 </div>
 <div class="content">
 <div class="section" id="survey-of-existing-interfaces-and-implementations">
 <span id="survey-of-existing-interfaces-and-implementations"></span><h1>Survey of Existing Interfaces and Implementations<a class="headerlink" href="#survey-of-existing-interfaces-and-implementations" title="Permalink to this headline">¶</a></h1>
 <p>Commonly used deep learning libraries with good RNN/LSTM support include <a class="reference external" href="http://deeplearning.net/software/theano/library/scan.html">Theano</a> and its wrappers <a class="reference external" href="http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html">Lasagne</a> and <a class="reference external" href="http://keras.io/layers/recurrent/">Keras</a>; <a class="reference external" href="https://cntk.codeplex.com/">CNTK</a>; <a class="reference external" href="https://www.tensorflow.org/versions/master/tutorials/recurrent/index.html">TensorFlow</a>; and various implementations in Torch, such as <a class="reference external" href="https://github.com/karpathy/char-rnn">this well-known character-level language model tutorial</a>, <a class="reference external" href="https://github.com/Element-Research/rnn">this</a>.</p>
 <p>In this document, we present a comparative analysis of the approaches taken by these libraries.</p>
 <div class="section" id="theano">
 <span id="theano"></span><h2>Theano<a class="headerlink" href="#theano" title="Permalink to this headline">¶</a></h2>
 <p>In Theano, RNN support comes via its <a class="reference external" href="http://deeplearning.net/software/theano/library/scan.html">scan operator</a>,
 which allows construction of a loop where the number of iterations is specified
 as a runtime value of a symbolic variable.
 You can find an official example of an LSTM implementation with scan
 <a class="reference external" href="http://deeplearning.net/tutorial/lstm.html">here</a>.</p>
 <div class="section" id="implementation">
 <span id="implementation"></span><h3>Implementation<a class="headerlink" href="#implementation" title="Permalink to this headline">¶</a></h3>
 <p>I’m not very familiar with the Theano internals,
 but it seems from <a class="reference external" href="https://github.com/Theano/Theano/blob/master/theano/scan_module/scan_op.py#L1225">theano/scan_module/scan_op.py#execute</a>
 that the scan operator is implemented with a loop in Python
 that performs one iteration at a time:</p>
 <div class="highlight-python"><div class="highlight"><pre><span></span>    <span class="n">fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fn</span><span class="o">.</span><span class="n">fn</span>

     <span class="k">while</span> <span class="p">(</span><span class="n">i</span> <span class="o"><</span> <span class="n">n_steps</span><span class="p">)</span> <span class="ow">and</span> <span class="n">cond</span><span class="p">:</span>
         <span class="c1"># ...</span>
         <span class="n">fn</span><span class="p">()</span>
 </pre></div>
 </div>
 <p>The <code class="docutils literal"><span class="pre">grad</span></code> function in Theano constructs a symbolic graph for computing gradients. So the <code class="docutils literal"><span class="pre">grad</span></code> for the scan operator is actually implemented by <a class="reference external" href="https://github.com/Theano/Theano/blob/master/theano/scan_module/scan_op.py#L2527">constructing another scan operator</a>:</p>
 <div class="highlight-python"><div class="highlight"><pre><span></span>    <span class="n">local_op</span> <span class="o">=</span> <span class="n">Scan</span><span class="p">(</span><span class="n">inner_gfn_ins</span><span class="p">,</span> <span class="n">inner_gfn_outs</span><span class="p">,</span> <span class="n">info</span><span class="p">)</span>
     <span class="n">outputs</span> <span class="o">=</span> <span class="n">local_op</span><span class="p">(</span><span class="o">*</span><span class="n">outer_inputs</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>The <a class="reference external" href="http://deeplearning.net/software/theano/library/scan.html#optimizing-scan-s-performance">performance guide</a> for Theano’s scan operator suggests minimizing the usage of the scan. This might be due to the fact that the loop is executed in Python, which might be a bit slow (due to context switching and the performance of Python itself). Moreover, because no unrolling is performed, the graph optimizer can’t see the big picture.</p>
 <p>If I understand correctly, when multiple RNN/LSTM layers are stacked, instead of a single loop with each iteration computing the whole feedforward network operation, the computation sequentially does a separate loop for each layer that uses the scan operator. If all of the intermediate values are stored to support computing the gradients, this is fine. Otherwise, using a single loop could be more memory efficient.</p>
 </div>
 <div class="section" id="lasagne">
 <span id="lasagne"></span><h3>Lasagne<a class="headerlink" href="#lasagne" title="Permalink to this headline">¶</a></h3>
 <p>The documentation for RNN in Lasagne can be found <a class="reference external" href="http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html">here</a>. In Lasagne, a recurrent layer is just like a standard layer, except that the input shape is expected to be <code class="docutils literal"><span class="pre">(batch_size,</span> <span class="pre">sequence_length,</span> <span class="pre">feature_dimension)</span></code>. The output shape is then <code class="docutils literal"><span class="pre">(batch_size,</span> <span class="pre">sequence_length,</span> <span class="pre">output_dimension)</span></code>.</p>
 <p>Both <code class="docutils literal"><span class="pre">batch_size</span></code> and <code class="docutils literal"><span class="pre">sequence_length</span></code> are specified as <code class="docutils literal"><span class="pre">None</span></code>, and inferred from the data. Alternatively, when memory is sufficient and the (maximum) sequence length is known beforehand, you can set <code class="docutils literal"><span class="pre">unroll_scan</span></code> to <code class="docutils literal"><span class="pre">False</span></code>. Then Lasagne will unroll the graph explicitly, instead of using the Theano <code class="docutils literal"><span class="pre">scan</span></code> operator. Explicitly unrolling is implemented in <a class="reference external" href="https://github.com/Lasagne/Lasagne/blob/master/lasagne/utils.py#L340">utils.py#unroll_scan</a>.</p>
 <p>The recurrent layer also accepts a <code class="docutils literal"><span class="pre">mask_input</span></code>, to support variable length sequences (e.g., when sequences within a mini-batch have different lengths. The mask has the shape <code class="docutils literal"><span class="pre">(batch_size,</span> <span class="pre">sequence_length)</span></code>.</p>
 </div>
 <div class="section" id="keras">
 <span id="keras"></span><h3>Keras<a class="headerlink" href="#keras" title="Permalink to this headline">¶</a></h3>
 <p>The documentation for RNN in Keras can be found <a class="reference external" href="http://keras.io/layers/recurrent/">here</a>. The interface in Keras is similar to the interface in Lasagne. The input is expected to be of shape <code class="docutils literal"><span class="pre">(batch_size,</span> <span class="pre">sequence_length,</span> <span class="pre">feature_dimension)</span></code>, and the output shape (if <code class="docutils literal"><span class="pre">return_sequences</span></code> is <code class="docutils literal"><span class="pre">True</span></code>) is <code class="docutils literal"><span class="pre">(batch_size,</span> <span class="pre">sequence_length,</span> <span class="pre">feature_dimension)</span></code>.</p>
 <p>Keras currently supports both a Theano and a TensorFlow back end. RNN for the Theano back end is <a class="reference external" href="https://github.com/fchollet/keras/blob/master/keras/backend/theano_backend.py#L432">implemented with the scan operator</a>. For TensorFlow, it seems to be <a class="reference external" href="https://github.com/fchollet/keras/blob/master/keras/backend/tensorflow_backend.py#L396">implemented via explicitly unrolling</a>. The documentation says that for the TensorFlow back end, the sequence length must be specified beforehand, and masking is currently not working (because <code class="docutils literal"><span class="pre">tf.reduce_any</span></code> is not functioning yet).</p>
 </div>
 </div>
 <div class="section" id="torch">
 <span id="torch"></span><h2>Torch<a class="headerlink" href="#torch" title="Permalink to this headline">¶</a></h2>
 <p><a class="reference external" href="https://github.com/karpathy/char-rnn">karpathy/char-rnn</a> is implemented by <a class="reference external" href="https://github.com/karpathy/char-rnn/blob/master/model/RNN.lua#L15">explicitly unrolling</a>. On the contrary, <a class="reference external" href="https://github.com/Element-Research/rnn">Element-Research/rnn</a> runs sequence iteration in Lua. It actually has a very modular design:</p>
 <ul class="simple">
 <li>The basic RNN/LSTM modules run only <em>one</em> time step per one call of <code class="docutils literal"><span class="pre">forward</span></code> (and accumulate/store necessary information to support backward computation, if needed). You could have detailed control when using this API directly.</li>
 <li>A collection of <code class="docutils literal"><span class="pre">Sequencer</span></code>s are defined to model common scenarios, like forwarding sequence, bi-directional sequence, attention models, etc.</li>
 <li>There are other utility modules, like masking to support variable length sequences, etc.</li>
 </ul>
 </div>
 <div class="section" id="cntk">
 <span id="cntk"></span><h2>CNTK<a class="headerlink" href="#cntk" title="Permalink to this headline">¶</a></h2>
 <p>CNTK looks quite different from other common deep learning libraries. I don’t understand it very well. I will talk with Yu to get more details.</p>
 <p>It seems that the basic data types are matrices (although there is also a <code class="docutils literal"><span class="pre">TensorView</span></code> utility class). The mini-batch data for sequence data is packed in a matrix with N-row being <code class="docutils literal"><span class="pre">feature_dimension</span></code> and N-column being <code class="docutils literal"><span class="pre">sequence_length</span> <span class="pre">*</span> <span class="pre">batch_size</span></code> (see Figure 2.9 on page 50 of the <a class="reference external" href="http://research.microsoft.com/pubs/226641/CNTKBook-20151201.pdf">CNTKBook</a>).</p>
 <p>Recurrent networks are first-class citizens in CNTK. In section 5.2.1.8 of the CNTKBook, you can see an example of a customized computation node. The node needs to explicitly define the functions for standard forward and forward with a time index, which is used for RNN evaluation:</p>
 <div class="highlight-cpp"><div class="highlight"><pre><span></span>    <span class="k">virtual</span> <span class="kt">void</span> <span class="nf">EvaluateThisNode</span><span class="p">()</span>
     <span class="p">{</span>
         <span class="n">EvaluateThisNodeS</span><span class="p">(</span><span class="n">FunctionValues</span><span class="p">(),</span> <span class="n">Inputs</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">-></span>
             <span class="n">FunctionValues</span><span class="p">(),</span> <span class="n">Inputs</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">-></span><span class="n">FunctionValues</span><span class="p">());</span>
     <span class="p">}</span>
     <span class="k">virtual</span> <span class="kt">void</span> <span class="nf">EvaluateThisNode</span><span class="p">(</span><span class="k">const</span> <span class="kt">size_t</span> <span class="n">timeIdxInSeq</span><span class="p">)</span>
     <span class="p">{</span>
         <span class="n">Matrix</span><span class="o"><</span><span class="n">ElemType</span><span class="o">></span> <span class="n">sliceInputValue</span> <span class="o">=</span> <span class="n">Inputs</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">-></span>
             <span class="n">FunctionValues</span><span class="p">().</span><span class="n">ColumnSlice</span><span class="p">(</span><span class="n">timeIdxInSeq</span> <span class="o">*</span>
             <span class="n">m_samplesInRecurrentStep</span><span class="p">,</span> <span class="n">m_samplesInRecurrentStep</span><span class="p">);</span>
         <span class="n">Matrix</span><span class="o"><</span><span class="n">ElemType</span><span class="o">></span> <span class="n">sliceOutputValue</span> <span class="o">=</span>    <span class="n">m_functionValues</span><span class="p">.</span>
             <span class="n">ColumnSlice</span><span class="p">(</span><span class="n">timeIdxInSeq</span> <span class="o">*</span> <span class="n">m_samplesInRecurrentStep</span><span class="p">,</span>
             <span class="n">m_samplesInRecurrentStep</span><span class="p">);</span>
         <span class="n">EvaluateThisNodeS</span><span class="p">(</span><span class="n">sliceOutputValue</span><span class="p">,</span> <span class="n">Inputs</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">-></span>
             <span class="n">FunctionValues</span><span class="p">(),</span> <span class="n">sliceInput1Value</span><span class="p">);</span>
     <span class="p">}</span>
 </pre></div>
 </div>
 <p>The function <code class="docutils literal"><span class="pre">ColumnSlice(start_col,</span> <span class="pre">num_col)</span></code> takes out the packed data for that time index, as described above (here <code class="docutils literal"><span class="pre">m_samplesInRecurrentStep</span></code> must be the mini-batch size).</p>
 <p>The low-level API for recurrent connection seem to be a <em>delay node</em>. But I’m not sure how to use this low-level API. The <a class="reference external" href="https://cntk.codeplex.com/SourceControl/latest#Examples/Text/PennTreebank/Config/rnn.config">example of PTB language model</a> uses a very high-level API (simply setting <code class="docutils literal"><span class="pre">recurrentLayer</span> <span class="pre">=</span> <span class="pre">1</span></code> in the config).</p>
 </div>
 <div class="section" id="tensorflow">
 <span id="tensorflow"></span><h2>TensorFlow<a class="headerlink" href="#tensorflow" title="Permalink to this headline">¶</a></h2>
 <p>The <a class="reference external" href="https://www.tensorflow.org/versions/master/tutorials/recurrent/index.html#recurrent-neural-networks">current example of RNNLM</a> in TensorFlow uses explicit unrolling for a predefined number of time steps. The white-paper mentions that an advanced control flow API (Theano’s scan-like) is planned.</p>
 </div>
 <div class="section" id="next-steps">
 <span id="next-steps"></span><h2>Next Steps<a class="headerlink" href="#next-steps" title="Permalink to this headline">¶</a></h2>
 <div class="toctree-wrapper compound">
 <ul>
 <li class="toctree-l1"><a class="reference external" href="http://mxnet.io/architecture/overview.html">MXNet System Overview</a></li>
 </ul>
 </div>
 </div>
 </div>
 <div class="container">
 <div class="footer">
 <p> © 2015-2017 DMLC. All rights reserved. </p>
 </div>
 </div>
 </div>
 <div aria-label="main navigation" class="sphinxsidebar rightsidebar" role="navigation">
 <div class="sphinxsidebarwrapper">
 <h3><a href="../index.html">Table Of Contents</a></h3>
 <ul>
 <li><a class="reference internal" href="#">Survey of Existing Interfaces and Implementations</a><ul>
 <li><a class="reference internal" href="#theano">Theano</a><ul>
 <li><a class="reference internal" href="#implementation">Implementation</a></li>
 <li><a class="reference internal" href="#lasagne">Lasagne</a></li>
 <li><a class="reference internal" href="#keras">Keras</a></li>
 </ul>
 </li>
 <li><a class="reference internal" href="#torch">Torch</a></li>
 <li><a class="reference internal" href="#cntk">CNTK</a></li>
 <li><a class="reference internal" href="#tensorflow">TensorFlow</a></li>
 <li><a class="reference internal" href="#next-steps">Next Steps</a></li>
 </ul>
 </li>
 </ul>
 </div>
 </div>
 </div> <!-- pagename != index -->
 <script crossorigin="anonymous" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>
 <script src="../_static/js/sidebar.js" type="text/javascript"></script>
 <script src="../_static/js/search.js" type="text/javascript"></script>
 <script src="../_static/js/navbar.js" type="text/javascript"></script>
 <script src="../_static/js/clipboard.min.js" type="text/javascript"></script>
 <script src="../_static/js/copycode.js" type="text/javascript"></script>
 <script type="text/javascript">
         $('body').ready(function () {
             $('body').css('visibility', 'visible');
         });
     </script>
 </div></body>
 </html>