blob: 35f4f89d08b5defcc53d612c863afb060cdbaa1b [file]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta content="IE=edge" http-equiv="X-UA-Compatible"/>
<meta content="width=device-width, initial-scale=1" name="viewport"/>
<meta content="SVRG Optimization in Python Module API" property="og:title">
<meta content="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/og-logo.png" property="og:image">
<meta content="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/og-logo.png" property="og:image:secure_url">
<meta content="SVRG Optimization in Python Module API" property="og:description"/>
<title>SVRG Optimization in Python Module API — mxnet documentation</title>
<link crossorigin="anonymous" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" rel="stylesheet"/>
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css" rel="stylesheet"/>
<link href="../../../_static/basic.css" rel="stylesheet" type="text/css">
<link href="../../../_static/pygments.css" rel="stylesheet" type="text/css">
<link href="../../../_static/mxnet.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../../../',
VERSION: '',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script src="https://code.jquery.com/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="../../../_static/underscore.js" type="text/javascript"></script>
<script src="../../../_static/searchtools_custom.js" type="text/javascript"></script>
<script src="../../../_static/doctools.js" type="text/javascript"></script>
<script src="../../../_static/selectlang.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML" type="text/javascript"></script>
<script type="text/javascript"> jQuery(function() { Search.loadIndex("/versions/1.5.0/searchindex.js"); Search.init();}); </script>
<!-- -->
<!-- <script type="text/javascript" src="../../../_static/jquery.js"></script> -->
<!-- -->
<!-- <script type="text/javascript" src="../../../_static/underscore.js"></script> -->
<!-- -->
<!-- <script type="text/javascript" src="../../../_static/doctools.js"></script> -->
<!-- -->
<!-- <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> -->
<!-- -->
<link href="../../../genindex.html" rel="index" title="Index">
<link href="../../../search.html" rel="search" title="Search"/>
<link href="../index.html" rel="up" title="MXNet - Python API"/>
<link href="../gluon/gluon.html" rel="next" title="Gluon Package"/>
<link href="../../../tutorials/onnx/fine_tuning_gluon.html" rel="prev" title="Fine-tuning an ONNX model with MXNet/Gluon"/>
<link href="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet-icon.png" rel="icon" type="image/png"/>
</link></link></link></meta></meta></meta></head>
<body background="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet-background-compressed.jpeg" role="document">
<div class="content-block"><div class="navbar navbar-fixed-top">
<div class="container" id="navContainer">
<div class="innder" id="header-inner">
<h1 id="logo-wrap">
<a href="../../../" id="logo"><img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet_logo.png"/></a>
</h1>
<nav class="nav-bar" id="main-nav">
<a class="main-nav-link" href="/versions/1.5.0/install/index.html">Install</a>
<span id="dropdown-menu-position-anchor">
<a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">Gluon <span class="caret"></span></a>
<ul class="dropdown-menu navbar-menu" id="package-dropdown-menu">
<li><a class="main-nav-link" href="/versions/1.5.0/tutorials/gluon/gluon.html">About</a></li>
<li><a class="main-nav-link" href="https://www.d2l.ai/">Dive into Deep Learning</a></li>
<li><a class="main-nav-link" href="https://gluon-cv.mxnet.io">GluonCV Toolkit</a></li>
<li><a class="main-nav-link" href="https://gluon-nlp.mxnet.io/">GluonNLP Toolkit</a></li>
</ul>
</span>
<span id="dropdown-menu-position-anchor">
<a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">API <span class="caret"></span></a>
<ul class="dropdown-menu navbar-menu" id="package-dropdown-menu">
<li><a class="main-nav-link" href="/versions/1.5.0/api/python/index.html">Python</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/c++/index.html">C++</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/clojure/index.html">Clojure</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/java/index.html">Java</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/julia/index.html">Julia</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/perl/index.html">Perl</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/r/index.html">R</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/scala/index.html">Scala</a></li>
</ul>
</span>
<span id="dropdown-menu-position-anchor-docs">
<a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">Docs <span class="caret"></span></a>
<ul class="dropdown-menu navbar-menu" id="package-dropdown-menu-docs">
<li><a class="main-nav-link" href="/versions/1.5.0/faq/index.html">FAQ</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/tutorials/index.html">Tutorials</a>
<li><a class="main-nav-link" href="https://github.com/apache/incubator-mxnet/tree/1.5.0/example">Examples</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/architecture/index.html">Architecture</a></li>
<li><a class="main-nav-link" href="https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home">Developer Wiki</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/model_zoo/index.html">Model Zoo</a></li>
<li><a class="main-nav-link" href="https://github.com/onnx/onnx-mxnet">ONNX</a></li>
</li></ul>
</span>
<span id="dropdown-menu-position-anchor-community">
<a aria-expanded="true" aria-haspopup="true" class="main-nav-link dropdown-toggle" data-toggle="dropdown" href="#" role="button">Community <span class="caret"></span></a>
<ul class="dropdown-menu navbar-menu" id="package-dropdown-menu-community">
<li><a class="main-nav-link" href="http://discuss.mxnet.io">Forum</a></li>
<li><a class="main-nav-link" href="https://github.com/apache/incubator-mxnet/tree/1.5.0">Github</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/community/contribute.html">Contribute</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/community/ecosystem.html">Ecosystem</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/community/powered_by.html">Powered By</a></li>
</ul>
</span>
<span id="dropdown-menu-position-anchor-version" style="position: relative"><a href="#" class="main-nav-link dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true">1.5.0<span class="caret"></span></a><ul id="package-dropdown-menu" class="dropdown-menu"><li><a href="/">master</a></li><li><a href="/versions/1.7.0/">1.7.0</a></li><li><a href=/versions/1.6.0/>1.6.0</a></li><li><a href=/versions/1.5.0/>1.5.0</a></li><li><a href=/versions/1.4.1/>1.4.1</a></li><li><a href=/versions/1.3.1/>1.3.1</a></li><li><a href=/versions/1.2.1/>1.2.1</a></li><li><a href=/versions/1.1.0/>1.1.0</a></li><li><a href=/versions/1.0.0/>1.0.0</a></li><li><a href=/versions/0.12.1/>0.12.1</a></li><li><a href=/versions/0.11.0/>0.11.0</a></li></ul></span></nav>
<script> function getRootPath(){ return "../../../" } </script>
<div class="burgerIcon dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button">☰</a>
<ul class="dropdown-menu" id="burgerMenu">
<li><a href="/versions/1.5.0/install/index.html">Install</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/tutorials/index.html">Tutorials</a></li>
<li class="dropdown-submenu dropdown">
<a aria-expanded="true" aria-haspopup="true" class="dropdown-toggle burger-link" data-toggle="dropdown" href="#" tabindex="-1">Gluon</a>
<ul class="dropdown-menu navbar-menu" id="package-dropdown-menu">
<li><a class="main-nav-link" href="/versions/1.5.0/tutorials/gluon/gluon.html">About</a></li>
<li><a class="main-nav-link" href="http://gluon.mxnet.io">The Straight Dope (Tutorials)</a></li>
<li><a class="main-nav-link" href="https://gluon-cv.mxnet.io">GluonCV Toolkit</a></li>
<li><a class="main-nav-link" href="https://gluon-nlp.mxnet.io/">GluonNLP Toolkit</a></li>
</ul>
</li>
<li class="dropdown-submenu">
<a aria-expanded="true" aria-haspopup="true" class="dropdown-toggle burger-link" data-toggle="dropdown" href="#" tabindex="-1">API</a>
<ul class="dropdown-menu">
<li><a class="main-nav-link" href="/versions/1.5.0/api/python/index.html">Python</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/c++/index.html">C++</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/clojure/index.html">Clojure</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/java/index.html">Java</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/julia/index.html">Julia</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/perl/index.html">Perl</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/r/index.html">R</a></li>
<li><a class="main-nav-link" href="/versions/1.5.0/api/scala/index.html">Scala</a></li>
</ul>
</li>
<li class="dropdown-submenu">
<a aria-expanded="true" aria-haspopup="true" class="dropdown-toggle burger-link" data-toggle="dropdown" href="#" tabindex="-1">Docs</a>
<ul class="dropdown-menu">
<li><a href="/versions/1.5.0/faq/index.html" tabindex="-1">FAQ</a></li>
<li><a href="/versions/1.5.0/tutorials/index.html" tabindex="-1">Tutorials</a></li>
<li><a href="https://github.com/apache/incubator-mxnet/tree/1.5.0/example" tabindex="-1">Examples</a></li>
<li><a href="/versions/1.5.0/architecture/index.html" tabindex="-1">Architecture</a></li>
<li><a href="https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home" tabindex="-1">Developer Wiki</a></li>
<li><a href="/versions/1.5.0/model_zoo/index.html" tabindex="-1">Gluon Model Zoo</a></li>
<li><a href="https://github.com/onnx/onnx-mxnet" tabindex="-1">ONNX</a></li>
</ul>
</li>
<li class="dropdown-submenu dropdown">
<a aria-haspopup="true" class="dropdown-toggle burger-link" data-toggle="dropdown" href="#" role="button" tabindex="-1">Community</a>
<ul class="dropdown-menu">
<li><a href="http://discuss.mxnet.io" tabindex="-1">Forum</a></li>
<li><a href="https://github.com/apache/incubator-mxnet/tree/1.5.0" tabindex="-1">Github</a></li>
<li><a href="/versions/1.5.0/community/contribute.html" tabindex="-1">Contribute</a></li>
<li><a href="/versions/1.5.0/community/ecosystem.html" tabindex="-1">Ecosystem</a></li>
<li><a href="/versions/1.5.0/community/powered_by.html" tabindex="-1">Powered By</a></li>
</ul>
</li>
<li id="dropdown-menu-position-anchor-version-mobile" class="dropdown-submenu" style="position: relative"><a href="#" tabindex="-1">1.5.0</a><ul class="dropdown-menu"><li><a tabindex="-1" href=/>master</a></li><li><a tabindex="-1" href=/versions/1.6.0/>1.6.0</a></li><li><a tabindex="-1" href=/versions/1.5.0/>1.5.0</a></li><li><a tabindex="-1" href=/versions/1.4.1/>1.4.1</a></li><li><a tabindex="-1" href=/versions/1.3.1/>1.3.1</a></li><li><a tabindex="-1" href=/versions/1.2.1/>1.2.1</a></li><li><a tabindex="-1" href=/versions/1.1.0/>1.1.0</a></li><li><a tabindex="-1" href=/versions/1.0.0/>1.0.0</a></li><li><a tabindex="-1" href=/versions/0.12.1/>0.12.1</a></li><li><a tabindex="-1" href=/versions/0.11.0/>0.11.0</a></li></ul></li></ul>
</div>
<div class="plusIcon dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#" role="button"><span aria-hidden="true" class="glyphicon glyphicon-plus"></span></a>
<ul class="dropdown-menu dropdown-menu-right" id="plusMenu"></ul>
</div>
<div id="search-input-wrap">
<form action="../../../search.html" autocomplete="off" class="" method="get" role="search">
<div class="form-group inner-addon left-addon">
<i class="glyphicon glyphicon-search"></i>
<input class="form-control" name="q" placeholder="Search" type="text"/>
</div>
<input name="check_keywords" type="hidden" value="yes">
<input name="area" type="hidden" value="default"/>
</input></form>
<div id="search-preview"></div>
</div>
<div id="searchIcon">
<span aria-hidden="true" class="glyphicon glyphicon-search"></span>
</div>
<!-- <div id="lang-select-wrap"> -->
<!-- <label id="lang-select-label"> -->
<!-- <\!-- <i class="fa fa-globe"></i> -\-> -->
<!-- <span></span> -->
<!-- </label> -->
<!-- <select id="lang-select"> -->
<!-- <option value="en">Eng</option> -->
<!-- <option value="zh">中文</option> -->
<!-- </select> -->
<!-- </div> -->
<!-- <a id="mobile-nav-toggle">
<span class="mobile-nav-toggle-bar"></span>
<span class="mobile-nav-toggle-bar"></span>
<span class="mobile-nav-toggle-bar"></span>
</a> -->
</div>
</div>
</div>
<script type="text/javascript">
$('body').css('background', 'white');
</script>
<div class="container">
<div class="row">
<div aria-label="main navigation" class="sphinxsidebar leftsidebar" role="navigation">
<div class="sphinxsidebarwrapper">
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../index.html">MXNet APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../architecture/index.html">MXNet Architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../community/index.html">MXNet Community</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq/index.html">MXNet FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../gluon/index.html">About Gluon</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../install/index.html">Installing MXNet</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../install/index.html#nvidia-jetson-tx-family">Nvidia Jetson TX family</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../install/index.html#source-download">Source Download</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../model_zoo/index.html">MXNet Model Zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorials/index.html">Tutorials</a></li>
</ul>
</div>
</div>
<div class="content">
<div class="page-tracker"></div>
<!--- Licensed to the Apache Software Foundation (ASF) under one -->
<!--- or more contributor license agreements. See the NOTICE file -->
<!--- distributed with this work for additional information -->
<!--- regarding copyright ownership. The ASF licenses this file -->
<!--- to you under the Apache License, Version 2.0 (the -->
<!--- "License"); you may not use this file except in compliance -->
<!--- with the License. You may obtain a copy of the License at --><!--- http://www.apache.org/licenses/LICENSE-2.0 --><!--- Unless required by applicable law or agreed to in writing, -->
<!--- software distributed under the License is distributed on an -->
<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
<!--- KIND, either express or implied. See the License for the -->
<!--- specific language governing permissions and limitations -->
<!--- under the License. --><div class="section" id="svrg-optimization-in-python-module-api">
<span id="svrg-optimization-in-python-module-api"></span><h1>SVRG Optimization in Python Module API<a class="headerlink" href="#svrg-optimization-in-python-module-api" title="Permalink to this headline"></a></h1>
<div class="section" id="overview">
<span id="overview"></span><h2>Overview<a class="headerlink" href="#overview" title="Permalink to this headline"></a></h2>
<p>SVRG which stands for Stochastic Variance Reduced Gradients, is an optimization technique that was first introduced in
paper <em>Accelerating Stochastic Gradient Descent using Predictive Variance Reduction</em> in 2013. It is complement to SGD
(Stochastic Gradient Descent), which is known for large scale optimization but suffers from slow convergence
asymptotically due to its inherent variance. SGD approximates the full gradients using a small batch of data or
a single data sample, which will introduce variance and thus requires to start with a small learning rate in order to
ensure convergence. SVRG remedies the problem by keeping track of a version of estimated weights that close to the
optimal parameter values and maintaining an average of full gradients over a full pass of data. The average of full
gradients is calculated with respect to the weights from the last m-th epochs in the training. SVRG uses a different
update rule: gradients w.r.t current parameter values minus gradients w.r.t to parameters from the last m-th epochs
plus the average of full gradients over all data.</p>
<p>Key Characteristics of SVRG:</p>
<ul class="simple">
<li>Employs explicit variance reduction by using a different update rule compared to SGD.</li>
<li>Ability to use relatively large learning rate, which leads to faster convergence compared to SGD.</li>
<li>Guarantees for fast convergence for smooth and strongly convex functions.</li>
</ul>
<p>SVRG optimization is implemented as a SVRGModule in <code class="docutils literal"><span class="pre">mxnet.contrib.svrg_optimization</span></code>, which is an extension of the
existing <code class="docutils literal"><span class="pre">mxnet.module.Module</span></code> APIs and encapsulates SVRG optimization logic within several new functions. SVRGModule
API changes compared to Module API to end users are minimal.</p>
<p>In distributed training, each worker gets the same special weights from the last m-th epoch and calculates the full
gradients with respect to its own shard of data. The standard SVRG optimization requires building a global full
gradients, which is calculated by aggregating the full gradients from each worker and averaging over the number of
workers. The workaround is to keep an additional set of keys in the KVStore that maps to full gradients.
The <code class="docutils literal"><span class="pre">_SVRGOptimizer</span></code> is designed to wrap two optimizers, an <code class="docutils literal"><span class="pre">_AssignmentOptimizer</span></code> which is used for full gradients
accumulation in the KVStore and a regular optimizer that performs actual update rule to the parameters.
The <code class="docutils literal"><span class="pre">_SVRGOptimizer</span></code> and <code class="docutils literal"><span class="pre">_AssignmentOptimizer</span></code> are designed to be used in <code class="docutils literal"><span class="pre">SVRGModule</span></code> only.</p>
<div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">This package contains experimental APIs and may change in the near future.</p>
</div>
<p>This document lists the SVRGModule APIs in MXNet/Contrib package:</p>
<table border="1" class="longtable docutils">
<colgroup>
<col width="10%"/>
<col width="90%"/>
</colgroup>
<tbody valign="top">
<tr class="row-odd"><td><a class="reference internal" href="#module-mxnet.contrib.svrg_optimization.svrg_module" title="mxnet.contrib.svrg_optimization.svrg_module"><code class="xref py py-obj docutils literal"><span class="pre">mxnet.contrib.svrg_optimization.svrg_module</span></code></a></td>
<td>A <cite>SVRGModule</cite> implements the <cite>Module</cite> API by wrapping an auxiliary module to perform SVRG optimization logic.</td>
</tr>
</tbody>
</table>
<div class="section" id="intermediate-level-api-for-svrgmodule">
<span id="intermediate-level-api-for-svrgmodule"></span><h3>Intermediate Level API for SVRGModule<a class="headerlink" href="#intermediate-level-api-for-svrgmodule" title="Permalink to this headline"></a></h3>
<p>The only extra step to use a SVRGModule compared to use a Module is to check if the current epoch should update the
full gradients over all data. Code snippets below demonstrate the suggested usage of SVRGModule using intermediate
level APIs.</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">mod</span> <span class="o">=</span> <span class="n">SVRGModule</span><span class="p">(</span><span class="n">symbol</span><span class="o">=</span><span class="n">model</span><span class="p">,</span> <span class="n">update_freq</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="p">[</span><span class="s1">'data'</span><span class="p">],</span> <span class="n">label_names</span><span class="o">=</span><span class="p">[</span><span class="s1">'lin_reg_label'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">mod</span><span class="o">.</span><span class="n">bind</span><span class="p">(</span><span class="n">data_shapes</span><span class="o">=</span><span class="n">di</span><span class="o">.</span><span class="n">provide_data</span><span class="p">,</span> <span class="n">label_shapes</span><span class="o">=</span><span class="n">di</span><span class="o">.</span><span class="n">provide_label</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">mod</span><span class="o">.</span><span class="n">init_params</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">mod</span><span class="o">.</span><span class="n">init_optimizer</span><span class="p">(</span><span class="n">optimizer</span><span class="o">=</span><span class="s1">'sgd'</span><span class="p">,</span> <span class="n">optimizer_params</span><span class="o">=</span><span class="p">((</span><span class="s1">'learning_rate'</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">),</span> <span class="p">),</span> <span class="n">kvstore</span><span class="o">=</span><span class="s1">'local'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_epochs</span><span class="p">):</span>
<span class="gp">... </span> <span class="k">if</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">mod</span><span class="o">.</span><span class="n">update_freq</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="gp">... </span> <span class="n">mod</span><span class="o">.</span><span class="n">update_full_grads</span><span class="p">(</span><span class="n">di</span><span class="p">)</span>
<span class="gp">... </span> <span class="n">di</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
<span class="gp">... </span> <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">di</span><span class="p">:</span>
<span class="gp">... </span> <span class="n">mod</span><span class="o">.</span><span class="n">forward_backward</span><span class="p">(</span><span class="n">data_batch</span><span class="o">=</span><span class="n">batch</span><span class="p">)</span>
<span class="gp">... </span> <span class="n">mod</span><span class="o">.</span><span class="n">update</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="high-level-api-for-svrgmodule">
<span id="high-level-api-for-svrgmodule"></span><h3>High Level API for SVRGModule<a class="headerlink" href="#high-level-api-for-svrgmodule" title="Permalink to this headline"></a></h3>
<p>The high level API usage of SVRGModule remains exactly the same as Module API. Code snippets below gives an example of
suggested usage of high level API.</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">mod</span> <span class="o">=</span> <span class="n">SVRGModule</span><span class="p">(</span><span class="n">symbol</span><span class="o">=</span><span class="n">model</span><span class="p">,</span> <span class="n">update_freq</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="p">[</span><span class="s1">'data'</span><span class="p">],</span> <span class="n">label_names</span><span class="o">=</span><span class="p">[</span><span class="s1">'lin_reg_label'</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">mod</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">di</span><span class="p">,</span> <span class="n">num_epochs</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s1">'sgd'</span><span class="p">,</span> <span class="n">optimizer_params</span><span class="o">=</span><span class="p">((</span><span class="s1">'learning_rate'</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">),</span> <span class="p">))</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="api-reference">
<span id="api-reference"></span><h2>API reference<a class="headerlink" href="#api-reference" title="Permalink to this headline"></a></h2>
<script src="../../../_static/js/auto_module_index.js" type="text/javascript"></script><span class="target" id="module-mxnet.contrib.svrg_optimization.svrg_module"></span><p>A <cite>SVRGModule</cite> implements the <cite>Module</cite> API by wrapping an auxiliary module to perform
SVRG optimization logic.</p>
<dl class="class">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule">
<em class="property">class </em><code class="descclassname">mxnet.contrib.svrg_optimization.svrg_module.</code><code class="descname">SVRGModule</code><span class="sig-paren">(</span><em>symbol</em>, <em>data_names=('data'</em>, <em>)</em>, <em>label_names=('softmax_label'</em>, <em>)</em>, <em>logger=<module 'logging' from '/usr/lib/python2.7/logging/__init__.pyc'></em>, <em>context=cpu(0)</em>, <em>work_load_list=None</em>, <em>fixed_param_names=None</em>, <em>state_names=None</em>, <em>group2ctxs=None</em>, <em>compression_params=None</em>, <em>update_freq=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule" title="Permalink to this definition"></a></dt>
<dd><p>SVRGModule is a module that encapsulates two Modules to accommodate the SVRG optimization technique.
It is functionally the same as Module API, except it is implemented using SVRG optimization logic.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>symbol</strong> (<a class="reference internal" href="../symbol/symbol.html#mxnet.symbol.Symbol" title="mxnet.symbol.Symbol"><em>Symbol</em></a>) – </li>
<li><strong>data_names</strong> (<em>list of str</em>) – Defaults to <cite>(‘data’)</cite> for a typical model used in image classification.</li>
<li><strong>label_names</strong> (<em>list of str</em>) – Defaults to <cite>(‘softmax_label’)</cite> for a typical model used in image classification.</li>
<li><strong>logger</strong> (<em>Logger</em>) – Defaults to <cite>logging</cite>.</li>
<li><strong>context</strong> (<em>Context</em><em> or </em><em>list of Context</em>) – Defaults to <code class="docutils literal"><span class="pre">mx.cpu()</span></code>.</li>
<li><strong>work_load_list</strong> (<em>list of number</em>) – Default <code class="docutils literal"><span class="pre">None</span></code>, indicating uniform workload.</li>
<li><strong>fixed_param_names</strong> (<em>list of str</em>) – Default <code class="docutils literal"><span class="pre">None</span></code>, indicating no network parameters are fixed.</li>
<li><strong>state_names</strong> (<em>list of str</em>) – states are similar to data and label, but not provided by data iterator. Instead they are initialized to 0 and can be set by <cite>set_states()</cite>.</li>
<li><strong>group2ctxs</strong> (<em>dict of str to context</em><em> or </em><em>list of context</em><em>, or </em><em>list of dict of str to context</em>) – Default is <cite>None</cite>. Mapping the <cite>ctx_group</cite> attribute to the context assignment.</li>
<li><strong>compression_params</strong> (<em>dict</em>) – Specifies type of gradient compression and additional arguments depending on the type of compression being used. For example, 2bit compression requires a threshold. Arguments would then be {‘type’:‘2bit’, ‘threshold’:0.5} See mxnet.KVStore.set_gradient_compression method for more details on gradient compression. update_freq: int
Specifies the number of times to update the full gradients to be used in the SVRG optimization. For instance, update_freq = 2 will calculates the gradients over all data every two epochs</li>
</ul>
</td>
</tr>
</tbody>
</table>
<p class="rubric">Examples</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="c1"># An example of declaring and using SVRGModule.</span>
<span class="gp">>>> </span><span class="n">mod</span> <span class="o">=</span> <span class="n">SVRGModule</span><span class="p">(</span><span class="n">symbol</span><span class="o">=</span><span class="n">lro</span><span class="p">,</span> <span class="n">data_names</span><span class="o">=</span><span class="p">[</span><span class="s1">'data'</span><span class="p">],</span> <span class="n">label_names</span><span class="o">=</span><span class="p">[</span><span class="s1">'lin_reg_label'</span><span class="p">],</span> <span class="n">update_freq</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">mod</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">di</span><span class="p">,</span> <span class="n">eval_metric</span><span class="o">=</span><span class="s1">'mse'</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s1">'sgd'</span><span class="p">,</span> <span class="n">optimizer_params</span><span class="o">=</span><span class="p">((</span><span class="s1">'learning_rate'</span><span class="p">,</span> <span class="mf">0.025</span><span class="p">),),</span>
<span class="gp">>>> </span> <span class="n">num_epoch</span><span class="o">=</span><span class="n">num_epoch</span><span class="p">,</span> <span class="n">kvstore</span><span class="o">=</span><span class="s1">'local'</span><span class="p">)</span>
</pre></div>
</div>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.reshape">
<code class="descname">reshape</code><span class="sig-paren">(</span><em>data_shapes</em>, <em>label_shapes=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.reshape"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.reshape" title="Permalink to this definition"></a></dt>
<dd><p>Reshapes both modules for new input shapes.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>data_shapes</strong> (<em>list of</em><em> (</em><em>str</em><em>, </em><em>tuple</em><em>)</em>) – Typically is <code class="docutils literal"><span class="pre">data_iter.provide_data</span></code>.</li>
<li><strong>label_shapes</strong> (<em>list of</em><em> (</em><em>str</em><em>, </em><em>tuple</em><em>)</em>) – Typically is <code class="docutils literal"><span class="pre">data_iter.provide_label</span></code>.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.init_optimizer">
<code class="descname">init_optimizer</code><span class="sig-paren">(</span><em>kvstore='local'</em>, <em>optimizer='sgd'</em>, <em>optimizer_params=(('learning_rate'</em>, <em>0.01)</em>, <em>)</em>, <em>force_init=False</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.init_optimizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.init_optimizer" title="Permalink to this definition"></a></dt>
<dd><p>Installs and initializes SVRGOptimizer. The SVRGOptimizer is a wrapper class for a regular optimizer that is
passed in and a special AssignmentOptimizer to accumulate the full gradients. If KVStore is ‘local’ or None,
the full gradients will be accumulated locally without pushing to the KVStore. Otherwise, additional keys will
be pushed to accumulate the full gradients in the KVStore.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>kvstore</strong> (<em>str</em><em> or </em><a class="reference internal" href="../kvstore/kvstore.html#mxnet.kvstore.KVStore" title="mxnet.kvstore.KVStore"><em>KVStore</em></a>) – Default <cite>‘local’</cite>.</li>
<li><strong>optimizer</strong> (<em>str</em><em> or </em><a class="reference internal" href="../optimization/optimization.html#mxnet.optimizer.Optimizer" title="mxnet.optimizer.Optimizer"><em>Optimizer</em></a>) – Default <cite>‘sgd’</cite></li>
<li><strong>optimizer_params</strong> (<em>dict</em>) – Default <cite>((‘learning_rate’, 0.01),)</cite>. The default value is not a dictionary,
just to avoid pylint warning of dangerous default values.</li>
<li><strong>force_init</strong> (<em>bool</em>) – Default <code class="docutils literal"><span class="pre">False</span></code>, indicating whether we should force re-initializing the
optimizer in the case an optimizer is already installed.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.bind">
<code class="descname">bind</code><span class="sig-paren">(</span><em>data_shapes</em>, <em>label_shapes=None</em>, <em>for_training=True</em>, <em>inputs_need_grad=False</em>, <em>force_rebind=False</em>, <em>shared_module=None</em>, <em>grad_req='write'</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.bind"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.bind" title="Permalink to this definition"></a></dt>
<dd><p>Binds the symbols to construct executors for both two modules. This is necessary before one
can perform computation with the SVRGModule.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>data_shapes</strong> (<em>list of</em><em> (</em><em>str</em><em>, </em><em>tuple</em><em>)</em>) – Typically is <code class="docutils literal"><span class="pre">data_iter.provide_data</span></code>.</li>
<li><strong>label_shapes</strong> (<em>list of</em><em> (</em><em>str</em><em>, </em><em>tuple</em><em>)</em>) – Typically is <code class="docutils literal"><span class="pre">data_iter.provide_label</span></code>.</li>
<li><strong>for_training</strong> (<em>bool</em>) – Default is <code class="docutils literal"><span class="pre">True</span></code>. Whether the executors should be bound for training.</li>
<li><strong>inputs_need_grad</strong> (<em>bool</em>) – Default is <code class="docutils literal"><span class="pre">False</span></code>. Whether the gradients to the input data need to be computed.
Typically this is not needed. But this might be needed when implementing composition
of modules.</li>
<li><strong>force_rebind</strong> (<em>bool</em>) – Default is <code class="docutils literal"><span class="pre">False</span></code>. This function does nothing if the executors are already
bound. But with this <code class="docutils literal"><span class="pre">True</span></code>, the executors will be forced to rebind.</li>
<li><strong>shared_module</strong> (<a class="reference internal" href="../module/module.html#mxnet.module.Module" title="mxnet.module.Module"><em>Module</em></a>) – Default is <code class="docutils literal"><span class="pre">None</span></code>. This is used in bucketing. When not <code class="docutils literal"><span class="pre">None</span></code>, the shared module
essentially corresponds to a different bucket – a module with different symbol
but with the same sets of parameters (e.g. unrolled RNNs with different lengths).</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.forward">
<code class="descname">forward</code><span class="sig-paren">(</span><em>data_batch</em>, <em>is_train=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.forward" title="Permalink to this definition"></a></dt>
<dd><p>Forward computation for both two modules. It supports data batches with different shapes, such as
different batch sizes or different image sizes.
If reshaping of data batch relates to modification of symbol or module, such as
changing image layout ordering or switching from training to predicting, module
rebinding is required.</p>
<div class="admonition seealso">
<p class="first admonition-title">See also</p>
<p class="last"><code class="xref py py-meth docutils literal"><span class="pre">BaseModule.forward()</span></code></p>
</div>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>data_batch</strong> (<em>DataBatch</em>) – Could be anything with similar API implemented.</li>
<li><strong>is_train</strong> (<em>bool</em>) – Default is <code class="docutils literal"><span class="pre">None</span></code>, which means <code class="docutils literal"><span class="pre">is_train</span></code> takes the value of <code class="docutils literal"><span class="pre">self.for_training</span></code>.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.backward">
<code class="descname">backward</code><span class="sig-paren">(</span><em>out_grads=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.backward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.backward" title="Permalink to this definition"></a></dt>
<dd><p>Backward computation.</p>
<div class="admonition seealso">
<p class="first admonition-title">See also</p>
<p class="last"><code class="xref py py-meth docutils literal"><span class="pre">BaseModule.backward()</span></code></p>
</div>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>out_grads</strong> (<a class="reference internal" href="../ndarray/ndarray.html#mxnet.ndarray.NDArray" title="mxnet.ndarray.NDArray"><em>NDArray</em></a><em> or </em><em>list of NDArray</em><em>, </em><em>optional</em>) – Gradient on the outputs to be propagated back.
This parameter is only needed when bind is called
on outputs that are not a loss function.</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.update">
<code class="descname">update</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.update"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.update" title="Permalink to this definition"></a></dt>
<dd><p>Updates parameters according to the installed optimizer and the gradients computed
in the previous forward-backward batch. The gradients in the _exec_group will be overwritten
using the gradients calculated by the SVRG update rule.</p>
<p>When KVStore is used to update parameters for multi-device or multi-machine training,
a copy of the parameters is stored in KVStore. Note that for <cite>row_sparse</cite> parameters,
this function does update the copy of parameters in KVStore, but doesn’t broadcast the
updated parameters to all devices / machines. Please call <cite>prepare</cite> to broadcast
<cite>row_sparse</cite> parameters with the next batch of data.</p>
<div class="admonition seealso">
<p class="first admonition-title">See also</p>
<p class="last"><code class="xref py py-meth docutils literal"><span class="pre">BaseModule.update()</span></code></p>
</div>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.update_full_grads">
<code class="descname">update_full_grads</code><span class="sig-paren">(</span><em>train_data</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.update_full_grads"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.update_full_grads" title="Permalink to this definition"></a></dt>
<dd><p>Computes the gradients over all data w.r.t weights of past
m epochs. For distributed env, it will accumulate full grads in the kvstore.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>train_data</strong> (<em>DataIter</em>) – Train data iterator</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.fit">
<code class="descname">fit</code><span class="sig-paren">(</span><em>train_data</em>, <em>eval_data=None</em>, <em>eval_metric='acc'</em>, <em>epoch_end_callback=None</em>, <em>batch_end_callback=None</em>, <em>kvstore='local'</em>, <em>optimizer='sgd'</em>, <em>optimizer_params=(('learning_rate'</em>, <em>0.01)</em>, <em>)</em>, <em>eval_end_callback=None</em>, <em>eval_batch_end_callback=None</em>, <em>initializer=<mxnet.initializer.Uniform object></em>, <em>arg_params=None</em>, <em>aux_params=None</em>, <em>allow_missing=False</em>, <em>force_rebind=False</em>, <em>force_init=False</em>, <em>begin_epoch=0</em>, <em>num_epoch=None</em>, <em>validation_metric=None</em>, <em>monitor=None</em>, <em>sparse_row_id_fn=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.fit"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.fit" title="Permalink to this definition"></a></dt>
<dd><p>Trains the module parameters.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>train_data</strong> (<em>DataIter</em>) – Train DataIter.</li>
<li><strong>eval_data</strong> (<em>DataIter</em>) – If not <code class="docutils literal"><span class="pre">None</span></code>, will be used as validation set and the performance
after each epoch will be evaluated.</li>
<li><strong>eval_metric</strong> (<em>str</em><em> or </em><a class="reference internal" href="../metric/metric.html#mxnet.metric.EvalMetric" title="mxnet.metric.EvalMetric"><em>EvalMetric</em></a>) – Defaults to ‘accuracy’. The performance measure used to display during training.
Other possible predefined metrics are:
‘ce’ (CrossEntropy), ‘f1’, ‘mae’, ‘mse’, ‘rmse’, ‘top_k_accuracy’.</li>
<li><strong>epoch_end_callback</strong> (<em>function</em><em> or </em><em>list of functions</em>) – Each callback will be called with the current <cite>epoch</cite>, <cite>symbol</cite>, <cite>arg_params</cite>
and <cite>aux_params</cite>.</li>
<li><strong>batch_end_callback</strong> (<em>function</em><em> or </em><em>list of function</em>) – Each callback will be called with a <cite>BatchEndParam</cite>.</li>
<li><strong>kvstore</strong> (<em>str</em><em> or </em><a class="reference internal" href="../kvstore/kvstore.html#mxnet.kvstore.KVStore" title="mxnet.kvstore.KVStore"><em>KVStore</em></a>) – Defaults to ‘local’.</li>
<li><strong>optimizer</strong> (<em>str</em><em> or </em><a class="reference internal" href="../optimization/optimization.html#mxnet.optimizer.Optimizer" title="mxnet.optimizer.Optimizer"><em>Optimizer</em></a>) – Defaults to ‘sgd’.</li>
<li><strong>optimizer_params</strong> (<em>dict</em>) – Defaults to <code class="docutils literal"><span class="pre">(('learning_rate',</span> <span class="pre">0.01),)</span></code>. The parameters for
the optimizer constructor.
The default value is not a dict, just to avoid pylint warning on dangerous
default values.</li>
<li><strong>eval_end_callback</strong> (<em>function</em><em> or </em><em>list of function</em>) – These will be called at the end of each full evaluation, with the metrics over
the entire evaluation set.</li>
<li><strong>eval_batch_end_callback</strong> (<em>function</em><em> or </em><em>list of function</em>) – These will be called at the end of each mini-batch during evaluation.</li>
<li><strong>initializer</strong> (<a class="reference internal" href="../optimization/optimization.html#mxnet.initializer.Initializer" title="mxnet.initializer.Initializer"><em>Initializer</em></a>) – The initializer is called to initialize the module parameters when they are
not already initialized.</li>
<li><strong>arg_params</strong> (<em>dict</em>) – Defaults to <code class="docutils literal"><span class="pre">None</span></code>, if not <code class="docutils literal"><span class="pre">None</span></code>, should be existing parameters from a trained
model or loaded from a checkpoint (previously saved model). In this case,
the value here will be used to initialize the module parameters, unless they
are already initialized by the user via a call to <cite>init_params</cite> or <cite>fit</cite>.
<cite>arg_params</cite> has a higher priority than <cite>initializer</cite>.</li>
<li><strong>aux_params</strong> (<em>dict</em>) – Defaults to <code class="docutils literal"><span class="pre">None</span></code>. Similar to <cite>arg_params</cite>, except for auxiliary states.</li>
<li><strong>allow_missing</strong> (<em>bool</em>) – Defaults to <code class="docutils literal"><span class="pre">False</span></code>. Indicates whether to allow missing parameters when <cite>arg_params</cite>
and <cite>aux_params</cite> are not <code class="docutils literal"><span class="pre">None</span></code>. If this is <code class="docutils literal"><span class="pre">True</span></code>, then the missing parameters
will be initialized via the <cite>initializer</cite>.</li>
<li><strong>force_rebind</strong> (<em>bool</em>) – Defaults to <code class="docutils literal"><span class="pre">False</span></code>. Whether to force rebinding the executors if already bound.</li>
<li><strong>force_init</strong> (<em>bool</em>) – Defaults to <code class="docutils literal"><span class="pre">False</span></code>. Indicates whether to force initialization even if the
parameters are already initialized.</li>
<li><strong>begin_epoch</strong> (<em>int</em>) – Defaults to 0. Indicates the starting epoch. Usually, if resumed from a
checkpoint saved at a previous training phase at epoch N, then this value should be
N+1.</li>
<li><strong>num_epoch</strong> (<em>int</em>) – Number of epochs for training.</li>
<li><strong>sparse_row_id_fn</strong> (<em>A callback function</em>) – The function takes <cite>data_batch</cite> as an input and returns a dict of
str -> NDArray. The resulting dict is used for pulling row_sparse
parameters from the kvstore, where the str key is the name of the param,
and the value is the row id of the param to pull.</li>
<li><strong>validation_metric</strong> (<em>str</em><em> or </em><a class="reference internal" href="../metric/metric.html#mxnet.metric.EvalMetric" title="mxnet.metric.EvalMetric"><em>EvalMetric</em></a>) – The performance measure used to display during validation.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.prepare">
<code class="descname">prepare</code><span class="sig-paren">(</span><em>data_batch</em>, <em>sparse_row_id_fn=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/mxnet/contrib/svrg_optimization/svrg_module.html#SVRGModule.prepare"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mxnet.contrib.svrg_optimization.svrg_module.SVRGModule.prepare" title="Permalink to this definition"></a></dt>
<dd><p>Prepares two modules for processing a data batch.</p>
<p>Usually involves switching bucket and reshaping.
For modules that contain <cite>row_sparse</cite> parameters in KVStore,
it prepares the <cite>row_sparse</cite> parameters based on the sparse_row_id_fn.</p>
<p>When KVStore is used to update parameters for multi-device or multi-machine training,
a copy of the parameters are stored in KVStore. Note that for <cite>row_sparse</cite> parameters,
the <cite>update()</cite> updates the copy of parameters in KVStore, but doesn’t broadcast
the updated parameters to all devices / machines. The <cite>prepare</cite> function is used to
broadcast <cite>row_sparse</cite> parameters with the next batch of data.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name"/>
<col class="field-body"/>
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>data_batch</strong> (<em>DataBatch</em>) – The current batch of data for forward computation.</li>
<li><strong>sparse_row_id_fn</strong> (<em>A callback function</em>) – The function takes <cite>data_batch</cite> as an input and returns a dict of
str -> NDArray. The resulting dict is used for pulling row_sparse
parameters from the kvstore, where the str key is the name of the param,
and the value is the row id of the param to pull.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
</dd></dl>
<script>auto_index("api-reference");</script></div>
</div>
</div>
</div>
<div aria-label="main navigation" class="sphinxsidebar rightsidebar" role="navigation">
<div class="sphinxsidebarwrapper">
<h3><a href="../../../index.html">Table Of Contents</a></h3>
<ul>
<li><a class="reference internal" href="#">SVRG Optimization in Python Module API</a><ul>
<li><a class="reference internal" href="#overview">Overview</a><ul>
<li><a class="reference internal" href="#intermediate-level-api-for-svrgmodule">Intermediate Level API for SVRGModule</a></li>
<li><a class="reference internal" href="#high-level-api-for-svrgmodule">High Level API for SVRGModule</a></li>
</ul>
</li>
<li><a class="reference internal" href="#api-reference">API reference</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div><div class="footer">
<div class="section-disclaimer">
<div class="container">
<div>
<img height="60" src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/apache_incubator_logo.png"/>
<p>
Apache MXNet is an effort undergoing incubation at The Apache Software Foundation (ASF), <strong>sponsored by the <i>Apache Incubator</i></strong>. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
</p>
<p>
"Copyright © 2017-2018, The Apache Software Foundation
Apache MXNet, MXNet, Apache, the Apache feather, and the Apache MXNet project logo are either registered trademarks or trademarks of the Apache Software Foundation."
</p>
</div>
</div>
</div>
</div> <!-- pagename != index -->
</div>
<script crossorigin="anonymous" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>
<script src="../../../_static/js/sidebar.js" type="text/javascript"></script>
<script src="../../../_static/js/search.js" type="text/javascript"></script>
<script src="../../../_static/js/navbar.js" type="text/javascript"></script>
<script src="../../../_static/js/clipboard.min.js" type="text/javascript"></script>
<script src="../../../_static/js/copycode.js" type="text/javascript"></script>
<script src="../../../_static/js/page.js" type="text/javascript"></script>
<script src="../../../_static/js/docversion.js" type="text/javascript"></script>
<script type="text/javascript">
$('body').ready(function () {
$('body').css('visibility', 'visible');
});
</script>
</body>
</html>