versions/1.9.1/api/python/docs/tutorials/extend/custom_layer.html - mxnet-site - Git at Google

 <!DOCTYPE html>

 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
     <meta charset="utf-8" />
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     <meta http-equiv="x-ua-compatible" content="ie=edge">
     <style>
     .dropdown {
         position: relative;
         display: inline-block;
     }

     .dropdown-content {
         display: none;
         position: absolute;
         background-color: #f9f9f9;
         min-width: 160px;
         box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
         padding: 12px 16px;
         z-index: 1;
         text-align: left;
     }

     .dropdown:hover .dropdown-content {
         display: block;
     }

     .dropdown-option:hover {
         color: #FF4500;
     }

     .dropdown-option-active {
         color: #FF4500;
         font-weight: lighter;
     }

     .dropdown-option {
         color: #000000;
         font-weight: lighter;
     }

     .dropdown-header {
         color: #FFFFFF;
         display: inline-flex;
     }

     .dropdown-caret {
         width: 18px;
     }

     .dropdown-caret-path {
         fill: #FFFFFF;
     }
     </style>

     <title>Custom Layers &#8212; Apache MXNet  documentation</title>

     <link rel="stylesheet" href="../../_static/basic.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
     <link rel="stylesheet" type="text/css" href="../../_static/mxnet.css" />
     <link rel="stylesheet" href="../../_static/material-design-lite-1.3.0/material.blue-deep_orange.min.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/sphinx_materialdesign_theme.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/fontawesome/all.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/fonts.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/feedback.css" type="text/css" />
     <script id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
     <script src="../../_static/jquery.js"></script>
     <script src="../../_static/underscore.js"></script>
     <script src="../../_static/doctools.js"></script>
     <script src="../../_static/language_data.js"></script>
     <script src="../../_static/matomo_analytics.js"></script>
     <script src="../../_static/autodoc.js"></script>
     <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
     <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true, "ignoreClass": "document", "processClass": "math|output_area"}})</script>
     <link rel="shortcut icon" href="../../_static/mxnet-icon.png"/>
     <link rel="index" title="Index" href="../../genindex.html" />
     <link rel="search" title="Search" href="../../search.html" />
     <link rel="next" title="Custom Numpy Operators" href="customop.html" />
     <link rel="prev" title="Extend" href="index.html" />
   </head>
 <body><header class="site-header" role="banner">
   <div class="wrapper">
       <a class="site-title" rel="author" href="/versions/1.9.1/"><img
             src="../../_static/mxnet_logo.png" class="site-header-logo"></a>
     <nav class="site-nav">
       <input type="checkbox" id="nav-trigger" class="nav-trigger"/>
       <label for="nav-trigger">
           <span class="menu-icon">
             <svg viewBox="0 0 18 15" width="18px" height="15px">
               <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
             </svg>
           </span>
       </label>

       <div class="trigger">
         <a class="page-link" href="/versions/1.9.1/get_started">Get Started</a>
         <a class="page-link" href="/versions/1.9.1/features">Features</a>
         <a class="page-link" href="/versions/1.9.1/ecosystem">Ecosystem</a>
         <a class="page-link page-current" href="/versions/1.9.1/api">Docs & Tutorials</a>
         <a class="page-link" href="/versions/1.9.1/trusted_by">Trusted By</a>
         <a class="page-link" href="https://github.com/apache/mxnet">GitHub</a>
         <div class="dropdown" style="min-width:100px">
           <span class="dropdown-header">Apache
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content" style="min-width:250px">
             <a href="https://www.apache.org/foundation/">Apache Software Foundation</a>
             <a href="https://incubator.apache.org/">Apache Incubator</a>
             <a href="https://www.apache.org/licenses/">License</a>
             <a href="/versions/1.9.1/api/faq/security.html">Security</a>
             <a href="https://privacy.apache.org/policies/privacy-policy-public.html">Privacy</a>
             <a href="https://www.apache.org/events/current-event">Events</a>
             <a href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a>
             <a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
           </div>
         </div>
         <div class="dropdown">
           <span class="dropdown-header">1.9.1
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content">
             <a class="dropdown-option" href="/">master</a><br>
             <a class="dropdown-option-active" href="/versions/1.9.1/">1.9.1</a><br>
             <a class="dropdown-option" href="/versions/1.8.0/">1.8.0</a><br>
             <a class="dropdown-option" href="/versions/1.7.0/">1.7.0</a><br>
             <a class="dropdown-option" href="/versions/1.6.0/">1.6.0</a><br>
             <a class="dropdown-option" href="/versions/1.5.0/">1.5.0</a><br>
             <a class="dropdown-option" href="/versions/1.4.1/">1.4.1</a><br>
             <a class="dropdown-option" href="/versions/1.3.1/">1.3.1</a><br>
             <a class="dropdown-option" href="/versions/1.2.1/">1.2.1</a><br>
             <a class="dropdown-option" href="/versions/1.1.0/">1.1.0</a><br>
             <a class="dropdown-option" href="/versions/1.0.0/">1.0.0</a><br>
             <a class="dropdown-option" href="/versions/0.12.1/">0.12.1</a><br>
             <a class="dropdown-option" href="/versions/0.11.0/">0.11.0</a>
           </div>
         </div>
       </div>
     </nav>
   </div>
 </header>
     <div class="mdl-layout mdl-js-layout mdl-layout--fixed-header mdl-layout--fixed-drawer"><header class="mdl-layout__header mdl-layout__header--waterfall ">
     <div class="mdl-layout__header-row">

         <nav class="mdl-navigation breadcrumb">
             <a class="mdl-navigation__link" href="../index.html">Python Tutorials</a><i class="material-icons">navigate_next</i>
             <a class="mdl-navigation__link" href="index.html">Extend</a><i class="material-icons">navigate_next</i>
             <a class="mdl-navigation__link is-active">Custom Layers</a>
         </nav>
         <div class="mdl-layout-spacer"></div>
         <nav class="mdl-navigation">

 <form class="form-inline pull-sm-right" action="../../search.html" method="get">
       <div class="mdl-textfield mdl-js-textfield mdl-textfield--expandable mdl-textfield--floating-label mdl-textfield--align-right">
         <label id="quick-search-icon" class="mdl-button mdl-js-button mdl-button--icon"  for="waterfall-exp">
           <i class="material-icons">search</i>
         </label>
         <div class="mdl-textfield__expandable-holder">
           <input class="mdl-textfield__input" type="text" name="q"  id="waterfall-exp" placeholder="Search" />
           <input type="hidden" name="check_keywords" value="yes" />
           <input type="hidden" name="area" value="default" />
         </div>
       </div>
       <div class="mdl-tooltip" data-mdl-for="quick-search-icon">
       Quick search
       </div>
 </form>

 <a id="button-show-source"
     class="mdl-button mdl-js-button mdl-button--icon"
     href="../../_sources/tutorials/extend/custom_layer.ipynb" rel="nofollow">
   <i class="material-icons">code</i>
 </a>
 <div class="mdl-tooltip" data-mdl-for="button-show-source">
 Show Source
 </div>
         </nav>
     </div>
     <div class="mdl-layout__header-row header-links">
       <div class="mdl-layout-spacer"></div>
       <nav class="mdl-navigation">
       </nav>
     </div>
 </header><header class="mdl-layout__drawer">

       <div class="globaltoc">
         <span class="mdl-layout-title toc">Table Of Contents</span>


             <nav class="mdl-navigation">
                 <ul class="current">
 <li class="toctree-l1 current"><a class="reference internal" href="../index.html">Python Tutorials</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="../getting-started/index.html">Getting Started</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/crash-course/index.html">Crash Course</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/1-ndarray.html">Manipulate data with <code class="docutils literal notranslate"><span class="pre">ndarray</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/2-nn.html">Create a neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/3-autograd.html">Automatic differentiation with <code class="docutils literal notranslate"><span class="pre">autograd</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/4-train.html">Train the neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/5-predict.html">Predict with a pre-trained model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/6-use_gpus.html">Use GPUs</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/to-mxnet/index.html">Moving to MXNet from Other Frameworks</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/to-mxnet/pytorch.html">PyTorch vs Apache MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/gluon_from_experiment_to_deployment.html">Gluon: from experiment to deployment</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/logistic_regression_explained.html">Logistic regression explained</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/image/mnist.html">MNIST</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../packages/index.html">Packages</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../packages/autograd/index.html">Automatic Differentiation</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/gluon/index.html">Gluon</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/blocks/index.html">Blocks</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/custom-layer.html">Custom Layers</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/custom_layer_beginners.html">Customer Layers (Beginners)</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/hybridize.html">Hybridize</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/init.html">Initialization</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/naming.html">Parameter and Block Naming</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/nn.html">Layers and Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/parameters.html">Parameter Management</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/save_load_params.html">Saving and Loading Gluon Models</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/activations/activations.html">Activation Blocks</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/data/index.html">Data Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html#Spatial-Augmentation">Spatial Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html#Color-Augmentation">Color Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html#Composed-Augmentations">Composed Augmentations</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html">Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html#Using-own-data-with-included-Datasets">Using own data with included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html#Using-own-data-with-custom-Datasets">Using own data with custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html#Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader">Appendix: Upgrading from Module <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> to Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/image/index.html">Image Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/image-augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/info_gan.html">Image similarity search with InfoGAN</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/mnist.html">Handwritten Digit Recognition</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/pretrained_models.html">Using pre-trained models in MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/loss/index.html">Losses</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/loss/custom-loss.html">Custom Loss Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/loss/kl_divergence.html">Kullback-Leibler (KL) Divergence</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/loss/loss.html">Loss functions</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/text/index.html">Text Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/text/gnmt.html">Google Neural Machine Translation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/text/transformer.html">Machine Translation with Transformer</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/training/index.html">Training</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/fit_api_tutorial.html">MXNet Gluon Fit API</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/trainer.html">Trainer</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/learning_rates/index.html">Learning Rates</a><ul>
 <li class="toctree-l6"><a class="reference internal" href="../packages/gluon/training/learning_rates/learning_rate_finder.html">Learning Rate Finder</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../packages/gluon/training/learning_rates/learning_rate_schedules.html">Learning Rate Schedules</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../packages/gluon/training/learning_rates/learning_rate_schedules_advanced.html">Advanced Learning Rate Schedules</a></li>
 </ul>
 </li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/normalization/index.html">Normalization Blocks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/kvstore/index.html">KVStore</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/kvstore/kvstore.html">Distributed Key-Value Store</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/ndarray/index.html">NDArray</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/01-ndarray-intro.html">An Intro: Manipulate Data the MXNet Way with NDArray</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/02-ndarray-operations.html">NDArray Operations</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/03-ndarray-contexts.html">NDArray Contexts</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/gotchas_numpy_in_mxnet.html">Gotchas using NumPy in Apache MXNet</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/sparse/index.html">Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/csr.html">CSRNDArray - NDArray in Compressed Sparse Row Storage Format</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/row_sparse.html">RowSparseNDArray - NDArray for Sparse Gradient Updates</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/train.html">Train a Linear Regression Model with Sparse Symbols</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/train_gluon.html">Sparse NDArrays with Gluon</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/onnx/index.html">ONNX</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/onnx/fine_tuning_gluon.html">Fine-tuning an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/onnx/inference_on_onnx_model.html">Running inference on MXNet/Gluon from an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/onnx/super_resolution.html">Importing an ONNX model into MXNet</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/deploy/export/onnx.html">Export ONNX Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/optimizer/index.html">Optimizers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/viz/index.html">Visualization</a><ul>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/visualize_graph">Visualize networks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../performance/index.html">Performance</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../performance/compression/index.html">Compression</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../performance/compression/int8.html">Deploy with int-8</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/float16">Float16</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/gradient_compression">Gradient Compression</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html">GluonCV with Quantized Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../performance/backend/index.html">Accelerated Backend Tools</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/mkldnn/index.html">Intel MKL-DNN</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/mkldnn/mkldnn_quantization.html">Quantize with MKL-DNN backend</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/mkldnn/mkldnn_quantization.html#Improving-accuracy-with-Intel®-Neural-Compressor">Improving accuracy with Intel® Neural Compressor</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/mkldnn/mkldnn_readme.html">Install MXNet with MKL-DNN</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/tensorrt/index.html">TensorRT</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/tensorrt/tensorrt.html">Optimizing Deep Learning Computation Graphs with TensorRT</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/tvm.html">Use TVM</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/profiler.html">Profiling MXNet Models</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/amp.html">Using AMP: Automatic Mixed Precision</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../deploy/index.html">Deployment</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../deploy/export/index.html">Export</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/export/onnx.html">Exporting to ONNX format</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html">Export Gluon CV Models</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html">Save / Load Parameters</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../deploy/inference/index.html">Inference</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/cpp.html">Deploy into C++</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/image_classification_jetson.html">Image Classication using pretrained ResNet-50 model on Jetson module</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/scala.html">Deploy into a Java or Scala Environment</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/wine_detector.html">Real-time Object Detection with MXNet On The Raspberry Pi</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../deploy/run-on-aws/index.html">Run on AWS</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/run-on-aws/use_ec2.html">Run on an EC2 Instance</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/run-on-aws/use_sagemaker.html">Run on Amazon SageMaker</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/run-on-aws/cloud.html">MXNet on the Cloud</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2 current"><a class="reference internal" href="index.html">Extend</a><ul class="current">
 <li class="toctree-l3 current"><a class="current reference internal" href="#">Custom Layers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="customop.html">Custom Numpy Operators</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/new_op">New Operator Creation</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/add_op_in_backend">New Operator in MXNet Backend</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="../../api/index.html">Python API</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="../../api/ndarray/index.html">mxnet.ndarray</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/ndarray.html">ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/contrib/index.html">ndarray.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/image/index.html">ndarray.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/linalg/index.html">ndarray.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/op/index.html">ndarray.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/random/index.html">ndarray.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/register/index.html">ndarray.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/sparse/index.html">ndarray.sparse</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/utils/index.html">ndarray.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/gluon/index.html">mxnet.gluon</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/block.html">gluon.Block</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/hybrid_block.html">gluon.HybridBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/symbol_block.html">gluon.SymbolBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/constant.html">gluon.Constant</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/parameter.html">gluon.Parameter</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/parameter_dict.html">gluon.ParameterDict</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/trainer.html">gluon.Trainer</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/contrib/index.html">gluon.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/data/index.html">gluon.data</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../api/gluon/data/vision/index.html">data.vision</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../api/gluon/data/vision/datasets/index.html">vision.datasets</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../api/gluon/data/vision/transforms/index.html">vision.transforms</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/loss/index.html">gluon.loss</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/model_zoo/index.html">gluon.model_zoo.vision</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/nn/index.html">gluon.nn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/rnn/index.html">gluon.rnn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/utils/index.html">gluon.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/autograd/index.html">mxnet.autograd</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/initializer/index.html">mxnet.initializer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/optimizer/index.html">mxnet.optimizer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/lr_scheduler/index.html">mxnet.lr_scheduler</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/metric/index.html">mxnet.metric</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/kvstore/index.html">mxnet.kvstore</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/symbol/index.html">mxnet.symbol</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/symbol.html">symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/contrib/index.html">symbol.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/image/index.html">symbol.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/linalg/index.html">symbol.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/op/index.html">symbol.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/random/index.html">symbol.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/register/index.html">symbol.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/sparse/index.html">symbol.sparse</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/module/index.html">mxnet.module</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/contrib/index.html">mxnet.contrib</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/autograd/index.html">contrib.autograd</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/io/index.html">contrib.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/ndarray/index.html">contrib.ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/onnx/index.html">contrib.onnx</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/quantization/index.html">contrib.quantization</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/symbol/index.html">contrib.symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/tensorboard/index.html">contrib.tensorboard</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/tensorrt/index.html">contrib.tensorrt</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/text/index.html">contrib.text</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/mxnet/index.html">mxnet</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/attribute/index.html">mxnet.attribute</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/base/index.html">mxnet.base</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/callback/index.html">mxnet.callback</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/context/index.html">mxnet.context</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/engine/index.html">mxnet.engine</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/executor/index.html">mxnet.executor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/executor_manager/index.html">mxnet.executor_manager</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/image/index.html">mxnet.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/io/index.html">mxnet.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/kvstore_server/index.html">mxnet.kvstore_server</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/libinfo/index.html">mxnet.libinfo</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/log/index.html">mxnet.log</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/model/index.html">mxnet.model</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/monitor/index.html">mxnet.monitor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/name/index.html">mxnet.name</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/notebook/index.html">mxnet.notebook</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/operator/index.html">mxnet.operator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/profiler/index.html">mxnet.profiler</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/random/index.html">mxnet.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/recordio/index.html">mxnet.recordio</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/registry/index.html">mxnet.registry</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/rtc/index.html">mxnet.rtc</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/runtime/index.html">mxnet.runtime</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/test_utils/index.html">mxnet.test_utils</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/torch/index.html">mxnet.torch</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/util/index.html">mxnet.util</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/visualization/index.html">mxnet.visualization</a></li>
 </ul>
 </li>
 </ul>
 </li>
 </ul>

             </nav>

         </div>

 </header>
         <main class="mdl-layout__content" tabIndex="0">

         <script type="text/javascript" src="../../_static/sphinx_materialdesign_theme.js "></script>
         <script type="text/javascript" src="../../_static/feedback.js"></script>
     <header class="mdl-layout__drawer">

       <div class="globaltoc">
         <span class="mdl-layout-title toc">Table Of Contents</span>


             <nav class="mdl-navigation">
                 <ul class="current">
 <li class="toctree-l1 current"><a class="reference internal" href="../index.html">Python Tutorials</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="../getting-started/index.html">Getting Started</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/crash-course/index.html">Crash Course</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/1-ndarray.html">Manipulate data with <code class="docutils literal notranslate"><span class="pre">ndarray</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/2-nn.html">Create a neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/3-autograd.html">Automatic differentiation with <code class="docutils literal notranslate"><span class="pre">autograd</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/4-train.html">Train the neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/5-predict.html">Predict with a pre-trained model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/crash-course/6-use_gpus.html">Use GPUs</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/to-mxnet/index.html">Moving to MXNet from Other Frameworks</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../getting-started/to-mxnet/pytorch.html">PyTorch vs Apache MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/gluon_from_experiment_to_deployment.html">Gluon: from experiment to deployment</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../getting-started/logistic_regression_explained.html">Logistic regression explained</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/image/mnist.html">MNIST</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../packages/index.html">Packages</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../packages/autograd/index.html">Automatic Differentiation</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/gluon/index.html">Gluon</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/blocks/index.html">Blocks</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/custom-layer.html">Custom Layers</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/custom_layer_beginners.html">Customer Layers (Beginners)</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/hybridize.html">Hybridize</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/init.html">Initialization</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/naming.html">Parameter and Block Naming</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/nn.html">Layers and Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/parameters.html">Parameter Management</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/save_load_params.html">Saving and Loading Gluon Models</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/blocks/activations/activations.html">Activation Blocks</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/data/index.html">Data Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html#Spatial-Augmentation">Spatial Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html#Color-Augmentation">Color Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/data_augmentation.html#Composed-Augmentations">Composed Augmentations</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html">Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html#Using-own-data-with-included-Datasets">Using own data with included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html#Using-own-data-with-custom-Datasets">Using own data with custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/data/datasets.html#Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader">Appendix: Upgrading from Module <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> to Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/image/index.html">Image Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/image-augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/info_gan.html">Image similarity search with InfoGAN</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/mnist.html">Handwritten Digit Recognition</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/image/pretrained_models.html">Using pre-trained models in MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/loss/index.html">Losses</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/loss/custom-loss.html">Custom Loss Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/loss/kl_divergence.html">Kullback-Leibler (KL) Divergence</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/loss/loss.html">Loss functions</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/text/index.html">Text Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/text/gnmt.html">Google Neural Machine Translation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/text/transformer.html">Machine Translation with Transformer</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/gluon/training/index.html">Training</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/fit_api_tutorial.html">MXNet Gluon Fit API</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/trainer.html">Trainer</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/learning_rates/index.html">Learning Rates</a><ul>
 <li class="toctree-l6"><a class="reference internal" href="../packages/gluon/training/learning_rates/learning_rate_finder.html">Learning Rate Finder</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../packages/gluon/training/learning_rates/learning_rate_schedules.html">Learning Rate Schedules</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../packages/gluon/training/learning_rates/learning_rate_schedules_advanced.html">Advanced Learning Rate Schedules</a></li>
 </ul>
 </li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/gluon/training/normalization/index.html">Normalization Blocks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/kvstore/index.html">KVStore</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/kvstore/kvstore.html">Distributed Key-Value Store</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/ndarray/index.html">NDArray</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/01-ndarray-intro.html">An Intro: Manipulate Data the MXNet Way with NDArray</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/02-ndarray-operations.html">NDArray Operations</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/03-ndarray-contexts.html">NDArray Contexts</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/gotchas_numpy_in_mxnet.html">Gotchas using NumPy in Apache MXNet</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/ndarray/sparse/index.html">Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/csr.html">CSRNDArray - NDArray in Compressed Sparse Row Storage Format</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/row_sparse.html">RowSparseNDArray - NDArray for Sparse Gradient Updates</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/train.html">Train a Linear Regression Model with Sparse Symbols</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../packages/ndarray/sparse/train_gluon.html">Sparse NDArrays with Gluon</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/onnx/index.html">ONNX</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../packages/onnx/fine_tuning_gluon.html">Fine-tuning an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/onnx/inference_on_onnx_model.html">Running inference on MXNet/Gluon from an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../packages/onnx/super_resolution.html">Importing an ONNX model into MXNet</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/deploy/export/onnx.html">Export ONNX Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/optimizer/index.html">Optimizers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../packages/viz/index.html">Visualization</a><ul>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/visualize_graph">Visualize networks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../performance/index.html">Performance</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../performance/compression/index.html">Compression</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../performance/compression/int8.html">Deploy with int-8</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/float16">Float16</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/gradient_compression">Gradient Compression</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html">GluonCV with Quantized Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../performance/backend/index.html">Accelerated Backend Tools</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/mkldnn/index.html">Intel MKL-DNN</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/mkldnn/mkldnn_quantization.html">Quantize with MKL-DNN backend</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/mkldnn/mkldnn_quantization.html#Improving-accuracy-with-Intel®-Neural-Compressor">Improving accuracy with Intel® Neural Compressor</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/mkldnn/mkldnn_readme.html">Install MXNet with MKL-DNN</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/tensorrt/index.html">TensorRT</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../performance/backend/tensorrt/tensorrt.html">Optimizing Deep Learning Computation Graphs with TensorRT</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/tvm.html">Use TVM</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/profiler.html">Profiling MXNet Models</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../performance/backend/amp.html">Using AMP: Automatic Mixed Precision</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../deploy/index.html">Deployment</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../deploy/export/index.html">Export</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/export/onnx.html">Exporting to ONNX format</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html">Export Gluon CV Models</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html">Save / Load Parameters</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../deploy/inference/index.html">Inference</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/cpp.html">Deploy into C++</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/image_classification_jetson.html">Image Classication using pretrained ResNet-50 model on Jetson module</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/scala.html">Deploy into a Java or Scala Environment</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/inference/wine_detector.html">Real-time Object Detection with MXNet On The Raspberry Pi</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../deploy/run-on-aws/index.html">Run on AWS</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/run-on-aws/use_ec2.html">Run on an EC2 Instance</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/run-on-aws/use_sagemaker.html">Run on Amazon SageMaker</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../deploy/run-on-aws/cloud.html">MXNet on the Cloud</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2 current"><a class="reference internal" href="index.html">Extend</a><ul class="current">
 <li class="toctree-l3 current"><a class="current reference internal" href="#">Custom Layers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="customop.html">Custom Numpy Operators</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/new_op">New Operator Creation</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/add_op_in_backend">New Operator in MXNet Backend</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="../../api/index.html">Python API</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="../../api/ndarray/index.html">mxnet.ndarray</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/ndarray.html">ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/contrib/index.html">ndarray.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/image/index.html">ndarray.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/linalg/index.html">ndarray.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/op/index.html">ndarray.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/random/index.html">ndarray.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/register/index.html">ndarray.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/sparse/index.html">ndarray.sparse</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/ndarray/utils/index.html">ndarray.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/gluon/index.html">mxnet.gluon</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/block.html">gluon.Block</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/hybrid_block.html">gluon.HybridBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/symbol_block.html">gluon.SymbolBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/constant.html">gluon.Constant</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/parameter.html">gluon.Parameter</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/parameter_dict.html">gluon.ParameterDict</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/trainer.html">gluon.Trainer</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/contrib/index.html">gluon.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/data/index.html">gluon.data</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../api/gluon/data/vision/index.html">data.vision</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../api/gluon/data/vision/datasets/index.html">vision.datasets</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../api/gluon/data/vision/transforms/index.html">vision.transforms</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/loss/index.html">gluon.loss</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/model_zoo/index.html">gluon.model_zoo.vision</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/nn/index.html">gluon.nn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/rnn/index.html">gluon.rnn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/gluon/utils/index.html">gluon.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/autograd/index.html">mxnet.autograd</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/initializer/index.html">mxnet.initializer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/optimizer/index.html">mxnet.optimizer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/lr_scheduler/index.html">mxnet.lr_scheduler</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/metric/index.html">mxnet.metric</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/kvstore/index.html">mxnet.kvstore</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/symbol/index.html">mxnet.symbol</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/symbol.html">symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/contrib/index.html">symbol.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/image/index.html">symbol.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/linalg/index.html">symbol.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/op/index.html">symbol.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/random/index.html">symbol.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/register/index.html">symbol.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/symbol/sparse/index.html">symbol.sparse</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/module/index.html">mxnet.module</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/contrib/index.html">mxnet.contrib</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/autograd/index.html">contrib.autograd</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/io/index.html">contrib.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/ndarray/index.html">contrib.ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/onnx/index.html">contrib.onnx</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/quantization/index.html">contrib.quantization</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/symbol/index.html">contrib.symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/tensorboard/index.html">contrib.tensorboard</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/tensorrt/index.html">contrib.tensorrt</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/contrib/text/index.html">contrib.text</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../api/mxnet/index.html">mxnet</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/attribute/index.html">mxnet.attribute</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/base/index.html">mxnet.base</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/callback/index.html">mxnet.callback</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/context/index.html">mxnet.context</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/engine/index.html">mxnet.engine</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/executor/index.html">mxnet.executor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/executor_manager/index.html">mxnet.executor_manager</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/image/index.html">mxnet.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/io/index.html">mxnet.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/kvstore_server/index.html">mxnet.kvstore_server</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/libinfo/index.html">mxnet.libinfo</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/log/index.html">mxnet.log</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/model/index.html">mxnet.model</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/monitor/index.html">mxnet.monitor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/name/index.html">mxnet.name</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/notebook/index.html">mxnet.notebook</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/operator/index.html">mxnet.operator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/profiler/index.html">mxnet.profiler</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/random/index.html">mxnet.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/recordio/index.html">mxnet.recordio</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/registry/index.html">mxnet.registry</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/rtc/index.html">mxnet.rtc</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/runtime/index.html">mxnet.runtime</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/test_utils/index.html">mxnet.test_utils</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/torch/index.html">mxnet.torch</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/util/index.html">mxnet.util</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../api/mxnet/visualization/index.html">mxnet.visualization</a></li>
 </ul>
 </li>
 </ul>
 </li>
 </ul>

             </nav>

         </div>

 </header>

     <div class="document">
         <div class="page-content" role="main">

   <!--- Licensed to the Apache Software Foundation (ASF) under one --><!--- or more contributor license agreements.  See the NOTICE file --><!--- distributed with this work for additional information --><!--- regarding copyright ownership.  The ASF licenses this file --><!--- to you under the Apache License, Version 2.0 (the --><!--- "License"); you may not use this file except in compliance --><!--- with the License.  You may obtain a copy of the License at --><!---   http://www.apache.org/licenses/LICENSE-2.0 --><!--- Unless required by applicable law or agreed to in writing, --><!--- software distributed under the License is distributed on an --><!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --><!--- KIND, either express or implied.  See the License for the --><!--- specific language governing permissions and limitations --><!--- under the License. --><div class="section" id="Custom-Layers">
 <h1>Custom Layers<a class="headerlink" href="#Custom-Layers" title="Permalink to this headline">¶</a></h1>
 <p>While Gluon API for Apache MxNet comes with <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/nn.html">a decent number of pre-defined layers</a>, at some point one may find that a new layer is needed. Adding a new layer in Gluon API is straightforward, yet there are a few things that one needs to keep in mind.</p>
 <p>In this article, I will cover how to create a new layer from scratch, how to use it, what are possible pitfalls and how to avoid them.</p>
 <div class="section" id="The-simplest-custom-layer">
 <h2>The simplest custom layer<a class="headerlink" href="#The-simplest-custom-layer" title="Permalink to this headline">¶</a></h2>
 <p>To create a new layer in Gluon API, one must create a class that inherits from <a class="reference external" href="https://github.com/apache/mxnet/blob/c9818480680f84daa6e281a974ab263691302ba8/python/mxnet/gluon/block.py#L128">Block</a> class. This class provides the most basic functionality, and all pre-defined layers inherit from it directly or via other subclasses. Because each layer in Apache MxNet inherits from <code class="docutils literal notranslate"><span class="pre">Block</span></code>, words “layer” and “block” are used interchangeable inside of the Apache MxNet community.</p>
 <p>The only instance method needed to be implemented is <a class="reference external" href="https://github.com/apache/mxnet/blob/c9818480680f84daa6e281a974ab263691302ba8/python/mxnet/gluon/block.py#L909">forward(self, x)</a>, which defines what exactly your layer is going to do during forward propagation. Notice, that it doesn’t require to provide what the block should do during back propogation. Back propogation pass for blocks is done by Apache MxNet for you.</p>
 <p>In the example below, we define a new layer and implement <code class="docutils literal notranslate"><span class="pre">forward()</span></code> method to normalize input data by fitting it into a range of [0, 1].</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Do some initial imports used throughout this tutorial</span>
 <span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">import</span> <span class="nn">mxnet</span> <span class="k">as</span> <span class="nn">mx</span>
 <span class="kn">from</span> <span class="nn">mxnet</span> <span class="kn">import</span> <span class="n">nd</span><span class="p">,</span> <span class="n">gluon</span><span class="p">,</span> <span class="n">autograd</span>
 <span class="kn">from</span> <span class="nn">mxnet.gluon.nn</span> <span class="kn">import</span> <span class="n">Dense</span>
 <span class="n">mx</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>                      <span class="c1"># Set seed for reproducable results</span>
 </pre></div>
 </div>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">NormalizationLayer</span><span class="p">(</span><span class="n">gluon</span><span class="o">.</span><span class="n">Block</span><span class="p">):</span>
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">NormalizationLayer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

     <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
         <span class="k">return</span> <span class="p">(</span><span class="n">x</span> <span class="o">-</span> <span class="n">nd</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> <span class="o">/</span> <span class="p">(</span><span class="n">nd</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">-</span> <span class="n">nd</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
 </pre></div>
 </div>
 <p>The rest of methods of the <code class="docutils literal notranslate"><span class="pre">Block</span></code> class are already implemented, and majority of them are used to work with parameters of a block. There is one very special method named <a class="reference external" href="https://github.com/apache/mxnet/blob/master/python/mxnet/gluon/block.py#L384">hybridize()</a>, though, which I am going to cover before moving to a more complex example of a custom layer.</p>
 </div>
 <div class="section" id="Hybridization-and-the-difference-between-Block-and-HybridBlock">
 <h2>Hybridization and the difference between Block and HybridBlock<a class="headerlink" href="#Hybridization-and-the-difference-between-Block-and-HybridBlock" title="Permalink to this headline">¶</a></h2>
 <p>Looking into implementation of <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/nn.html">existing layers</a>, one may find that more often a block inherits from a <a class="reference external" href="https://github.com/apache/mxnet/blob/master/python/mxnet/gluon/block.py#L428">HybridBlock</a>, instead of directly inheriting from <code class="docutils literal notranslate"><span class="pre">Block</span></code>.</p>
 <p>The reason for that is that <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code> allows to write custom layers that can be used in imperative programming as well as in symbolic programming. It is convinient to support both ways, because the imperative programming eases the debugging of the code and the symbolic one provides faster execution speed. You can learn more about the difference between symbolic vs. imperative programming from <a class="reference external" href="/api/architecture/program_model">this article</a>.</p>
 <p>Hybridization is a process that Apache MxNet uses to create a symbolic graph of a forward computation. This allows to increase computation performance by optimizing the computational symbolic graph. Once the symbolic graph is created, Apache MxNet caches and reuses it for subsequent computations.</p>
 <p>To simplify support of both imperative and symbolic programming, Apache MxNet introduce the <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code> class. Compare to the <code class="docutils literal notranslate"><span class="pre">Block</span></code> class, <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code> already has its <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/gluon.html#mxnet.gluon.HybridBlock.forward">forward()</a> method implemented, but it defines a <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/gluon.html#mxnet.gluon.HybridBlock.hybrid_forward">hybrid_forward()</a> method that needs to be implemented.</p>
 <p>The main difference between <code class="docutils literal notranslate"><span class="pre">forward()</span></code> and <code class="docutils literal notranslate"><span class="pre">hybrid_forward()</span></code> is an <code class="docutils literal notranslate"><span class="pre">F</span></code> argument. This argument sometimes is refered as a <code class="docutils literal notranslate"><span class="pre">backend</span></code> in the Apache MxNet community. Depending on if hybridization has been done or not, <code class="docutils literal notranslate"><span class="pre">F</span></code> can refer either to <a class="reference external" href="https://mxnet.apache.org/api/python/ndarray/ndarray.html">mxnet.ndarray API</a> or <a class="reference external" href="https://mxnet.apache.org/api/python/symbol/symbol.html">mxnet.symbol API</a>. The former is used for imperative programming, and the latter for symbolic programming.</p>
 <p>To support hybridization, it is important to use only methods avaible directly from <code class="docutils literal notranslate"><span class="pre">F</span></code> parameter. Usually, there are equivalent methods in both APIs, but sometimes there are mismatches or small variations. For example, by default, subtraction and division of NDArrays support broadcasting, while in Symbol API broadcasting is supported in a separate operators.</p>
 <p>Knowing this, we can can rewrite our example layer, using HybridBlock:</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">NormalizationHybridLayer</span><span class="p">(</span><span class="n">gluon</span><span class="o">.</span><span class="n">HybridBlock</span><span class="p">):</span>
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">NormalizationHybridLayer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

     <span class="k">def</span> <span class="nf">hybrid_forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">F</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
         <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">broadcast_div</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">broadcast_sub</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">F</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">x</span><span class="p">)),</span> <span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">broadcast_sub</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">F</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">x</span><span class="p">))))</span>
 </pre></div>
 </div>
 <p>Thanks to inheriting from HybridBlock, one can easily do forward pass on a given ndarray, either on CPU or GPU:</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">layer</span> <span class="o">=</span> <span class="n">NormalizationHybridLayer</span><span class="p">()</span>
 <span class="n">layer</span><span class="p">(</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="n">ctx</span><span class="o">=</span><span class="n">mx</span><span class="o">.</span><span class="n">cpu</span><span class="p">()))</span>
 </pre></div>
 </div>
 <p>Output:</p>
 <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="o">[</span><span class="m">0</span>.<span class="w">  </span><span class="m">0</span>.5<span class="w"> </span><span class="m">1</span>.<span class="w"> </span><span class="o">]</span>
 &lt;NDArray<span class="w"> </span><span class="m">3</span><span class="w"> </span>@cpu<span class="o">(</span><span class="m">0</span><span class="o">)</span>&gt;
 </pre></div>
 </div>
 <p>As a rule of thumb, one should always implement custom layers by inheriting from <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code>. This allows to have more flexibility, and doesn’t affect execution speed once hybridization is done.</p>
 <p>Unfortunately, at the moment of writing this tutorial, NLP related layers such as <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/rnn.html#mxnet.gluon.rnn.RNN">RNN</a>, <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/rnn.html#mxnet.gluon.rnn.GRU">GRU</a> and <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/rnn.html#mxnet.gluon.rnn.LSTM">LSTM</a> are directly inhereting from the <code class="docutils literal notranslate"><span class="pre">Block</span></code> class via common <code class="docutils literal notranslate"><span class="pre">_RNNLayer</span></code> class. That means that networks with such layers cannot be hybridized. But this might change
 in the future, so stay tuned.</p>
 <p>It is important to notice that hybridization has nothing to do with computation on GPU. One can train both hybridized and non-hybridized networks on both CPU and GPU, though hybridized networks would work faster. Though, it is hard to say in advance how much faster it is going to be.</p>
 </div>
 <div class="section" id="Adding-a-custom-layer-to-a-network">
 <h2>Adding a custom layer to a network<a class="headerlink" href="#Adding-a-custom-layer-to-a-network" title="Permalink to this headline">¶</a></h2>
 <p>While it is possible, custom layers are rarely used separately. Most often they are used with predefined layers to create a neural network. Output of one layer is used as an input of another layer.</p>
 <p>Depending on which class you used as a base one, you can use either <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/gluon.html#mxnet.gluon.nn.Sequential">Sequential</a> or <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/gluon.html#mxnet.gluon.nn.HybridSequential">HybridSequential</a> container to form a sequential neural network. By adding layers one by one, one adds dependencies of one layer’s input from another layer’s output. It is worth noting, that both <code class="docutils literal notranslate"><span class="pre">Sequential</span></code> and <code class="docutils literal notranslate"><span class="pre">HybridSequential</span></code>
 containers inherit from <code class="docutils literal notranslate"><span class="pre">Block</span></code> and <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code> respectively.</p>
 <p>Below is an example of how to create a simple neural network with a custom layer. In this example, <code class="docutils literal notranslate"><span class="pre">NormalizationHybridLayer</span></code> gets as an input the output from <code class="docutils literal notranslate"><span class="pre">Dense(5)</span></code> layer and pass its output as an input to <code class="docutils literal notranslate"><span class="pre">Dense(1)</span></code> layer.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">net</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">HybridSequential</span><span class="p">()</span>                         <span class="c1"># Define a Neural Network as a sequence of hybrid blocks</span>
 <span class="k">with</span> <span class="n">net</span><span class="o">.</span><span class="n">name_scope</span><span class="p">():</span>                                    <span class="c1"># Used to disambiguate saving and loading net parameters</span>
     <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Dense</span><span class="p">(</span><span class="mi">5</span><span class="p">))</span>                                     <span class="c1"># Add Dense layer with 5 neurons</span>
     <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">NormalizationHybridLayer</span><span class="p">())</span>                   <span class="c1"># Add our custom layer</span>
     <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Dense</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>                                     <span class="c1"># Add Dense layer with 1 neurons</span>


 <span class="n">net</span><span class="o">.</span><span class="n">initialize</span><span class="p">(</span><span class="n">mx</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">Xavier</span><span class="p">(</span><span class="n">magnitude</span><span class="o">=</span><span class="mf">2.24</span><span class="p">))</span>            <span class="c1"># Initialize parameters of all layers</span>
 <span class="n">net</span><span class="o">.</span><span class="n">hybridize</span><span class="p">()</span>                                           <span class="c1"># Create, optimize and cache computational graph</span>
 <span class="nb">input</span> <span class="o">=</span> <span class="n">nd</span><span class="o">.</span><span class="n">random_uniform</span><span class="p">(</span><span class="n">low</span><span class="o">=-</span><span class="mi">10</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span> <span class="c1"># Create 5 random examples with 2 feature each in range [-10, 10]</span>
 <span class="n">net</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>Output:</p>
 <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="o">[[</span>-0.13601446<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="m">0</span>.26103732<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-0.05046433<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-1.2375476<span class="w"> </span><span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-0.15506986<span class="o">]]</span>
 &lt;NDArray<span class="w"> </span>5x1<span class="w"> </span>@cpu<span class="o">(</span><span class="m">0</span><span class="o">)</span>&gt;
 </pre></div>
 </div>
 </div>
 <div class="section" id="Parameters-of-a-custom-layer">
 <h2>Parameters of a custom layer<a class="headerlink" href="#Parameters-of-a-custom-layer" title="Permalink to this headline">¶</a></h2>
 <p>Usually, a layer has a set of associated parameters, sometimes also referred as weights. This is an internal state of a layer. Most often, these parameters are the ones, that we want to learn during backpropogation step, but sometimes these parameters might be just constants we want to use during forward pass.</p>
 <p>All parameters of a block are stored and accessed via <a class="reference external" href="https://github.com/apache/mxnet/blob/master/python/mxnet/gluon/parameter.py#L508">ParameterDict</a> class. This class helps with initialization, updating, saving and loading of the parameters. Each layer can have multiple set of parameters, and all of them can be stored in a single instance of the <code class="docutils literal notranslate"><span class="pre">ParameterDict</span></code> class. On a block level, the instance of the <code class="docutils literal notranslate"><span class="pre">ParameterDict</span></code> class is accessible via <code class="docutils literal notranslate"><span class="pre">self.params</span></code> field, and outside of a
 block one can access all parameters of the network via <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/gluon.html#mxnet.gluon.Block.collect_params">collect_params()</a> method called on a <code class="docutils literal notranslate"><span class="pre">container</span></code>. <code class="docutils literal notranslate"><span class="pre">ParameterDict</span></code> uses <a class="reference external" href="https://mxnet.apache.org/api/python/gluon/gluon.html#mxnet.gluon.Parameter">Parameter</a> class to represent parameters inside of Apache MxNet neural network. If parameter doesn’t exist, trying to get a parameter via <code class="docutils literal notranslate"><span class="pre">self.params</span></code> will create it automatically.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">NormalizationHybridLayer</span><span class="p">(</span><span class="n">gluon</span><span class="o">.</span><span class="n">HybridBlock</span><span class="p">):</span>
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hidden_units</span><span class="p">,</span> <span class="n">scales</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">NormalizationHybridLayer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

         <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">name_scope</span><span class="p">():</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;weights&#39;</span><span class="p">,</span>
                                            <span class="n">shape</span><span class="o">=</span><span class="p">(</span><span class="n">hidden_units</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
                                            <span class="n">allow_deferred_init</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

             <span class="bp">self</span><span class="o">.</span><span class="n">scales</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;scales&#39;</span><span class="p">,</span>
                                       <span class="n">shape</span><span class="o">=</span><span class="n">scales</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span>
                                       <span class="n">init</span><span class="o">=</span><span class="n">mx</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">Constant</span><span class="p">(</span><span class="n">scales</span><span class="o">.</span><span class="n">asnumpy</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">()),</span> <span class="c1"># Convert to regular list to make this object serializable</span>
                                       <span class="n">differentiable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

     <span class="k">def</span> <span class="nf">hybrid_forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">F</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">scales</span><span class="p">):</span>
         <span class="n">normalized_data</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">broadcast_div</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">broadcast_sub</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">F</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">x</span><span class="p">)),</span> <span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">broadcast_sub</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">F</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">x</span><span class="p">))))</span>
         <span class="n">weighted_data</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">FullyConnected</span><span class="p">(</span><span class="n">normalized_data</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">num_hidden</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">no_bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
         <span class="n">scaled_data</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">broadcast_mul</span><span class="p">(</span><span class="n">scales</span><span class="p">,</span> <span class="n">weighted_data</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">scaled_data</span>
 </pre></div>
 </div>
 <p>In the example above 2 set of parameters are defined: 1. Parameter <code class="docutils literal notranslate"><span class="pre">weights</span></code> is trainable. Its shape is unknown during construction phase and will be infered on the first run of forward propogation; 1. Parameter <code class="docutils literal notranslate"><span class="pre">scale</span></code> is a constant that doesn’t change. Its shape is defined during construction.</p>
 <p>Notice a few aspects of this code: * <code class="docutils literal notranslate"><span class="pre">name_scope()</span></code> method is used to add a prefix to parameter names during saving and loading * Shape is not provided when creating <code class="docutils literal notranslate"><span class="pre">weights</span></code>. Instead it is going to be infered from the shape of the input * <code class="docutils literal notranslate"><span class="pre">Scales</span></code> parameter is initialized and marked as <code class="docutils literal notranslate"><span class="pre">differentiable=False</span></code>. * <code class="docutils literal notranslate"><span class="pre">F</span></code> backend is used for all calculations * The calculation of dot product is done using <code class="docutils literal notranslate"><span class="pre">F.FullyConnected()</span></code> method instead of <code class="docutils literal notranslate"><span class="pre">F.dot()</span></code> method. The one was chosen
 over another because the former supports automatic infering shapes of inputs while the latter doesn’t. This is extremely important to know, if one doesn’t want to hard code all the shapes. The best way to learn what operators supports automatic inference of input shapes at the moment is browsing C++ implementation of operators to see if one uses a method <code class="docutils literal notranslate"><span class="pre">SHAPE_ASSIGN_CHECK(*in_shape,</span> <span class="pre">fullc::kWeight,</span> <span class="pre">Shape2(param.num_hidden,</span> <span class="pre">num_input));</span></code> * <code class="docutils literal notranslate"><span class="pre">hybrid_forward()</span></code> method signature has changed.
 It accepts two new arguments: <code class="docutils literal notranslate"><span class="pre">weights</span></code> and <code class="docutils literal notranslate"><span class="pre">scales</span></code>.</p>
 <p>The last peculiarity is due to support of imperative and symbolic programming by <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code>. During training phase, parameters are passed to the layer by Apache MxNet framework as additional arguments to the method, because they might need to be converted to a <code class="docutils literal notranslate"><span class="pre">Symbol</span></code> depending on if the layer was hybridized. One shouldn’t use <code class="docutils literal notranslate"><span class="pre">self.weights</span></code> and <code class="docutils literal notranslate"><span class="pre">self.scales</span></code> or <code class="docutils literal notranslate"><span class="pre">self.params.get</span></code> in <code class="docutils literal notranslate"><span class="pre">hybrid_forward</span></code> except to get shapes of parameters.</p>
 <p>Running forward pass on this network is very similar to the previous example, so instead of just doing one forward pass, let’s run whole training for a few epochs to show that <code class="docutils literal notranslate"><span class="pre">scales</span></code> parameter doesn’t change during the training while <code class="docutils literal notranslate"><span class="pre">weights</span></code> parameter is changing.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">print_params</span><span class="p">(</span><span class="n">title</span><span class="p">,</span> <span class="n">net</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Helper function to print out the state of parameters of NormalizationHybridLayer</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="nb">print</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
     <span class="n">hybridlayer_params</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">net</span><span class="o">.</span><span class="n">collect_params</span><span class="p">()</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="s1">&#39;normalizationhybridlayer&#39;</span> <span class="ow">in</span> <span class="n">k</span> <span class="p">}</span>

     <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">hybridlayer_params</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
         <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> = </span><span class="si">{}</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">.</span><span class="n">data</span><span class="p">()))</span>

 <span class="n">net</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">HybridSequential</span><span class="p">()</span>                             <span class="c1"># Define a Neural Network as a sequence of hybrid blocks</span>
 <span class="k">with</span> <span class="n">net</span><span class="o">.</span><span class="n">name_scope</span><span class="p">():</span>                                        <span class="c1"># Used to disambiguate saving and loading net parameters</span>
     <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Dense</span><span class="p">(</span><span class="mi">5</span><span class="p">))</span>                                         <span class="c1"># Add Dense layer with 5 neurons</span>
     <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">NormalizationHybridLayer</span><span class="p">(</span><span class="n">hidden_units</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
                                      <span class="n">scales</span> <span class="o">=</span> <span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">2</span><span class="p">])))</span> <span class="c1"># Add our custom layer</span>
     <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Dense</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>                                         <span class="c1"># Add Dense layer with 1 neurons</span>


 <span class="n">net</span><span class="o">.</span><span class="n">initialize</span><span class="p">(</span><span class="n">mx</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">Xavier</span><span class="p">(</span><span class="n">magnitude</span><span class="o">=</span><span class="mf">2.24</span><span class="p">))</span>                <span class="c1"># Initialize parameters of all layers</span>
 <span class="n">net</span><span class="o">.</span><span class="n">hybridize</span><span class="p">()</span>                                               <span class="c1"># Create, optimize and cache computational graph</span>

 <span class="nb">input</span> <span class="o">=</span> <span class="n">nd</span><span class="o">.</span><span class="n">random_uniform</span><span class="p">(</span><span class="n">low</span><span class="o">=-</span><span class="mi">10</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>     <span class="c1"># Create 5 random examples with 2 feature each in range [-10, 10]</span>
 <span class="n">label</span> <span class="o">=</span> <span class="n">nd</span><span class="o">.</span><span class="n">random_uniform</span><span class="p">(</span><span class="n">low</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>

 <span class="n">mse_loss</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">loss</span><span class="o">.</span><span class="n">L2Loss</span><span class="p">()</span>                                <span class="c1"># Mean squared error between output and label</span>
 <span class="n">trainer</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">Trainer</span><span class="p">(</span><span class="n">net</span><span class="o">.</span><span class="n">collect_params</span><span class="p">(),</span>                 <span class="c1"># Init trainer with Stochastic Gradient Descent (sgd) optimization method and parameters for it</span>
                         <span class="s1">&#39;sgd&#39;</span><span class="p">,</span>
                         <span class="p">{</span><span class="s1">&#39;learning_rate&#39;</span><span class="p">:</span> <span class="mf">0.1</span><span class="p">,</span> <span class="s1">&#39;momentum&#39;</span><span class="p">:</span> <span class="mf">0.9</span> <span class="p">})</span>

 <span class="k">with</span> <span class="n">autograd</span><span class="o">.</span><span class="n">record</span><span class="p">():</span>                                       <span class="c1"># Autograd records computations done on NDArrays inside &quot;with&quot; block</span>
     <span class="n">output</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>                                       <span class="c1"># Run forward propogation</span>

     <span class="n">print_params</span><span class="p">(</span><span class="s2">&quot;=========== Parameters after forward pass ===========</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">net</span><span class="p">)</span>
     <span class="n">loss</span> <span class="o">=</span> <span class="n">mse_loss</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>                            <span class="c1"># Calculate MSE</span>

 <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>                                               <span class="c1"># Backward computes gradients and stores them as a separate array within each NDArray in .grad field</span>
 <span class="n">trainer</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>                                  <span class="c1"># Trainer updates parameters of every block, using .grad field using oprimization method (sgd in this example)</span>
                                                               <span class="c1"># We provide batch size that is used as a divider in cost function formula</span>
 <span class="n">print_params</span><span class="p">(</span><span class="s2">&quot;=========== Parameters after backward pass ===========</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">net</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>Output:</p>
 <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="o">===========</span><span class="w"> </span>Parameters<span class="w"> </span>after<span class="w"> </span>forward<span class="w"> </span><span class="nv">pass</span><span class="w"> </span><span class="o">===========</span>

 <span class="nv">hybridsequential94_normalizationhybridlayer0_weights</span><span class="w"> </span><span class="o">=</span>
 <span class="o">[[</span>-0.3983642<span class="w">  </span>-0.505708<span class="w">   </span>-0.02425683<span class="w"> </span>-0.3133553<span class="w">  </span>-0.35161012<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="m">0</span>.6467543<span class="w">   </span><span class="m">0</span>.3918715<span class="w">  </span>-0.6154656<span class="w">  </span>-0.20702496<span class="w"> </span>-0.4243446<span class="w"> </span><span class="o">]</span>
 <span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="m">0</span>.6077331<span class="w">   </span><span class="m">0</span>.03922009<span class="w">  </span><span class="m">0</span>.13425875<span class="w">  </span><span class="m">0</span>.5729856<span class="w">  </span>-0.14446527<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-0.3572498<span class="w">   </span><span class="m">0</span>.18545026<span class="w"> </span>-0.09098256<span class="w">  </span><span class="m">0</span>.5106366<span class="w">  </span>-0.35151464<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-0.39846328<span class="w">  </span><span class="m">0</span>.22245121<span class="w">  </span><span class="m">0</span>.13075739<span class="w">  </span><span class="m">0</span>.33387476<span class="w"> </span>-0.10088372<span class="o">]]</span>
 &lt;NDArray<span class="w"> </span>5x5<span class="w"> </span>@cpu<span class="o">(</span><span class="m">0</span><span class="o">)</span>&gt;

 <span class="nv">hybridsequential94_normalizationhybridlayer0_scales</span><span class="w"> </span><span class="o">=</span>
 <span class="o">[</span><span class="m">2</span>.<span class="o">]</span>
 &lt;NDArray<span class="w"> </span><span class="m">1</span><span class="w"> </span>@cpu<span class="o">(</span><span class="m">0</span><span class="o">)</span>&gt;

 <span class="o">===========</span><span class="w"> </span>Parameters<span class="w"> </span>after<span class="w"> </span>backward<span class="w"> </span><span class="nv">pass</span><span class="w"> </span><span class="o">===========</span>

 <span class="nv">hybridsequential94_normalizationhybridlayer0_weights</span><span class="w"> </span><span class="o">=</span>
 <span class="o">[[</span>-0.29839832<span class="w"> </span>-0.47213346<span class="w">  </span><span class="m">0</span>.08348035<span class="w"> </span>-0.2324698<span class="w">  </span>-0.27368504<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="m">0</span>.76268613<span class="w">  </span><span class="m">0</span>.43080837<span class="w"> </span>-0.49052125<span class="w"> </span>-0.11322092<span class="w"> </span>-0.3339738<span class="w"> </span><span class="o">]</span>
 <span class="w"> </span><span class="o">[</span><span class="w"> </span><span class="m">0</span>.48665082<span class="w"> </span>-0.00144657<span class="w">  </span><span class="m">0</span>.00376363<span class="w">  </span><span class="m">0</span>.47501418<span class="w"> </span>-0.23885089<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-0.22626656<span class="w">  </span><span class="m">0</span>.22944227<span class="w">  </span><span class="m">0</span>.05018325<span class="w">  </span><span class="m">0</span>.6166192<span class="w">  </span>-0.24941102<span class="o">]</span>
 <span class="w"> </span><span class="o">[</span>-0.44946212<span class="w">  </span><span class="m">0</span>.20532274<span class="w">  </span><span class="m">0</span>.07579394<span class="w">  </span><span class="m">0</span>.29261002<span class="w"> </span>-0.14063817<span class="o">]]</span>
 &lt;NDArray<span class="w"> </span>5x5<span class="w"> </span>@cpu<span class="o">(</span><span class="m">0</span><span class="o">)</span>&gt;

 <span class="nv">hybridsequential94_normalizationhybridlayer0_scales</span><span class="w"> </span><span class="o">=</span>
 <span class="o">[</span><span class="m">2</span>.<span class="o">]</span>
 &lt;NDArray<span class="w"> </span><span class="m">1</span><span class="w"> </span>@cpu<span class="o">(</span><span class="m">0</span><span class="o">)</span>&gt;
 </pre></div>
 </div>
 <p>As it is seen from the output above, <code class="docutils literal notranslate"><span class="pre">weights</span></code> parameter has been changed by the training and <code class="docutils literal notranslate"><span class="pre">scales</span></code> not.</p>
 </div>
 <div class="section" id="Conclusion">
 <h2>Conclusion<a class="headerlink" href="#Conclusion" title="Permalink to this headline">¶</a></h2>
 <p>One important quality of a Deep learning framework is extensibility. Empowered by flexible abstractions, like <code class="docutils literal notranslate"><span class="pre">Block</span></code> and <code class="docutils literal notranslate"><span class="pre">HybridBlock</span></code>, one can easily extend Apache MxNet functionality to match its needs.</p>
 </div>
 </div>


         <hr class="feedback-hr-top" />
 <div class="feedback-container">
     <div class="feedback-question">Did this page help you?</div>
     <div class="feedback-answer-container">
         <div class="feedback-answer yes-link" data-response="yes">Yes</div>
         <div class="feedback-answer no-link" data-response="no">No</div>
     </div>
     <div class="feedback-thank-you">Thanks for your feedback!</div>
 </div>
 <hr class="feedback-hr-bottom" />
         </div>
         <div class="side-doc-outline">
             <div class="side-doc-outline--content">
 <div class="localtoc">
     <p class="caption">
       <span class="caption-text">Table Of Contents</span>
     </p>
     <ul>
 <li><a class="reference internal" href="#">Custom Layers</a><ul>
 <li><a class="reference internal" href="#The-simplest-custom-layer">The simplest custom layer</a></li>
 <li><a class="reference internal" href="#Hybridization-and-the-difference-between-Block-and-HybridBlock">Hybridization and the difference between Block and HybridBlock</a></li>
 <li><a class="reference internal" href="#Adding-a-custom-layer-to-a-network">Adding a custom layer to a network</a></li>
 <li><a class="reference internal" href="#Parameters-of-a-custom-layer">Parameters of a custom layer</a></li>
 <li><a class="reference internal" href="#Conclusion">Conclusion</a></li>
 </ul>
 </li>
 </ul>

 </div>
             </div>
         </div>

       <div class="clearer"></div>
     </div><div class="pagenation">
      <a id="button-prev" href="index.html" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--colored" role="botton" accesskey="P">
          <i class="pagenation-arrow-L fas fa-arrow-left fa-lg"></i>
          <div class="pagenation-text">
             <span class="pagenation-direction">Previous</span>
             <div>Extend</div>
          </div>
      </a>
      <a id="button-next" href="customop.html" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--colored" role="botton" accesskey="N">
          <i class="pagenation-arrow-R fas fa-arrow-right fa-lg"></i>
         <div class="pagenation-text">
             <span class="pagenation-direction">Next</span>
             <div>Custom Numpy Operators</div>
         </div>
      </a>
   </div>
             <footer class="site-footer h-card">
     <div class="wrapper">
         <div class="row">
             <div class="col-4">
                 <h4 class="footer-category-title">Resources</h4>
                 <ul class="contact-list">
                     <li><a class="u-email" href="mailto:dev@mxnet.apache.org">Dev list</a></li>
                     <li><a class="u-email" href="mailto:user@mxnet.apache.org">User mailing list</a></li>
                     <li><a href="https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home">Developer Wiki</a></li>
                     <li><a href="https://issues.apache.org/jira/projects/MXNET/issues">Jira Tracker</a></li>
                     <li><a href="https://github.com/apache/mxnet/labels/Roadmap">Github Roadmap</a></li>
                     <li><a href="https://medium.com/apache-mxnet">Blog</a></li>
                     <li><a href="https://discuss.mxnet.io">Forum</a></li>
                     <li><a href="/community/contribute">Contribute</a></li>

                 </ul>
             </div>

             <div class="col-4"><ul class="social-media-list"><li><a href="https://github.com/apache/mxnet"><svg class="svg-icon"><use xlink:href="../../_static/minima-social-icons.svg#github"></use></svg> <span class="username">apache/mxnet</span></a></li><li><a href="https://www.twitter.com/apachemxnet"><svg class="svg-icon"><use xlink:href="../../_static/minima-social-icons.svg#twitter"></use></svg> <span class="username">apachemxnet</span></a></li><li><a href="https://youtube.com/apachemxnet"><svg class="svg-icon"><use xlink:href="../../_static/minima-social-icons.svg#youtube"></use></svg> <span class="username">apachemxnet</span></a></li></ul>
 </div>

             <div class="col-4 footer-text">
                 <p>A flexible and efficient library for deep learning.</p>
             </div>
         </div>
     </div>
 </footer>

 <footer class="site-footer2">
     <div class="wrapper">
         <div class="row">
             <div class="col-3">
                 <img src="../../_static/apache_incubator_logo.png" class="footer-logo col-2">
             </div>
             <div class="footer-bottom-warning col-9">
                 <p>Apache MXNet is an effort undergoing incubation at <a href="http://www.apache.org/">The Apache Software Foundation</a> (ASF), <span style="font-weight:bold">sponsored by the <i>Apache Incubator</i></span>. Incubation is required
                     of all newly accepted projects until a further review indicates that the infrastructure,
                     communications, and decision making process have stabilized in a manner consistent with other
                     successful ASF projects. While incubation status is not necessarily a reflection of the completeness
                     or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
                 </p><p>"Copyright © 2017-2018, The Apache Software Foundation Apache MXNet, MXNet, Apache, the Apache
                     feather, and the Apache MXNet project logo are either registered trademarks or trademarks of the
                     Apache Software Foundation."</p>
             </div>
         </div>
     </div>
 </footer>

   </body>
 </html>