versions/1.9.1/api/python/docs/tutorials/packages/gluon/data/datasets.html - mxnet-site - Git at Google

 <!DOCTYPE html>

 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
     <meta charset="utf-8" />
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     <meta http-equiv="x-ua-compatible" content="ie=edge">
     <style>
     .dropdown {
         position: relative;
         display: inline-block;
     }

     .dropdown-content {
         display: none;
         position: absolute;
         background-color: #f9f9f9;
         min-width: 160px;
         box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
         padding: 12px 16px;
         z-index: 1;
         text-align: left;
     }

     .dropdown:hover .dropdown-content {
         display: block;
     }

     .dropdown-option:hover {
         color: #FF4500;
     }

     .dropdown-option-active {
         color: #FF4500;
         font-weight: lighter;
     }

     .dropdown-option {
         color: #000000;
         font-weight: lighter;
     }

     .dropdown-header {
         color: #FFFFFF;
         display: inline-flex;
     }

     .dropdown-caret {
         width: 18px;
     }

     .dropdown-caret-path {
         fill: #FFFFFF;
     }
     </style>

     <title>Gluon Datasets and DataLoader &#8212; Apache MXNet  documentation</title>

     <link rel="stylesheet" href="../../../../_static/basic.css" type="text/css" />
     <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
     <link rel="stylesheet" type="text/css" href="../../../../_static/mxnet.css" />
     <link rel="stylesheet" href="../../../../_static/material-design-lite-1.3.0/material.blue-deep_orange.min.css" type="text/css" />
     <link rel="stylesheet" href="../../../../_static/sphinx_materialdesign_theme.css" type="text/css" />
     <link rel="stylesheet" href="../../../../_static/fontawesome/all.css" type="text/css" />
     <link rel="stylesheet" href="../../../../_static/fonts.css" type="text/css" />
     <link rel="stylesheet" href="../../../../_static/feedback.css" type="text/css" />
     <script id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
     <script src="../../../../_static/jquery.js"></script>
     <script src="../../../../_static/underscore.js"></script>
     <script src="../../../../_static/doctools.js"></script>
     <script src="../../../../_static/language_data.js"></script>
     <script src="../../../../_static/matomo_analytics.js"></script>
     <script src="../../../../_static/autodoc.js"></script>
     <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
     <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true, "ignoreClass": "document", "processClass": "math|output_area"}})</script>
     <link rel="shortcut icon" href="../../../../_static/mxnet-icon.png"/>
     <link rel="index" title="Index" href="../../../../genindex.html" />
     <link rel="search" title="Search" href="../../../../search.html" />
     <link rel="next" title="Image Tutorials" href="../image/index.html" />
     <link rel="prev" title="Image Augmentation" href="data_augmentation.html" />
   </head>
 <body><header class="site-header" role="banner">
   <div class="wrapper">
       <a class="site-title" rel="author" href="/versions/1.9.1/"><img
             src="../../../../_static/mxnet_logo.png" class="site-header-logo"></a>
     <nav class="site-nav">
       <input type="checkbox" id="nav-trigger" class="nav-trigger"/>
       <label for="nav-trigger">
           <span class="menu-icon">
             <svg viewBox="0 0 18 15" width="18px" height="15px">
               <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
             </svg>
           </span>
       </label>

       <div class="trigger">
         <a class="page-link" href="/versions/1.9.1/get_started">Get Started</a>
         <a class="page-link" href="/versions/1.9.1/features">Features</a>
         <a class="page-link" href="/versions/1.9.1/ecosystem">Ecosystem</a>
         <a class="page-link page-current" href="/versions/1.9.1/api">Docs & Tutorials</a>
         <a class="page-link" href="/versions/1.9.1/trusted_by">Trusted By</a>
         <a class="page-link" href="https://github.com/apache/mxnet">GitHub</a>
         <div class="dropdown" style="min-width:100px">
           <span class="dropdown-header">Apache
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content" style="min-width:250px">
             <a href="https://www.apache.org/foundation/">Apache Software Foundation</a>
             <a href="https://incubator.apache.org/">Apache Incubator</a>
             <a href="https://www.apache.org/licenses/">License</a>
             <a href="/versions/1.9.1/api/faq/security.html">Security</a>
             <a href="https://privacy.apache.org/policies/privacy-policy-public.html">Privacy</a>
             <a href="https://www.apache.org/events/current-event">Events</a>
             <a href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a>
             <a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
           </div>
         </div>
         <div class="dropdown">
           <span class="dropdown-header">1.9.1
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content">
             <a class="dropdown-option" href="/">master</a><br>
             <a class="dropdown-option-active" href="/versions/1.9.1/">1.9.1</a><br>
             <a class="dropdown-option" href="/versions/1.8.0/">1.8.0</a><br>
             <a class="dropdown-option" href="/versions/1.7.0/">1.7.0</a><br>
             <a class="dropdown-option" href="/versions/1.6.0/">1.6.0</a><br>
             <a class="dropdown-option" href="/versions/1.5.0/">1.5.0</a><br>
             <a class="dropdown-option" href="/versions/1.4.1/">1.4.1</a><br>
             <a class="dropdown-option" href="/versions/1.3.1/">1.3.1</a><br>
             <a class="dropdown-option" href="/versions/1.2.1/">1.2.1</a><br>
             <a class="dropdown-option" href="/versions/1.1.0/">1.1.0</a><br>
             <a class="dropdown-option" href="/versions/1.0.0/">1.0.0</a><br>
             <a class="dropdown-option" href="/versions/0.12.1/">0.12.1</a><br>
             <a class="dropdown-option" href="/versions/0.11.0/">0.11.0</a>
           </div>
         </div>
       </div>
     </nav>
   </div>
 </header>
     <div class="mdl-layout mdl-js-layout mdl-layout--fixed-header mdl-layout--fixed-drawer"><header class="mdl-layout__header mdl-layout__header--waterfall ">
     <div class="mdl-layout__header-row">

         <nav class="mdl-navigation breadcrumb">
             <a class="mdl-navigation__link" href="../../../index.html">Python Tutorials</a><i class="material-icons">navigate_next</i>
             <a class="mdl-navigation__link" href="../../index.html">Packages</a><i class="material-icons">navigate_next</i>
             <a class="mdl-navigation__link" href="../index.html">Gluon</a><i class="material-icons">navigate_next</i>
             <a class="mdl-navigation__link" href="index.html">Data Tutorials</a><i class="material-icons">navigate_next</i>
             <a class="mdl-navigation__link is-active">Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a>
         </nav>
         <div class="mdl-layout-spacer"></div>
         <nav class="mdl-navigation">

 <form class="form-inline pull-sm-right" action="../../../../search.html" method="get">
       <div class="mdl-textfield mdl-js-textfield mdl-textfield--expandable mdl-textfield--floating-label mdl-textfield--align-right">
         <label id="quick-search-icon" class="mdl-button mdl-js-button mdl-button--icon"  for="waterfall-exp">
           <i class="material-icons">search</i>
         </label>
         <div class="mdl-textfield__expandable-holder">
           <input class="mdl-textfield__input" type="text" name="q"  id="waterfall-exp" placeholder="Search" />
           <input type="hidden" name="check_keywords" value="yes" />
           <input type="hidden" name="area" value="default" />
         </div>
       </div>
       <div class="mdl-tooltip" data-mdl-for="quick-search-icon">
       Quick search
       </div>
 </form>

 <a id="button-show-source"
     class="mdl-button mdl-js-button mdl-button--icon"
     href="../../../../_sources/tutorials/packages/gluon/data/datasets.ipynb" rel="nofollow">
   <i class="material-icons">code</i>
 </a>
 <div class="mdl-tooltip" data-mdl-for="button-show-source">
 Show Source
 </div>
         </nav>
     </div>
     <div class="mdl-layout__header-row header-links">
       <div class="mdl-layout-spacer"></div>
       <nav class="mdl-navigation">
       </nav>
     </div>
 </header><header class="mdl-layout__drawer">

       <div class="globaltoc">
         <span class="mdl-layout-title toc">Table Of Contents</span>


             <nav class="mdl-navigation">
                 <ul class="current">
 <li class="toctree-l1 current"><a class="reference internal" href="../../../index.html">Python Tutorials</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="../../../getting-started/index.html">Getting Started</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/crash-course/index.html">Crash Course</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/1-ndarray.html">Manipulate data with <code class="docutils literal notranslate"><span class="pre">ndarray</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/2-nn.html">Create a neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/3-autograd.html">Automatic differentiation with <code class="docutils literal notranslate"><span class="pre">autograd</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/4-train.html">Train the neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/5-predict.html">Predict with a pre-trained model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/6-use_gpus.html">Use GPUs</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/to-mxnet/index.html">Moving to MXNet from Other Frameworks</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/to-mxnet/pytorch.html">PyTorch vs Apache MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/gluon_from_experiment_to_deployment.html">Gluon: from experiment to deployment</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/logistic_regression_explained.html">Logistic regression explained</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/image/mnist.html">MNIST</a></li>
 </ul>
 </li>
 <li class="toctree-l2 current"><a class="reference internal" href="../../index.html">Packages</a><ul class="current">
 <li class="toctree-l3"><a class="reference internal" href="../../autograd/index.html">Automatic Differentiation</a></li>
 <li class="toctree-l3 current"><a class="reference internal" href="../index.html">Gluon</a><ul class="current">
 <li class="toctree-l4"><a class="reference internal" href="../blocks/index.html">Blocks</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/custom-layer.html">Custom Layers</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/custom_layer_beginners.html">Customer Layers (Beginners)</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/hybridize.html">Hybridize</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/init.html">Initialization</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/naming.html">Parameter and Block Naming</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/nn.html">Layers and Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/parameters.html">Parameter Management</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/save_load_params.html">Saving and Loading Gluon Models</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/activations/activations.html">Activation Blocks</a></li>
 </ul>
 </li>
 <li class="toctree-l4 current"><a class="reference internal" href="index.html">Data Tutorials</a><ul class="current">
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html#Spatial-Augmentation">Spatial Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html#Color-Augmentation">Color Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html#Composed-Augmentations">Composed Augmentations</a></li>
 <li class="toctree-l5 current"><a class="current reference internal" href="#">Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 <li class="toctree-l5"><a class="reference internal" href="#Using-own-data-with-included-Datasets">Using own data with included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="#Using-own-data-with-custom-Datasets">Using own data with custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="#Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader">Appendix: Upgrading from Module <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> to Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../image/index.html">Image Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../image/image-augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../image/info_gan.html">Image similarity search with InfoGAN</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../image/mnist.html">Handwritten Digit Recognition</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../image/pretrained_models.html">Using pre-trained models in MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../loss/index.html">Losses</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../loss/custom-loss.html">Custom Loss Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../loss/kl_divergence.html">Kullback-Leibler (KL) Divergence</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../loss/loss.html">Loss functions</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../text/index.html">Text Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../text/gnmt.html">Google Neural Machine Translation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../text/transformer.html">Machine Translation with Transformer</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../training/index.html">Training</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../training/fit_api_tutorial.html">MXNet Gluon Fit API</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../training/trainer.html">Trainer</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../training/learning_rates/index.html">Learning Rates</a><ul>
 <li class="toctree-l6"><a class="reference internal" href="../training/learning_rates/learning_rate_finder.html">Learning Rate Finder</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../training/learning_rates/learning_rate_schedules.html">Learning Rate Schedules</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../training/learning_rates/learning_rate_schedules_advanced.html">Advanced Learning Rate Schedules</a></li>
 </ul>
 </li>
 <li class="toctree-l5"><a class="reference internal" href="../training/normalization/index.html">Normalization Blocks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../kvstore/index.html">KVStore</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../kvstore/kvstore.html">Distributed Key-Value Store</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../ndarray/index.html">NDArray</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/01-ndarray-intro.html">An Intro: Manipulate Data the MXNet Way with NDArray</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/02-ndarray-operations.html">NDArray Operations</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/03-ndarray-contexts.html">NDArray Contexts</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/gotchas_numpy_in_mxnet.html">Gotchas using NumPy in Apache MXNet</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/sparse/index.html">Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/csr.html">CSRNDArray - NDArray in Compressed Sparse Row Storage Format</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/row_sparse.html">RowSparseNDArray - NDArray for Sparse Gradient Updates</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/train.html">Train a Linear Regression Model with Sparse Symbols</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/train_gluon.html">Sparse NDArrays with Gluon</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../onnx/index.html">ONNX</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../onnx/fine_tuning_gluon.html">Fine-tuning an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../onnx/inference_on_onnx_model.html">Running inference on MXNet/Gluon from an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../onnx/super_resolution.html">Importing an ONNX model into MXNet</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/deploy/export/onnx.html">Export ONNX Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../optimizer/index.html">Optimizers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../viz/index.html">Visualization</a><ul>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/visualize_graph">Visualize networks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../performance/index.html">Performance</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../performance/compression/index.html">Compression</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/compression/int8.html">Deploy with int-8</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/float16">Float16</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/gradient_compression">Gradient Compression</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html">GluonCV with Quantized Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../performance/backend/index.html">Accelerated Backend Tools</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/mkldnn/index.html">Intel MKL-DNN</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/mkldnn/mkldnn_quantization.html">Quantize with MKL-DNN backend</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/mkldnn/mkldnn_quantization.html#Improving-accuracy-with-Intel®-Neural-Compressor">Improving accuracy with Intel® Neural Compressor</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/mkldnn/mkldnn_readme.html">Install MXNet with MKL-DNN</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/tensorrt/index.html">TensorRT</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/tensorrt/tensorrt.html">Optimizing Deep Learning Computation Graphs with TensorRT</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/tvm.html">Use TVM</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/profiler.html">Profiling MXNet Models</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/amp.html">Using AMP: Automatic Mixed Precision</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../deploy/index.html">Deployment</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../deploy/export/index.html">Export</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/export/onnx.html">Exporting to ONNX format</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html">Export Gluon CV Models</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html">Save / Load Parameters</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../deploy/inference/index.html">Inference</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/cpp.html">Deploy into C++</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/image_classification_jetson.html">Image Classication using pretrained ResNet-50 model on Jetson module</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/scala.html">Deploy into a Java or Scala Environment</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/wine_detector.html">Real-time Object Detection with MXNet On The Raspberry Pi</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../deploy/run-on-aws/index.html">Run on AWS</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/run-on-aws/use_ec2.html">Run on an EC2 Instance</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/run-on-aws/use_sagemaker.html">Run on Amazon SageMaker</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/run-on-aws/cloud.html">MXNet on the Cloud</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../extend/index.html">Extend</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../extend/custom_layer.html">Custom Layers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../extend/customop.html">Custom Numpy Operators</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/new_op">New Operator Creation</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/add_op_in_backend">New Operator in MXNet Backend</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../api/index.html">Python API</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/ndarray/index.html">mxnet.ndarray</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/ndarray.html">ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/contrib/index.html">ndarray.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/image/index.html">ndarray.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/linalg/index.html">ndarray.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/op/index.html">ndarray.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/random/index.html">ndarray.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/register/index.html">ndarray.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/sparse/index.html">ndarray.sparse</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/utils/index.html">ndarray.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/gluon/index.html">mxnet.gluon</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/block.html">gluon.Block</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/hybrid_block.html">gluon.HybridBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/symbol_block.html">gluon.SymbolBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/constant.html">gluon.Constant</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/parameter.html">gluon.Parameter</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/parameter_dict.html">gluon.ParameterDict</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/trainer.html">gluon.Trainer</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/contrib/index.html">gluon.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/data/index.html">gluon.data</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../../api/gluon/data/vision/index.html">data.vision</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../../../api/gluon/data/vision/datasets/index.html">vision.datasets</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../../../api/gluon/data/vision/transforms/index.html">vision.transforms</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/loss/index.html">gluon.loss</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/model_zoo/index.html">gluon.model_zoo.vision</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/nn/index.html">gluon.nn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/rnn/index.html">gluon.rnn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/utils/index.html">gluon.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/autograd/index.html">mxnet.autograd</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/initializer/index.html">mxnet.initializer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/optimizer/index.html">mxnet.optimizer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/lr_scheduler/index.html">mxnet.lr_scheduler</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/metric/index.html">mxnet.metric</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/kvstore/index.html">mxnet.kvstore</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/symbol/index.html">mxnet.symbol</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/symbol.html">symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/contrib/index.html">symbol.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/image/index.html">symbol.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/linalg/index.html">symbol.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/op/index.html">symbol.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/random/index.html">symbol.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/register/index.html">symbol.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/sparse/index.html">symbol.sparse</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/module/index.html">mxnet.module</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/contrib/index.html">mxnet.contrib</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/autograd/index.html">contrib.autograd</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/io/index.html">contrib.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/ndarray/index.html">contrib.ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/onnx/index.html">contrib.onnx</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/quantization/index.html">contrib.quantization</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/symbol/index.html">contrib.symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/tensorboard/index.html">contrib.tensorboard</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/tensorrt/index.html">contrib.tensorrt</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/text/index.html">contrib.text</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/mxnet/index.html">mxnet</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/attribute/index.html">mxnet.attribute</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/base/index.html">mxnet.base</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/callback/index.html">mxnet.callback</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/context/index.html">mxnet.context</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/engine/index.html">mxnet.engine</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/executor/index.html">mxnet.executor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/executor_manager/index.html">mxnet.executor_manager</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/image/index.html">mxnet.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/io/index.html">mxnet.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/kvstore_server/index.html">mxnet.kvstore_server</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/libinfo/index.html">mxnet.libinfo</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/log/index.html">mxnet.log</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/model/index.html">mxnet.model</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/monitor/index.html">mxnet.monitor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/name/index.html">mxnet.name</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/notebook/index.html">mxnet.notebook</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/operator/index.html">mxnet.operator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/profiler/index.html">mxnet.profiler</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/random/index.html">mxnet.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/recordio/index.html">mxnet.recordio</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/registry/index.html">mxnet.registry</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/rtc/index.html">mxnet.rtc</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/runtime/index.html">mxnet.runtime</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/test_utils/index.html">mxnet.test_utils</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/torch/index.html">mxnet.torch</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/util/index.html">mxnet.util</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/visualization/index.html">mxnet.visualization</a></li>
 </ul>
 </li>
 </ul>
 </li>
 </ul>

             </nav>

         </div>

 </header>
         <main class="mdl-layout__content" tabIndex="0">

         <script type="text/javascript" src="../../../../_static/sphinx_materialdesign_theme.js "></script>
         <script type="text/javascript" src="../../../../_static/feedback.js"></script>
     <header class="mdl-layout__drawer">

       <div class="globaltoc">
         <span class="mdl-layout-title toc">Table Of Contents</span>


             <nav class="mdl-navigation">
                 <ul class="current">
 <li class="toctree-l1 current"><a class="reference internal" href="../../../index.html">Python Tutorials</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="../../../getting-started/index.html">Getting Started</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/crash-course/index.html">Crash Course</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/1-ndarray.html">Manipulate data with <code class="docutils literal notranslate"><span class="pre">ndarray</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/2-nn.html">Create a neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/3-autograd.html">Automatic differentiation with <code class="docutils literal notranslate"><span class="pre">autograd</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/4-train.html">Train the neural network</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/5-predict.html">Predict with a pre-trained model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/crash-course/6-use_gpus.html">Use GPUs</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/to-mxnet/index.html">Moving to MXNet from Other Frameworks</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../getting-started/to-mxnet/pytorch.html">PyTorch vs Apache MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/gluon_from_experiment_to_deployment.html">Gluon: from experiment to deployment</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../getting-started/logistic_regression_explained.html">Logistic regression explained</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/image/mnist.html">MNIST</a></li>
 </ul>
 </li>
 <li class="toctree-l2 current"><a class="reference internal" href="../../index.html">Packages</a><ul class="current">
 <li class="toctree-l3"><a class="reference internal" href="../../autograd/index.html">Automatic Differentiation</a></li>
 <li class="toctree-l3 current"><a class="reference internal" href="../index.html">Gluon</a><ul class="current">
 <li class="toctree-l4"><a class="reference internal" href="../blocks/index.html">Blocks</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/custom-layer.html">Custom Layers</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/custom_layer_beginners.html">Customer Layers (Beginners)</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/hybridize.html">Hybridize</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/init.html">Initialization</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/naming.html">Parameter and Block Naming</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/nn.html">Layers and Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/parameters.html">Parameter Management</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/save_load_params.html">Saving and Loading Gluon Models</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../blocks/activations/activations.html">Activation Blocks</a></li>
 </ul>
 </li>
 <li class="toctree-l4 current"><a class="reference internal" href="index.html">Data Tutorials</a><ul class="current">
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html#Spatial-Augmentation">Spatial Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html#Color-Augmentation">Color Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="data_augmentation.html#Composed-Augmentations">Composed Augmentations</a></li>
 <li class="toctree-l5 current"><a class="current reference internal" href="#">Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 <li class="toctree-l5"><a class="reference internal" href="#Using-own-data-with-included-Datasets">Using own data with included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="#Using-own-data-with-custom-Datasets">Using own data with custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li class="toctree-l5"><a class="reference internal" href="#Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader">Appendix: Upgrading from Module <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> to Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../image/index.html">Image Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../image/image-augmentation.html">Image Augmentation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../image/info_gan.html">Image similarity search with InfoGAN</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../image/mnist.html">Handwritten Digit Recognition</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../image/pretrained_models.html">Using pre-trained models in MXNet</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../loss/index.html">Losses</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../loss/custom-loss.html">Custom Loss Blocks</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../loss/kl_divergence.html">Kullback-Leibler (KL) Divergence</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../loss/loss.html">Loss functions</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../text/index.html">Text Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../text/gnmt.html">Google Neural Machine Translation</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../text/transformer.html">Machine Translation with Transformer</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../training/index.html">Training</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../training/fit_api_tutorial.html">MXNet Gluon Fit API</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../training/trainer.html">Trainer</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../training/learning_rates/index.html">Learning Rates</a><ul>
 <li class="toctree-l6"><a class="reference internal" href="../training/learning_rates/learning_rate_finder.html">Learning Rate Finder</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../training/learning_rates/learning_rate_schedules.html">Learning Rate Schedules</a></li>
 <li class="toctree-l6"><a class="reference internal" href="../training/learning_rates/learning_rate_schedules_advanced.html">Advanced Learning Rate Schedules</a></li>
 </ul>
 </li>
 <li class="toctree-l5"><a class="reference internal" href="../training/normalization/index.html">Normalization Blocks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../kvstore/index.html">KVStore</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../kvstore/kvstore.html">Distributed Key-Value Store</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../ndarray/index.html">NDArray</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/01-ndarray-intro.html">An Intro: Manipulate Data the MXNet Way with NDArray</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/02-ndarray-operations.html">NDArray Operations</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/03-ndarray-contexts.html">NDArray Contexts</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/gotchas_numpy_in_mxnet.html">Gotchas using NumPy in Apache MXNet</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../ndarray/sparse/index.html">Tutorials</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/csr.html">CSRNDArray - NDArray in Compressed Sparse Row Storage Format</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/row_sparse.html">RowSparseNDArray - NDArray for Sparse Gradient Updates</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/train.html">Train a Linear Regression Model with Sparse Symbols</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../ndarray/sparse/train_gluon.html">Sparse NDArrays with Gluon</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../onnx/index.html">ONNX</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../onnx/fine_tuning_gluon.html">Fine-tuning an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../onnx/inference_on_onnx_model.html">Running inference on MXNet/Gluon from an ONNX model</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../onnx/super_resolution.html">Importing an ONNX model into MXNet</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/deploy/export/onnx.html">Export ONNX Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../optimizer/index.html">Optimizers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../viz/index.html">Visualization</a><ul>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/visualize_graph">Visualize networks</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../performance/index.html">Performance</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../performance/compression/index.html">Compression</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/compression/int8.html">Deploy with int-8</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/float16">Float16</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/faq/gradient_compression">Gradient Compression</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html">GluonCV with Quantized Models</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../performance/backend/index.html">Accelerated Backend Tools</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/mkldnn/index.html">Intel MKL-DNN</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/mkldnn/mkldnn_quantization.html">Quantize with MKL-DNN backend</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/mkldnn/mkldnn_quantization.html#Improving-accuracy-with-Intel®-Neural-Compressor">Improving accuracy with Intel® Neural Compressor</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/mkldnn/mkldnn_readme.html">Install MXNet with MKL-DNN</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/tensorrt/index.html">TensorRT</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../../performance/backend/tensorrt/tensorrt.html">Optimizing Deep Learning Computation Graphs with TensorRT</a></li>
 </ul>
 </li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/tvm.html">Use TVM</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/profiler.html">Profiling MXNet Models</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../performance/backend/amp.html">Using AMP: Automatic Mixed Precision</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../deploy/index.html">Deployment</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../deploy/export/index.html">Export</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/export/onnx.html">Exporting to ONNX format</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html">Export Gluon CV Models</a></li>
 <li class="toctree-l4"><a class="reference external" href="https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html">Save / Load Parameters</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../deploy/inference/index.html">Inference</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/cpp.html">Deploy into C++</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/image_classification_jetson.html">Image Classication using pretrained ResNet-50 model on Jetson module</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/scala.html">Deploy into a Java or Scala Environment</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/inference/wine_detector.html">Real-time Object Detection with MXNet On The Raspberry Pi</a></li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../deploy/run-on-aws/index.html">Run on AWS</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/run-on-aws/use_ec2.html">Run on an EC2 Instance</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/run-on-aws/use_sagemaker.html">Run on Amazon SageMaker</a></li>
 <li class="toctree-l4"><a class="reference internal" href="../../../deploy/run-on-aws/cloud.html">MXNet on the Cloud</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../extend/index.html">Extend</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../extend/custom_layer.html">Custom Layers</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../extend/customop.html">Custom Numpy Operators</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/new_op">New Operator Creation</a></li>
 <li class="toctree-l3"><a class="reference external" href="https://mxnet.apache.org/api/faq/add_op_in_backend">New Operator in MXNet Backend</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="../../../../api/index.html">Python API</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/ndarray/index.html">mxnet.ndarray</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/ndarray.html">ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/contrib/index.html">ndarray.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/image/index.html">ndarray.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/linalg/index.html">ndarray.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/op/index.html">ndarray.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/random/index.html">ndarray.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/register/index.html">ndarray.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/sparse/index.html">ndarray.sparse</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/ndarray/utils/index.html">ndarray.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/gluon/index.html">mxnet.gluon</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/block.html">gluon.Block</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/hybrid_block.html">gluon.HybridBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/symbol_block.html">gluon.SymbolBlock</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/constant.html">gluon.Constant</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/parameter.html">gluon.Parameter</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/parameter_dict.html">gluon.ParameterDict</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/trainer.html">gluon.Trainer</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/contrib/index.html">gluon.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/data/index.html">gluon.data</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="../../../../api/gluon/data/vision/index.html">data.vision</a><ul>
 <li class="toctree-l5"><a class="reference internal" href="../../../../api/gluon/data/vision/datasets/index.html">vision.datasets</a></li>
 <li class="toctree-l5"><a class="reference internal" href="../../../../api/gluon/data/vision/transforms/index.html">vision.transforms</a></li>
 </ul>
 </li>
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/loss/index.html">gluon.loss</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/model_zoo/index.html">gluon.model_zoo.vision</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/nn/index.html">gluon.nn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/rnn/index.html">gluon.rnn</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/gluon/utils/index.html">gluon.utils</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/autograd/index.html">mxnet.autograd</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/initializer/index.html">mxnet.initializer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/optimizer/index.html">mxnet.optimizer</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/lr_scheduler/index.html">mxnet.lr_scheduler</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/metric/index.html">mxnet.metric</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/kvstore/index.html">mxnet.kvstore</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/symbol/index.html">mxnet.symbol</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/symbol.html">symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/contrib/index.html">symbol.contrib</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/image/index.html">symbol.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/linalg/index.html">symbol.linalg</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/op/index.html">symbol.op</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/random/index.html">symbol.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/register/index.html">symbol.register</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/symbol/sparse/index.html">symbol.sparse</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/module/index.html">mxnet.module</a></li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/contrib/index.html">mxnet.contrib</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/autograd/index.html">contrib.autograd</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/io/index.html">contrib.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/ndarray/index.html">contrib.ndarray</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/onnx/index.html">contrib.onnx</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/quantization/index.html">contrib.quantization</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/symbol/index.html">contrib.symbol</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/tensorboard/index.html">contrib.tensorboard</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/tensorrt/index.html">contrib.tensorrt</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/contrib/text/index.html">contrib.text</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="../../../../api/mxnet/index.html">mxnet</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/attribute/index.html">mxnet.attribute</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/base/index.html">mxnet.base</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/callback/index.html">mxnet.callback</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/context/index.html">mxnet.context</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/engine/index.html">mxnet.engine</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/executor/index.html">mxnet.executor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/executor_manager/index.html">mxnet.executor_manager</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/image/index.html">mxnet.image</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/io/index.html">mxnet.io</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/kvstore_server/index.html">mxnet.kvstore_server</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/libinfo/index.html">mxnet.libinfo</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/log/index.html">mxnet.log</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/model/index.html">mxnet.model</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/monitor/index.html">mxnet.monitor</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/name/index.html">mxnet.name</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/notebook/index.html">mxnet.notebook</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/operator/index.html">mxnet.operator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/profiler/index.html">mxnet.profiler</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/random/index.html">mxnet.random</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/recordio/index.html">mxnet.recordio</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/registry/index.html">mxnet.registry</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/rtc/index.html">mxnet.rtc</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/runtime/index.html">mxnet.runtime</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/test_utils/index.html">mxnet.test_utils</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/torch/index.html">mxnet.torch</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/util/index.html">mxnet.util</a></li>
 <li class="toctree-l3"><a class="reference internal" href="../../../../api/mxnet/visualization/index.html">mxnet.visualization</a></li>
 </ul>
 </li>
 </ul>
 </li>
 </ul>

             </nav>

         </div>

 </header>

     <div class="document">
         <div class="page-content" role="main">

   <!--- Licensed to the Apache Software Foundation (ASF) under one --><!--- or more contributor license agreements.  See the NOTICE file --><!--- distributed with this work for additional information --><!--- regarding copyright ownership.  The ASF licenses this file --><!--- to you under the Apache License, Version 2.0 (the --><!--- "License"); you may not use this file except in compliance --><!--- with the License.  You may obtain a copy of the License at --><!---   http://www.apache.org/licenses/LICENSE-2.0 --><!--- Unless required by applicable law or agreed to in writing, --><!--- software distributed under the License is distributed on an --><!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --><!--- KIND, either express or implied.  See the License for the --><!--- specific language governing permissions and limitations --><!--- under the License. --><div class="section" id="Gluon-Datasets-and-DataLoader">
 <h1>Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code><a class="headerlink" href="#Gluon-Datasets-and-DataLoader" title="Permalink to this headline">¶</a></h1>
 <p>One of the most critical steps for model training and inference is loading the data: without data you can’t do Machine Learning! In this tutorial we use the Gluon API to define a <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#datasets">Dataset</a> and use a <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#dataloader">DataLoader</a> to iterate through the dataset in mini-batches.</p>
 <div class="section" id="Introduction-to-Datasets">
 <h2>Introduction to <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s<a class="headerlink" href="#Introduction-to-Datasets" title="Permalink to this headline">¶</a></h2>
 <p><a class="reference external" href="/api/python/docs/api/gluon/data/index.html#datasets">Dataset</a> objects are used to represent collections of data, and include methods to load and parse the data (that is often stored on disk). Gluon has a number of different <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> classes for working with image data straight out-of-the-box, but we’ll use the <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#mxnet.gluon.data.ArrayDataset">ArrayDataset</a> to introduce the idea of a <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>.</p>
 <p>We first start by generating random data <code class="docutils literal notranslate"><span class="pre">X</span></code> (with 3 variables) and corresponding random labels <code class="docutils literal notranslate"><span class="pre">y</span></code> to simulate a typical supervised learning task. We generate 10 samples and we pass them all to the <code class="docutils literal notranslate"><span class="pre">ArrayDataset</span></code>.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">mxnet</span> <span class="k">as</span> <span class="nn">mx</span>
 <span class="kn">import</span> <span class="nn">os</span>
 <span class="kn">import</span> <span class="nn">tarfile</span>

 <span class="n">mx</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">42</span><span class="p">)</span> <span class="c1"># Fix the seed for reproducibility</span>
 <span class="n">X</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
 <span class="n">y</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
 <span class="n">dataset</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">ArrayDataset</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>A key feature of a <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> is the <strong>ability to retrieve a single sample given an index</strong>. Our random data and labels were generated in memory, so this <code class="docutils literal notranslate"><span class="pre">ArrayDataset</span></code> doesn’t have to load anything from disk, but the interface is the same for all <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>’s.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sample_idx</span> <span class="o">=</span> <span class="mi">4</span>
 <span class="n">sample</span> <span class="o">=</span> <span class="n">dataset</span><span class="p">[</span><span class="n">sample_idx</span><span class="p">]</span>

 <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span>
 <span class="k">assert</span> <span class="n">sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="p">)</span>
 <span class="k">assert</span> <span class="n">sample</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="p">)</span>
 <span class="nb">print</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>( [ 0.4375872 0.29753461 0.89177299] &lt;NDArray 3 &#64;cpu(0)&gt;, [ 0.83261985] &lt;NDArray 1 &#64;cpu(0)&gt;)</p>
 <p>We get a tuple of a data sample and its corresponding label, which makes sense because we passed the data <code class="docutils literal notranslate"><span class="pre">X</span></code> and the labels <code class="docutils literal notranslate"><span class="pre">y</span></code> in that order when we instantiated the <code class="docutils literal notranslate"><span class="pre">ArrayDataset</span></code>. We don’t usually retrieve individual samples from <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> objects though (unless we’re quality checking the output samples). Instead we use a <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code>.</p>
 </div>
 <div class="section" id="Introduction-to-DataLoader">
 <h2>Introduction to <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code><a class="headerlink" href="#Introduction-to-DataLoader" title="Permalink to this headline">¶</a></h2>
 <p>A <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#dataloader">DataLoader</a> is used to create mini-batches of samples from a <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#datasets">Dataset</a>, and provides a convenient iterator interface for looping these batches. It’s typically much more efficient to pass a mini-batch of data through a neural network than a single sample at a time, because the computation can be performed in parallel. A required parameter of <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code> is the size of the
 mini-batches you want to create, called <code class="docutils literal notranslate"><span class="pre">batch_size</span></code>.</p>
 <p>Another benefit of using <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code> is the ability to easily load data in parallel using <a class="reference external" href="https://docs.python.org/3.6/library/multiprocessing.html">multiprocessing</a>. You can set the <code class="docutils literal notranslate"><span class="pre">num_workers</span></code> parameter to the number of CPUs available on your machine for maximum performance, or limit it to a lower number to spare resources.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">multiprocessing</span> <span class="kn">import</span> <span class="n">cpu_count</span>
 <span class="n">CPU_COUNT</span> <span class="o">=</span> <span class="n">cpu_count</span><span class="p">()</span>

 <span class="n">data_loader</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="n">CPU_COUNT</span><span class="p">)</span>

 <span class="k">for</span> <span class="n">X_batch</span><span class="p">,</span> <span class="n">y_batch</span> <span class="ow">in</span> <span class="n">data_loader</span><span class="p">:</span>
     <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;X_batch has shape </span><span class="si">{}</span><span class="s2">, and y_batch has shape </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">X_batch</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="n">y_batch</span><span class="o">.</span><span class="n">shape</span><span class="p">))</span>
 </pre></div>
 </div>
 <p><code class="docutils literal notranslate"><span class="pre">X_batch</span> <span class="pre">has</span> <span class="pre">shape</span> <span class="pre">(5,</span> <span class="pre">3),</span> <span class="pre">and</span> <span class="pre">y_batch</span> <span class="pre">has</span> <span class="pre">shape</span> <span class="pre">(5,</span> <span class="pre">1)</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">X_batch</span> <span class="pre">has</span> <span class="pre">shape</span> <span class="pre">(5,</span> <span class="pre">3),</span> <span class="pre">and</span> <span class="pre">y_batch</span> <span class="pre">has</span> <span class="pre">shape</span> <span class="pre">(5,</span> <span class="pre">1)</span></code></p>
 <p>We can see 2 mini-batches of data (and labels), each with 5 samples, which makes sense given we started with a dataset of 10 samples. When comparing the shape of the batches to the samples returned by the <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>, we’ve gained an extra dimension at the start which is sometimes called the batch axis.</p>
 <p>Our <code class="docutils literal notranslate"><span class="pre">data_loader</span></code> loop will stop when every sample of <code class="docutils literal notranslate"><span class="pre">dataset</span></code> has been returned as part of a batch. Sometimes the dataset length isn’t divisible by the mini-batch size, leaving a final batch with a smaller number of samples. <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code>’s default behavior is to return this smaller mini-batch, but this can be changed by setting the <code class="docutils literal notranslate"><span class="pre">last_batch</span></code> parameter to <code class="docutils literal notranslate"><span class="pre">discard</span></code> (which ignores the last batch) or <code class="docutils literal notranslate"><span class="pre">rollover</span></code> (which starts the next epoch with the remaining samples).</p>
 </div>
 <div class="section" id="Machine-learning-with-Datasets-and-DataLoaders">
 <h2>Machine learning with <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code>s<a class="headerlink" href="#Machine-learning-with-Datasets-and-DataLoaders" title="Permalink to this headline">¶</a></h2>
 <p>You will often use a few different <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> objects in your Machine Learning project. It’s essential to separate your training dataset from testing dataset, and it’s also good practice to have validation dataset (a.k.a. development dataset) that can be used for optimising hyperparameters.</p>
 <p>Using Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> objects, we define the data to be included in each of these separate datasets. Common use cases for loading data are covered already (e.g. <a class="reference external" href="/api/python/docs/api/gluon/data/vision/index.html">mxnet.gluon.data.vision.datasets.ImageFolderDataset</a>), but it’s simple to create your own custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> classes for other types of data. You can even use included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> objects for common datasets if you want to experiment quickly; they download and parse the data for
 you! In this example we use the <a class="reference external" href="https://github.com/zalandoresearch/fashion-mnist">Fashion MNIST</a> dataset from Zalando Research.</p>
 <p>Many of the image <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>’s accept a function (via the optional <code class="docutils literal notranslate"><span class="pre">transform</span></code> parameter) which is applied to each sample returned by the <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>. It’s useful for performing data augmentation, but can also be used for more simple data type conversion and pixel value scaling as seen below.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">label</span><span class="p">):</span>
     <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">)</span><span class="o">/</span><span class="mi">255</span>
     <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">label</span>

 <span class="n">train_dataset</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">vision</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">FashionMNIST</span><span class="p">(</span><span class="n">train</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">transform</span><span class="o">=</span><span class="n">transform</span><span class="p">)</span>
 <span class="n">valid_dataset</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">vision</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">FashionMNIST</span><span class="p">(</span><span class="n">train</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">transform</span><span class="o">=</span><span class="n">transform</span><span class="p">)</span>
 </pre></div>
 </div>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="n">matplotlib</span> <span class="n">inline</span>
 <span class="kn">from</span> <span class="nn">matplotlib.pylab</span> <span class="kn">import</span> <span class="n">imshow</span>

 <span class="n">sample_idx</span> <span class="o">=</span> <span class="mi">234</span>
 <span class="n">sample</span> <span class="o">=</span> <span class="n">train_dataset</span><span class="p">[</span><span class="n">sample_idx</span><span class="p">]</span>
 <span class="n">data</span> <span class="o">=</span> <span class="n">sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
 <span class="n">label</span> <span class="o">=</span> <span class="n">sample</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
 <span class="n">label_desc</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span><span class="s1">&#39;T-shirt/top&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span><span class="s1">&#39;Trouser&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span><span class="s1">&#39;Pullover&#39;</span><span class="p">,</span> <span class="mi">3</span><span class="p">:</span><span class="s1">&#39;Dress&#39;</span><span class="p">,</span> <span class="mi">4</span><span class="p">:</span><span class="s1">&#39;Coat&#39;</span><span class="p">,</span> <span class="mi">5</span><span class="p">:</span><span class="s1">&#39;Sandal&#39;</span><span class="p">,</span> <span class="mi">6</span><span class="p">:</span><span class="s1">&#39;Shirt&#39;</span><span class="p">,</span> <span class="mi">7</span><span class="p">:</span><span class="s1">&#39;Sneaker&#39;</span><span class="p">,</span> <span class="mi">8</span><span class="p">:</span><span class="s1">&#39;Bag&#39;</span><span class="p">,</span> <span class="mi">9</span><span class="p">:</span><span class="s1">&#39;Ankle boot&#39;</span><span class="p">}</span>

 <span class="n">imshow</span><span class="p">(</span><span class="n">data</span><span class="p">[:,:,</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">asnumpy</span><span class="p">(),</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">&#39;gray&#39;</span><span class="p">)</span>
 <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Data type: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">dtype</span><span class="p">))</span>
 <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Label: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label</span><span class="p">))</span>
 <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Label description: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label_desc</span><span class="p">[</span><span class="n">label</span><span class="p">]))</span>
 </pre></div>
 </div>
 <p><code class="docutils literal notranslate"><span class="pre">Data</span> <span class="pre">type:</span> <span class="pre">&lt;class</span> <span class="pre">'numpy.float32'&gt;</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">Label:</span> <span class="pre">8</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">Label</span> <span class="pre">description:</span> <span class="pre">Bag</span></code></p>
 <p><img alt="png" src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/gluon/datasets/caltech101_face.png" /></p>
 <p>When training machine learning models it is important to shuffle the training samples every time you pass through the dataset (i.e. each epoch). Sometimes the order of your samples will have a spurious relationship with the target variable, and shuffling the samples helps remove this. With <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#dataloader">DataLoader</a> it’s as simple as adding <code class="docutils literal notranslate"><span class="pre">shuffle=True</span></code>. You don’t need to shuffle the validation and testing data though.</p>
 <p>If you have more complex shuffling requirements (e.g. when handling sequential data), take a look at <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#mxnet.gluon.data.BatchSampler">mxnet.gluon.data.BatchSampler</a> and pass this to your <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code> instead.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">32</span>
 <span class="n">train_data_loader</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span><span class="n">train_dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="n">CPU_COUNT</span><span class="p">)</span>
 <span class="n">valid_data_loader</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span><span class="n">valid_dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="n">CPU_COUNT</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>With both <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code>s defined, we can now train a model to classify each image and evaluate the validation loss at each epoch. Our Fashion MNIST dataset has 10 classes including shirt, dress, sneakers, etc. We define a simple fully connected network with a softmax output and use cross entropy as our loss.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mxnet</span> <span class="kn">import</span> <span class="n">gluon</span><span class="p">,</span> <span class="n">autograd</span><span class="p">,</span> <span class="n">ndarray</span>

 <span class="k">def</span> <span class="nf">construct_net</span><span class="p">():</span>
     <span class="n">net</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">HybridSequential</span><span class="p">()</span>
     <span class="k">with</span> <span class="n">net</span><span class="o">.</span><span class="n">name_scope</span><span class="p">():</span>
         <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">gluon</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">))</span>
         <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">gluon</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="mi">64</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">))</span>
         <span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">gluon</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
     <span class="k">return</span> <span class="n">net</span>

 <span class="c1"># construct and initialize network.</span>
 <span class="n">ctx</span> <span class="o">=</span>  <span class="n">mx</span><span class="o">.</span><span class="n">gpu</span><span class="p">()</span> <span class="k">if</span> <span class="n">mx</span><span class="o">.</span><span class="n">context</span><span class="o">.</span><span class="n">num_gpus</span><span class="p">()</span> <span class="k">else</span> <span class="n">mx</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span>

 <span class="n">net</span> <span class="o">=</span> <span class="n">construct_net</span><span class="p">()</span>
 <span class="n">net</span><span class="o">.</span><span class="n">hybridize</span><span class="p">()</span>
 <span class="n">net</span><span class="o">.</span><span class="n">initialize</span><span class="p">(</span><span class="n">mx</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">Xavier</span><span class="p">(),</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
 <span class="c1"># define loss and trainer.</span>
 <span class="n">criterion</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">loss</span><span class="o">.</span><span class="n">SoftmaxCrossEntropyLoss</span><span class="p">()</span>
 <span class="n">trainer</span> <span class="o">=</span> <span class="n">gluon</span><span class="o">.</span><span class="n">Trainer</span><span class="p">(</span><span class="n">net</span><span class="o">.</span><span class="n">collect_params</span><span class="p">(),</span> <span class="s1">&#39;sgd&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s1">&#39;learning_rate&#39;</span><span class="p">:</span> <span class="mf">0.1</span><span class="p">})</span>
 </pre></div>
 </div>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">epochs</span> <span class="o">=</span> <span class="mi">5</span>
 <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">epochs</span><span class="p">):</span>
     <span class="c1"># training loop (with autograd and trainer steps, etc.)</span>
     <span class="n">cumulative_train_loss</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
     <span class="n">training_samples</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">train_data_loader</span><span class="p">):</span>
         <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">as_in_context</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">784</span><span class="p">))</span> <span class="c1"># 28*28=784</span>
         <span class="n">label</span> <span class="o">=</span> <span class="n">label</span><span class="o">.</span><span class="n">as_in_context</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span>
         <span class="k">with</span> <span class="n">autograd</span><span class="o">.</span><span class="n">record</span><span class="p">():</span>
             <span class="n">output</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
             <span class="n">loss</span> <span class="o">=</span> <span class="n">criterion</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>
         <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
         <span class="n">trainer</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
         <span class="n">cumulative_train_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
         <span class="n">training_samples</span> <span class="o">+=</span> <span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
     <span class="n">train_loss</span> <span class="o">=</span> <span class="n">cumulative_train_loss</span><span class="o">.</span><span class="n">asscalar</span><span class="p">()</span><span class="o">/</span><span class="n">training_samples</span>

     <span class="c1"># validation loop</span>
     <span class="n">cumulative_valid_loss</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">ctx</span><span class="p">)</span>
     <span class="n">valid_samples</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">valid_data_loader</span><span class="p">):</span>
         <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">as_in_context</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">784</span><span class="p">))</span> <span class="c1"># 28*28=784</span>
         <span class="n">label</span> <span class="o">=</span> <span class="n">label</span><span class="o">.</span><span class="n">as_in_context</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span>
         <span class="n">output</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
         <span class="n">loss</span> <span class="o">=</span> <span class="n">criterion</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>
         <span class="n">cumulative_valid_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
         <span class="n">valid_samples</span> <span class="o">+=</span> <span class="n">data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
     <span class="n">valid_loss</span> <span class="o">=</span> <span class="n">cumulative_valid_loss</span><span class="o">.</span><span class="n">asscalar</span><span class="p">()</span><span class="o">/</span><span class="n">valid_samples</span>

     <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Epoch </span><span class="si">{}</span><span class="s2">, training loss: </span><span class="si">{:.2f}</span><span class="s2">, validation loss: </span><span class="si">{:.2f}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">valid_loss</span><span class="p">))</span>
 </pre></div>
 </div>
 <p><code class="docutils literal notranslate"><span class="pre">Epoch</span> <span class="pre">0,</span> <span class="pre">training</span> <span class="pre">loss:</span> <span class="pre">0.54,</span> <span class="pre">validation</span> <span class="pre">loss:</span> <span class="pre">0.45</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">...</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">Epoch</span> <span class="pre">4,</span> <span class="pre">training</span> <span class="pre">loss:</span> <span class="pre">0.32,</span> <span class="pre">validation</span> <span class="pre">loss:</span> <span class="pre">0.33</span></code></p>
 </div>
 </div>
 <div class="section" id="Using-own-data-with-included-Datasets">
 <h1>Using own data with included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s<a class="headerlink" href="#Using-own-data-with-included-Datasets" title="Permalink to this headline">¶</a></h1>
 <p>Gluon has a number of different <a class="reference external" href="https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=dataset#mxnet.gluon.data.Dataset">Dataset</a> classes for working with your own image data straight out-of-the-box. You can get started quickly using the <a class="reference external" href="/api/python/docs/api/gluon/data/vision/datasets/index.html#mxnet.gluon.data.vision.datasets.ImageFolderDataset">mxnet.gluon.data.vision.datasets.ImageFolderDataset</a> which loads images directly from a user-defined folder, and infers the
 label (i.e. class) from the folders.</p>
 <p>We will run through an example for image classification, but a similar process applies for other vision tasks. If you already have your own collection of images to work with you should partition your data into training and test sets, and place all objects of the same class into seperate folders. Similar to:</p>
 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">./</span><span class="n">images</span><span class="o">/</span><span class="n">train</span><span class="o">/</span><span class="n">car</span><span class="o">/</span><span class="n">abc</span><span class="o">.</span><span class="n">jpg</span>
 <span class="o">./</span><span class="n">images</span><span class="o">/</span><span class="n">train</span><span class="o">/</span><span class="n">car</span><span class="o">/</span><span class="n">efg</span><span class="o">.</span><span class="n">jpg</span>
 <span class="o">./</span><span class="n">images</span><span class="o">/</span><span class="n">train</span><span class="o">/</span><span class="n">bus</span><span class="o">/</span><span class="n">hij</span><span class="o">.</span><span class="n">jpg</span>
 <span class="o">./</span><span class="n">images</span><span class="o">/</span><span class="n">train</span><span class="o">/</span><span class="n">bus</span><span class="o">/</span><span class="n">klm</span><span class="o">.</span><span class="n">jpg</span>
 <span class="o">./</span><span class="n">images</span><span class="o">/</span><span class="n">test</span><span class="o">/</span><span class="n">car</span><span class="o">/</span><span class="n">xyz</span><span class="o">.</span><span class="n">jpg</span>
 <span class="o">./</span><span class="n">images</span><span class="o">/</span><span class="n">test</span><span class="o">/</span><span class="n">bus</span><span class="o">/</span><span class="n">uvw</span><span class="o">.</span><span class="n">jpg</span>
 </pre></div>
 </div>
 <p>You can download the Caltech 101 dataset if you don’t already have images to work with for this example, but please note the download is 126MB.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">data_folder</span> <span class="o">=</span> <span class="s2">&quot;data&quot;</span>
 <span class="n">dataset_name</span> <span class="o">=</span> <span class="s2">&quot;101_ObjectCategories&quot;</span>
 <span class="n">archive_file</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">.tar.gz&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">)</span>
 <span class="n">archive_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_folder</span><span class="p">,</span> <span class="n">archive_file</span><span class="p">)</span>
 <span class="n">data_url</span> <span class="o">=</span> <span class="s2">&quot;https://s3.us-east-2.amazonaws.com/mxnet-public/&quot;</span>

 <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">archive_path</span><span class="p">):</span>
     <span class="n">mx</span><span class="o">.</span><span class="n">test_utils</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data_url</span><span class="p">,</span> <span class="n">archive_file</span><span class="p">),</span> <span class="n">dirname</span> <span class="o">=</span> <span class="n">data_folder</span><span class="p">)</span>
     <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Extracting </span><span class="si">{}</span><span class="s1"> in </span><span class="si">{}</span><span class="s1">...&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_file</span><span class="p">,</span> <span class="n">data_folder</span><span class="p">))</span>
     <span class="n">tar</span> <span class="o">=</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">archive_path</span><span class="p">,</span> <span class="s2">&quot;r:gz&quot;</span><span class="p">)</span>
     <span class="n">tar</span><span class="o">.</span><span class="n">extractall</span><span class="p">(</span><span class="n">data_folder</span><span class="p">)</span>
     <span class="n">tar</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
     <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Data extracted.&#39;</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>After downloading and extracting the data archive, we have two folders: <code class="docutils literal notranslate"><span class="pre">data/101_ObjectCategories</span></code> and <code class="docutils literal notranslate"><span class="pre">data/101_ObjectCategories_test</span></code>. We load the data into separate training and testing <a class="reference external" href="/api/python/docs/api/gluon/data/vision/datasets/index.html#mxnet.gluon.data.vision.datasets.ImageFolderDataset">ImageFolderDataset</a>s.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">training_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_folder</span><span class="p">,</span> <span class="n">dataset_name</span><span class="p">)</span>
 <span class="n">testing_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_folder</span><span class="p">,</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">_test&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">))</span>
 </pre></div>
 </div>
 <p>We instantiate the <a class="reference external" href="https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=imagefolderdataset#mxnet.gluon.data.vision.datasets.ImageFolderDataset">ImageFolderDataset</a>s by providing the path to the data, and the folder structure will be traversed to determine which image classes are available and which images correspond to each class. You must take care to ensure the same classes are both the training and testing datasets, otherwise the label encodings can get muddled.</p>
 <p>Optionally, you can pass a <code class="docutils literal notranslate"><span class="pre">transform</span></code> parameter to these <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>’s as we’ve seen before.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">train_dataset</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">vision</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">ImageFolderDataset</span><span class="p">(</span><span class="n">training_path</span><span class="p">)</span>
 <span class="n">test_dataset</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">gluon</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">vision</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">ImageFolderDataset</span><span class="p">(</span><span class="n">testing_path</span><span class="p">)</span>
 </pre></div>
 </div>
 <p>Samples from these datasets are tuples of data and label. Images are loaded from disk, decoded and optionally transformed when the <code class="docutils literal notranslate"><span class="pre">__getitem__(i)</span></code> method is called (equivalent to <code class="docutils literal notranslate"><span class="pre">train_dataset[i]</span></code>).</p>
 <p>As with the Fashion MNIST dataset the labels will be integer encoded. You can use the <code class="docutils literal notranslate"><span class="pre">synsets</span></code> property of the <a class="reference external" href="https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=imagefolderdataset#mxnet.gluon.data.vision.datasets.ImageFolderDataset">ImageFolderDataset</a>s to retrieve the original descriptions (e.g. <code class="docutils literal notranslate"><span class="pre">train_dataset.synsets[i]</span></code>).</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sample_idx</span> <span class="o">=</span> <span class="mi">539</span>
 <span class="n">sample</span> <span class="o">=</span> <span class="n">train_dataset</span><span class="p">[</span><span class="n">sample_idx</span><span class="p">]</span>
 <span class="n">data</span> <span class="o">=</span> <span class="n">sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
 <span class="n">label</span> <span class="o">=</span> <span class="n">sample</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>

 <span class="n">imshow</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">asnumpy</span><span class="p">(),</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">&#39;gray&#39;</span><span class="p">)</span>
 <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Data type: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">dtype</span><span class="p">))</span>
 <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Label: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">label</span><span class="p">))</span>
 <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Label description: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">train_dataset</span><span class="o">.</span><span class="n">synsets</span><span class="p">[</span><span class="n">label</span><span class="p">]))</span>
 <span class="k">assert</span> <span class="n">label</span> <span class="o">==</span> <span class="mi">1</span>
 </pre></div>
 </div>
 <p><code class="docutils literal notranslate"><span class="pre">Data</span> <span class="pre">type:</span> <span class="pre">&lt;class</span> <span class="pre">'numpy.uint8'&gt;</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">Label:</span> <span class="pre">1</span></code></p>
 <p><code class="docutils literal notranslate"><span class="pre">Label</span> <span class="pre">description:</span> <span class="pre">Faces_easy</span></code></p>
 <p><img alt="png" src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/gluon/datasets/caltech101_face.png" /></p>
 </div>
 <div class="section" id="Using-own-data-with-custom-Datasets">
 <h1>Using own data with custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s<a class="headerlink" href="#Using-own-data-with-custom-Datasets" title="Permalink to this headline">¶</a></h1>
 <p>Sometimes you have data that doesn’t quite fit the format expected by the included <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#mxnet.gluon.data.Dataset">Dataset</a>s. You might be able to preprocess your data to fit the expected format, but it is easy to create your own dataset to do this.</p>
 <p>All you need to do is create a class that implements a <code class="docutils literal notranslate"><span class="pre">__getitem__</span></code> method, that returns a sample (i.e. a tuple of <a class="reference external" href="/api/python/docs/api/ndarray/ndarray.html#mxnet.ndarray.NDArray">mx.nd.NDArray</a>’s).</p>
 </div>
 <div class="section" id="Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader">
 <h1>Appendix: Upgrading from Module <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> to Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code><a class="headerlink" href="#Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader" title="Permalink to this headline">¶</a></h1>
 <p>Before Gluon’s <a class="reference external" href="/api/python/docs/api/gluon/data/index.html#dataloader">DataLoader</a>, MXNet used <a class="reference external" href="/api/python/docs/api/mxnet/io/index.html#mxnet.io.DataIter">DataIter</a> objects for loading data for training and testing. <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> has a similar interface for iterating through data, but it isn’t directly compatible with typical Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code> loops. Unlike Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code> which often returns a tuple of <code class="docutils literal notranslate"><span class="pre">(data,</span> <span class="pre">label)</span></code>, a <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> returns a
 <a class="reference external" href="/api/python/docs/api/mxnet/io/index.html#mxnet.io.DataBatch">DataBatch</a> object that has <code class="docutils literal notranslate"><span class="pre">data</span></code> and <code class="docutils literal notranslate"><span class="pre">label</span></code> properties. Switching to <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code>’s is highly recommended when using Gluon, but you’ll need to take care of pre-processing steps such as augmentations in a <code class="docutils literal notranslate"><span class="pre">transform</span></code> function.</p>
 <p>So you can get up and running with Gluon quicker if you have already implemented complex pre-processing steps using <code class="docutils literal notranslate"><span class="pre">DataIter</span></code>, we have provided a simple class to wrap existing <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> objects so they can be used in a typical Gluon training loop. You can use this class for <code class="docutils literal notranslate"><span class="pre">DataIter</span></code>s such as <a class="reference external" href="/api/python/docs/api/mxnet/image/index.html#mxnet.image.ImageIter">mxnet.image.ImageIter</a> and
 <a class="reference external" href="/api/python/docs/api/mxnet/io/index.html#mxnet.io.ImageDetRecordIter">mxnet.io.ImageRecordIter</a> that have single data and label arrays.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">DataIterLoader</span><span class="p">():</span>
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data_iter</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">data_iter</span> <span class="o">=</span> <span class="n">data_iter</span>

     <span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">data_iter</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
         <span class="k">return</span> <span class="bp">self</span>

     <span class="k">def</span> <span class="fm">__next__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="n">batch</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_iter</span><span class="o">.</span><span class="fm">__next__</span><span class="p">()</span>
         <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">label</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
         <span class="n">data</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
         <span class="n">label</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">label</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
         <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">label</span>

     <span class="k">def</span> <span class="nf">next</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__next__</span><span class="p">()</span> <span class="c1"># for Python 2</span>
 </pre></div>
 </div>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">data_iter</span> <span class="o">=</span> <span class="n">mx</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">NDArrayIter</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">X</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="n">y</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
 <span class="n">data_iter_loader</span> <span class="o">=</span> <span class="n">DataIterLoader</span><span class="p">(</span><span class="n">data_iter</span><span class="p">)</span>
 <span class="k">for</span> <span class="n">X_batch</span><span class="p">,</span> <span class="n">y_batch</span> <span class="ow">in</span> <span class="n">data_iter_loader</span><span class="p">:</span>
     <span class="k">assert</span> <span class="n">X_batch</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
     <span class="k">assert</span> <span class="n">y_batch</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
 </pre></div>
 </div>
 <!-- INSERT SOURCE DOWNLOAD BUTTONS --></div>


         <hr class="feedback-hr-top" />
 <div class="feedback-container">
     <div class="feedback-question">Did this page help you?</div>
     <div class="feedback-answer-container">
         <div class="feedback-answer yes-link" data-response="yes">Yes</div>
         <div class="feedback-answer no-link" data-response="no">No</div>
     </div>
     <div class="feedback-thank-you">Thanks for your feedback!</div>
 </div>
 <hr class="feedback-hr-bottom" />
         </div>
         <div class="side-doc-outline">
             <div class="side-doc-outline--content">
 <div class="localtoc">
     <p class="caption">
       <span class="caption-text">Table Of Contents</span>
     </p>
     <ul>
 <li><a class="reference internal" href="#">Gluon <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a><ul>
 <li><a class="reference internal" href="#Introduction-to-Datasets">Introduction to <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li><a class="reference internal" href="#Introduction-to-DataLoader">Introduction to <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 <li><a class="reference internal" href="#Machine-learning-with-Datasets-and-DataLoaders">Machine learning with <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s and <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code>s</a></li>
 </ul>
 </li>
 <li><a class="reference internal" href="#Using-own-data-with-included-Datasets">Using own data with included <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li><a class="reference internal" href="#Using-own-data-with-custom-Datasets">Using own data with custom <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s</a></li>
 <li><a class="reference internal" href="#Appendix:-Upgrading-from-Module-DataIter-to-Gluon-DataLoader">Appendix: Upgrading from Module <code class="docutils literal notranslate"><span class="pre">DataIter</span></code> to Gluon <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code></a></li>
 </ul>

 </div>
             </div>
         </div>

       <div class="clearer"></div>
     </div><div class="pagenation">
      <a id="button-prev" href="data_augmentation.html" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--colored" role="botton" accesskey="P">
          <i class="pagenation-arrow-L fas fa-arrow-left fa-lg"></i>
          <div class="pagenation-text">
             <span class="pagenation-direction">Previous</span>
             <div>Image Augmentation</div>
          </div>
      </a>
      <a id="button-next" href="../image/index.html" class="mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--colored" role="botton" accesskey="N">
          <i class="pagenation-arrow-R fas fa-arrow-right fa-lg"></i>
         <div class="pagenation-text">
             <span class="pagenation-direction">Next</span>
             <div>Image Tutorials</div>
         </div>
      </a>
   </div>
             <footer class="site-footer h-card">
     <div class="wrapper">
         <div class="row">
             <div class="col-4">
                 <h4 class="footer-category-title">Resources</h4>
                 <ul class="contact-list">
                     <li><a class="u-email" href="mailto:dev@mxnet.apache.org">Dev list</a></li>
                     <li><a class="u-email" href="mailto:user@mxnet.apache.org">User mailing list</a></li>
                     <li><a href="https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home">Developer Wiki</a></li>
                     <li><a href="https://issues.apache.org/jira/projects/MXNET/issues">Jira Tracker</a></li>
                     <li><a href="https://github.com/apache/mxnet/labels/Roadmap">Github Roadmap</a></li>
                     <li><a href="https://medium.com/apache-mxnet">Blog</a></li>
                     <li><a href="https://discuss.mxnet.io">Forum</a></li>
                     <li><a href="/community/contribute">Contribute</a></li>

                 </ul>
             </div>

             <div class="col-4"><ul class="social-media-list"><li><a href="https://github.com/apache/mxnet"><svg class="svg-icon"><use xlink:href="../../../../_static/minima-social-icons.svg#github"></use></svg> <span class="username">apache/mxnet</span></a></li><li><a href="https://www.twitter.com/apachemxnet"><svg class="svg-icon"><use xlink:href="../../../../_static/minima-social-icons.svg#twitter"></use></svg> <span class="username">apachemxnet</span></a></li><li><a href="https://youtube.com/apachemxnet"><svg class="svg-icon"><use xlink:href="../../../../_static/minima-social-icons.svg#youtube"></use></svg> <span class="username">apachemxnet</span></a></li></ul>
 </div>

             <div class="col-4 footer-text">
                 <p>A flexible and efficient library for deep learning.</p>
             </div>
         </div>
     </div>
 </footer>

 <footer class="site-footer2">
     <div class="wrapper">
         <div class="row">
             <div class="col-3">
                 <img src="../../../../_static/apache_incubator_logo.png" class="footer-logo col-2">
             </div>
             <div class="footer-bottom-warning col-9">
                 <p>Apache MXNet is an effort undergoing incubation at <a href="http://www.apache.org/">The Apache Software Foundation</a> (ASF), <span style="font-weight:bold">sponsored by the <i>Apache Incubator</i></span>. Incubation is required
                     of all newly accepted projects until a further review indicates that the infrastructure,
                     communications, and decision making process have stabilized in a manner consistent with other
                     successful ASF projects. While incubation status is not necessarily a reflection of the completeness
                     or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
                 </p><p>"Copyright © 2017-2018, The Apache Software Foundation Apache MXNet, MXNet, Apache, the Apache
                     feather, and the Apache MXNet project logo are either registered trademarks or trademarks of the
                     Apache Software Foundation."</p>
             </div>
         </div>
     </div>
 </footer>

   </body>
 </html>