Google Git
Sign in
apache / singa-site / be983bc44db8673b80f36c133d1ceb12926645fd / . / content / zh / docs / data.html
blob: f51dbf29c57152630ca77aea51527237b02c6472 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>数据(Data) &mdash; singa 2.0.0 documentation</title>
<script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'2.0.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
<script type="text/javascript" src="../_static/underscore.js"></script>
<script type="text/javascript" src="../_static/doctools.js"></script>
<script type="text/javascript" src="../_static/js/theme.js"></script>
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="图像工具" href="image_tool.html" />
<link rel="prev" title="优化器(Optimizer)" href="optimizer.html" />
<link href="../_static/style.css" rel="stylesheet" type="text/css">
<!--link href="../_static/fontawesome-all.min.css" rel="stylesheet" type="text/css"-->
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"
integrity="sha384-DNOHZ68U8hZfKXOrtjWvjxusGo9WQnrNx2sqG0tfsghAvtVlRW3tvkXWZh58N9jp" crossorigin="anonymous">
<style>
.fa:hover {
opacity: 0.7;
}
.fab:hover {
opacity: 0.7;
}
</style>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../index.html" class="icon icon-home"> singa
<img src="../_static/singa.png" class="logo" alt="Logo"/>
</a>
<div class="version">
latest
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="index.html">文档</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="installation.html">安装</a></li>
<li class="toctree-l2"><a class="reference internal" href="software_stack.html">软件架构</a></li>
<li class="toctree-l2"><a class="reference internal" href="device.html">设备(Device)</a></li>
<li class="toctree-l2"><a class="reference internal" href="tensor.html">张量(Tensor)</a></li>
<li class="toctree-l2"><a class="reference internal" href="layer.html">层(Layer)</a></li>
<li class="toctree-l2"><a class="reference internal" href="net.html">前馈网络</a></li>
<li class="toctree-l2"><a class="reference internal" href="initializer.html">初始化器(Initializer)</a></li>
<li class="toctree-l2"><a class="reference internal" href="loss.html">损失(Loss)</a></li>
<li class="toctree-l2"><a class="reference internal" href="metric.html">度量(Metric)</a></li>
<li class="toctree-l2"><a class="reference internal" href="optimizer.html">优化器(Optimizer)</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">数据(Data)</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#class-singa-data-imagebatchiter-img-list-file-batch-size-image-transform-shuffle-true-delimiter-image-folder-none-capacity-10">class singa.data.ImageBatchIter(img_list_file, batch_size, image_transform, shuffle=True, delimiter=&#8216; &#8216;, image_folder=None, capacity=10)</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="image_tool.html">图像工具</a></li>
<li class="toctree-l2"><a class="reference internal" href="snapshot.html">Snapshot</a></li>
<li class="toctree-l2"><a class="reference internal" href="utils.html">Utils</a></li>
<li class="toctree-l2"><a class="reference internal" href="model_zoo/index.html">模型库</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../downloads.html">下载 SINGA</a></li>
</ul>
<p class="caption"><span class="caption-text">开发</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../develop/schedule.html">开发时间表</a></li>
<li class="toctree-l1"><a class="reference internal" href="../develop/how-contribute.html">如何贡献给 SINGA</a></li>
<li class="toctree-l1"><a class="reference internal" href="../develop/contribute-code.html">如何贡献代码</a></li>
</ul>
<p class="caption"><span class="caption-text">社区</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../community/source-repository.html">源代码库</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/mail-lists.html">项目邮件列表</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/issue-tracking.html">问题追踪</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/team-list.html">SINGA团队</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../index.html">singa</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../index.html">Docs</a> &raquo;</li>
<li><a href="index.html">文档</a> &raquo;</li>
<li>数据(Data)</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
--><div class="section" id="data">
<h1>数据(Data)<a class="headerlink" href="#data" title="Permalink to this headline"></a></h1>
<p>这个模块包含加载和预获取批数据的类。</p>
<p>示例用法：</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">image_tool</span>
<span class="kn">from</span> <span class="nn">PIL</span> <span class="kn">import</span> <span class="n">Image</span>
<span class="n">tool</span> <span class="o">=</span> <span class="n">image_tool</span><span class="o">.</span><span class="n">ImageTool</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">image_transform</span><span class="p">(</span><span class="n">img_path</span><span class="p">):</span>
<span class="k">global</span> <span class="n">tool</span>
<span class="k">return</span> <span class="n">tool</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">img_path</span><span class="p">)</span><span class="o">.</span><span class="n">resize_by_range</span><span class="p">(</span>
<span class="p">(</span><span class="mi">112</span><span class="p">,</span> <span class="mi">128</span><span class="p">))</span><span class="o">.</span><span class="n">random_crop</span><span class="p">(</span>
<span class="p">(</span><span class="mi">96</span><span class="p">,</span> <span class="mi">96</span><span class="p">))</span><span class="o">.</span><span class="n">flip</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">ImageBatchIter</span><span class="p">(</span><span class="s1">&#39;train.txt&#39;</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span>
<span class="n">image_transform</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
<span class="n">image_folder</span><span class="o">=</span><span class="s1">&#39;images/&#39;</span><span class="p">,</span>
<span class="n">capacity</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="n">data</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
<span class="c1"># imgs is a numpy array for a batch of images,</span>
<span class="c1"># shape: batch_size, 3 (RGB), height, width</span>
<span class="n">imgs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
<span class="c1"># convert numpy array back into images</span>
<span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">imgs</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span>
<span class="n">img</span> <span class="o">=</span> <span class="n">Image</span><span class="o">.</span><span class="n">fromarray</span><span class="p">(</span><span class="n">imgs</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
<span class="s1">&#39;RGB&#39;</span><span class="p">)</span>
<span class="n">img</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s1">&#39;img</span><span class="si">%d</span><span class="s1">.png&#39;</span> <span class="o">%</span> <span class="n">idx</span><span class="p">)</span>
<span class="n">data</span><span class="o">.</span><span class="n">end</span><span class="p">()</span>
</pre></div>
</div>
<hr class="docutils" />
<div class="section" id="class-singa-data-imagebatchiter-img-list-file-batch-size-image-transform-shuffle-true-delimiter-image-folder-none-capacity-10">
<h2>class singa.data.ImageBatchIter(img_list_file, batch_size, image_transform, shuffle=True, delimiter=&#8216; &#8216;, image_folder=None, capacity=10)<a class="headerlink" href="#class-singa-data-imagebatchiter-img-list-file-batch-size-image-transform-shuffle-true-delimiter-image-folder-none-capacity-10" title="Permalink to this headline"></a></h2>
<p>迭代地从数据集中获取批数据。</p>
<p><strong>参数：</strong></p>
<ul class="simple">
<li><strong>img_list_file (str)</strong> – 包含源数据的文件名；每行包含image_path_suffix和标签</li>
<li><strong>batch_size (int)</strong> – 每个mini-bach包含的样本数目</li>
<li><strong>image_transform</strong> – 图像增强函数；它接受完整的图像路径并输出一系列增强后的图像</li>
<li><strong>shuffle (boolean)</strong> – 为真表示对列表做搅乱</li>
<li><strong>delimiter (char)</strong> – image_path_suffix和标签之间的分割符, 例如空格或逗号</li>
<li><strong>image_folder (boolean)</strong> – 图片路径的前缀</li>
<li><strong>capacity (int)</strong> – 内部队列的最大mini-batch数目</li>
</ul>
<hr class="docutils" />
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="image_tool.html" class="btn btn-neutral float-right" title="图像工具" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="optimizer.html" class="btn btn-neutral float-left" title="优化器(Optimizer)" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2019 The Apache Software Foundation. All rights reserved. Apache SINGA, Apache, the Apache feather logo, and the Apache SINGA project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
<span class="rst-current-version" data-toggle="rst-current-version">
<span class="fa fa-book"> singa </span>
v: latest
<span class="fa fa-caret-down"></span>
</span>
<div class="rst-other-versions">
<dl>
<dt>Languages</dt>
<dd><a href="../../index.html">English</a></dd>
<dd><a href=".././index.html">中文</a></dd>
</dl>
<dl>
<dt>Versions</dt>
<dd><a href="http://singa.apache.org/v0.3.0/">0.3</a></dd>
<dd><a href="http://singa.apache.org/v1.1.0/">1.1</a></dd>
</dl>
</div>
<a href="http://www.apache.org"
style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Foundation</a>
<a href="http://www.apache.org/events/current-event"
style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Events</a>
<a href="http://www.apache.org/foundation/thanks.html"
style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Thanks</a>
<a href="http://www.apache.org/foundation/sponsorship.html"
style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">Sponsorship</a>
<a href="http://www.apache.org/licenses/"
style="color:lightblue;padding: 5px; font-size: 10px; text-align: center; text-decoration: none; margin: 5px 2px;">License</a>
<br>
<a href="https://github.com/apache/singa" class="fa fa-github"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
<a href="https://aws.amazon.com/marketplace/seller-profile?id=5bcac385-12c4-4802-aec7-351e09b77b4c" class="fab fa-aws"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
<a href="https://hub.docker.com/r/apache/singa/" class="fab fa-docker"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
<a href="https://www.linkedin.com/groups/13550034" class="fa fa-linkedin"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
<a href="https://twitter.com/ApacheSinga" class="fa fa-twitter"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
<a href="https://www.facebook.com/Apache-SINGA-347284219056544/" class="fa fa-facebook"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
<a href="https://www.researchgate.net/project/Apache-SINGA" class="fab fa-researchgate"
style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a>
</div>
<a href="https://github.com/apache/singa">
<img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;"
src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png" alt="Fork me on GitHub">
</a>
</body>
</html>