blob: 8607bc870f887af1874a78a29a5887eb50c80220 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>apache_beam.utils.retry module &mdash; Apache Beam documentation</title>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Apache Beam documentation" href="index.html"/>
<link rel="up" title="apache_beam.utils package" href="apache_beam.utils.html"/>
<link rel="next" title="apache_beam.utils.timestamp module" href="apache_beam.utils.timestamp.html"/>
<link rel="prev" title="apache_beam.utils.proto_utils module" href="apache_beam.utils.proto_utils.html"/>
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="index.html" class="icon icon-home"> Apache Beam
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="apache_beam.coders.html">apache_beam.coders package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.internal.html">apache_beam.internal package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.io.html">apache_beam.io package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.metrics.html">apache_beam.metrics package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.options.html">apache_beam.options package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.portability.html">apache_beam.portability package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.runners.html">apache_beam.runners package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.testing.html">apache_beam.testing package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.tools.html">apache_beam.tools package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.transforms.html">apache_beam.transforms package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.typehints.html">apache_beam.typehints package</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="apache_beam.utils.html">apache_beam.utils package</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="apache_beam.utils.html#submodules">Submodules</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.annotations.html">apache_beam.utils.annotations module</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.plugin.html">apache_beam.utils.plugin module</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.processes.html">apache_beam.utils.processes module</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.profiler.html">apache_beam.utils.profiler module</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.proto_utils.html">apache_beam.utils.proto_utils module</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">apache_beam.utils.retry module</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.timestamp.html">apache_beam.utils.timestamp module</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.utils.urns.html">apache_beam.utils.urns module</a></li>
</ul>
</li>
</ul>
</li>
</ul>
<ul>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.error.html">apache_beam.error module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.version.html">apache_beam.version module</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Apache Beam</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li><a href="apache_beam.utils.html">apache_beam.utils package</a> &raquo;</li>
<li>apache_beam.utils.retry module</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/apache_beam.utils.retry.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-apache_beam.utils.retry">
<span id="apache-beam-utils-retry-module"></span><h1>apache_beam.utils.retry module<a class="headerlink" href="#module-apache_beam.utils.retry" title="Permalink to this headline"></a></h1>
<p>Retry decorators for calls raising exceptions.</p>
<p>For internal use only; no backwards-compatibility guarantees.</p>
<p>This module is used mostly to decorate all integration points where the code
makes calls to remote services. Searching through the code base for &#64;retry
should find all such places. For this reason even places where retry is not
needed right now use a &#64;retry.no_retries decorator.</p>
<dl class="exception">
<dt id="apache_beam.utils.retry.PermanentException">
<em class="property">exception </em><code class="descclassname">apache_beam.utils.retry.</code><code class="descname">PermanentException</code><a class="reference internal" href="_modules/apache_beam/utils/retry.html#PermanentException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.PermanentException" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/2/library/exceptions.html#exceptions.Exception" title="(in Python v2.7)"><code class="xref py py-class docutils literal"><span class="pre">exceptions.Exception</span></code></a></p>
<p>Base class for exceptions that should not be retried.</p>
</dd></dl>
<dl class="class">
<dt id="apache_beam.utils.retry.FuzzedExponentialIntervals">
<em class="property">class </em><code class="descclassname">apache_beam.utils.retry.</code><code class="descname">FuzzedExponentialIntervals</code><span class="sig-paren">(</span><em>initial_delay_secs</em>, <em>num_retries</em>, <em>factor=2</em>, <em>fuzz=0.5</em>, <em>max_delay_secs=3600</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#FuzzedExponentialIntervals"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.FuzzedExponentialIntervals" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">future.types.newobject.newobject</span></code></p>
<p>Iterable for intervals that are exponentially spaced, with fuzzing.</p>
<p>On iteration, yields retry interval lengths, in seconds. Every iteration over
this iterable will yield differently fuzzed interval lengths, as long as fuzz
is nonzero.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>initial_delay_secs</strong> – The delay before the first retry, in seconds.</li>
<li><strong>num_retries</strong> – The total number of times to retry.</li>
<li><strong>factor</strong> – The exponential factor to use on subsequent retries.
Default is 2 (doubling).</li>
<li><strong>fuzz</strong> – A value between 0 and 1, indicating the fraction of fuzz. For a
given delay d, the fuzzed delay is randomly chosen between
[(1 - fuzz) * d, d].</li>
<li><strong>max_delay_secs</strong> – Maximum delay (in seconds). After this limit is reached,
further tries use max_delay_sec instead of exponentially increasing
the time. Defaults to 1 hour.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.retry_on_server_errors_filter">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">retry_on_server_errors_filter</code><span class="sig-paren">(</span><em>exception</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#retry_on_server_errors_filter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.retry_on_server_errors_filter" title="Permalink to this definition"></a></dt>
<dd><p>Filter allowing retries on server errors and non-HttpErrors.</p>
</dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.retry_on_server_errors_and_notfound_filter">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">retry_on_server_errors_and_notfound_filter</code><span class="sig-paren">(</span><em>exception</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#retry_on_server_errors_and_notfound_filter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.retry_on_server_errors_and_notfound_filter" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.retry_on_server_errors_and_timeout_filter">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">retry_on_server_errors_and_timeout_filter</code><span class="sig-paren">(</span><em>exception</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#retry_on_server_errors_and_timeout_filter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.retry_on_server_errors_and_timeout_filter" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.retry_on_server_errors_timeout_or_quota_issues_filter">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">retry_on_server_errors_timeout_or_quota_issues_filter</code><span class="sig-paren">(</span><em>exception</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#retry_on_server_errors_timeout_or_quota_issues_filter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.retry_on_server_errors_timeout_or_quota_issues_filter" title="Permalink to this definition"></a></dt>
<dd><p>Retry on server, timeout and 403 errors.</p>
<p>403 errors can be accessDenied, billingNotEnabled, and also quotaExceeded,
rateLimitExceeded.</p>
</dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.retry_on_beam_io_error_filter">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">retry_on_beam_io_error_filter</code><span class="sig-paren">(</span><em>exception</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#retry_on_beam_io_error_filter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.retry_on_beam_io_error_filter" title="Permalink to this definition"></a></dt>
<dd><p>Filter allowing retries on Beam IO errors.</p>
</dd></dl>
<dl class="class">
<dt id="apache_beam.utils.retry.Clock">
<em class="property">class </em><code class="descclassname">apache_beam.utils.retry.</code><code class="descname">Clock</code><a class="reference internal" href="_modules/apache_beam/utils/retry.html#Clock"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.Clock" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">future.types.newobject.newobject</span></code></p>
<p>A simple clock implementing sleep().</p>
<dl class="method">
<dt id="apache_beam.utils.retry.Clock.sleep">
<code class="descname">sleep</code><span class="sig-paren">(</span><em>value</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#Clock.sleep"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.Clock.sleep" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.no_retries">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">no_retries</code><span class="sig-paren">(</span><em>fun</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#no_retries"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.no_retries" title="Permalink to this definition"></a></dt>
<dd><p>A retry decorator for places where we do not want retries.</p>
</dd></dl>
<dl class="function">
<dt id="apache_beam.utils.retry.with_exponential_backoff">
<code class="descclassname">apache_beam.utils.retry.</code><code class="descname">with_exponential_backoff</code><span class="sig-paren">(</span><em>num_retries=7</em>, <em>initial_delay_secs=5.0</em>, <em>logger=&lt;function warning&gt;</em>, <em>retry_filter=&lt;function retry_on_server_errors_filter&gt;</em>, <em>clock=&lt;apache_beam.utils.retry.Clock object&gt;</em>, <em>fuzz=True</em>, <em>factor=2</em>, <em>max_delay_secs=3600</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/utils/retry.html#with_exponential_backoff"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.utils.retry.with_exponential_backoff" title="Permalink to this definition"></a></dt>
<dd><p>Decorator with arguments that control the retry logic.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>num_retries</strong> – The total number of times to retry.</li>
<li><strong>initial_delay_secs</strong> – The delay before the first retry, in seconds.</li>
<li><strong>logger</strong> – A callable used to report an exception. Must have the same signature
as functions in the standard logging module. The default is
logging.warning.</li>
<li><strong>retry_filter</strong> – A callable getting the exception raised and returning True
if the retry should happen. For instance we do not want to retry on
404 Http errors most of the time. The default value will return true
for server errors (HTTP status code &gt;= 500) and non Http errors.</li>
<li><strong>clock</strong> – A clock object implementing a sleep method. The default clock will
use time.sleep().</li>
<li><strong>fuzz</strong> – True if the delay should be fuzzed (default). During testing False
can be used so that the delays are not randomized.</li>
<li><strong>factor</strong> – The exponential factor to use on subsequent retries.
Default is 2 (doubling).</li>
<li><strong>max_delay_secs</strong> – Maximum delay (in seconds). After this limit is reached,
further tries use max_delay_sec instead of exponentially increasing
the time. Defaults to 1 hour.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first last">As per Python decorators with arguments pattern returns a decorator
for the function which in turn will return the wrapped (decorated) function.</p>
</td>
</tr>
</tbody>
</table>
<p>The decorator is intended to be used on callables that make HTTP or RPC
requests that can temporarily timeout or have transient errors. For instance
the make_http_request() call below will be retried 16 times with exponential
backoff and fuzzing of the delay interval (default settings).</p>
<p>from apache_beam.utils import retry
# …
&#64;retry.with_exponential_backoff()
make_http_request(args)</p>
</dd></dl>
</div>
</div>
<div class="articleComments">
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="apache_beam.utils.timestamp.html" class="btn btn-neutral float-right" title="apache_beam.utils.timestamp module" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="apache_beam.utils.proto_utils.html" class="btn btn-neutral" title="apache_beam.utils.proto_utils module" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright .
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
});
</script>
</body>
</html>