blob: 827f5901c0abe9b7430eb60f8fcc7802fefb2264 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>apache_beam.runners.interactive.interactive_environment module &mdash; Apache Beam documentation</title>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Apache Beam documentation" href="index.html"/>
<link rel="up" title="apache_beam.runners.interactive package" href="apache_beam.runners.interactive.html"/>
<link rel="next" title="apache_beam.runners.interactive.interactive_runner module" href="apache_beam.runners.interactive.interactive_runner.html"/>
<link rel="prev" title="apache_beam.runners.interactive.interactive_beam module" href="apache_beam.runners.interactive.interactive_beam.html"/>
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="index.html" class="icon icon-home"> Apache Beam
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="apache_beam.coders.html">apache_beam.coders package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.internal.html">apache_beam.internal package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.io.html">apache_beam.io package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.metrics.html">apache_beam.metrics package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.options.html">apache_beam.options package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.portability.html">apache_beam.portability package</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="apache_beam.runners.html">apache_beam.runners package</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="apache_beam.runners.html#subpackages">Subpackages</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="apache_beam.runners.dataflow.html">apache_beam.runners.dataflow package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.runners.direct.html">apache_beam.runners.direct package</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="apache_beam.runners.interactive.html">apache_beam.runners.interactive package</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="apache_beam.runners.interactive.html#subpackages">Subpackages</a></li>
<li class="toctree-l4 current"><a class="reference internal" href="apache_beam.runners.interactive.html#submodules">Submodules</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.runners.internal.html">apache_beam.runners.internal package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.runners.job.html">apache_beam.runners.job package</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="apache_beam.runners.html#submodules">Submodules</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.testing.html">apache_beam.testing package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.tools.html">apache_beam.tools package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.transforms.html">apache_beam.transforms package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.typehints.html">apache_beam.typehints package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.utils.html">apache_beam.utils package</a></li>
</ul>
<ul>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.error.html">apache_beam.error module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.version.html">apache_beam.version module</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Apache Beam</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li><a href="apache_beam.runners.html">apache_beam.runners package</a> &raquo;</li>
<li><a href="apache_beam.runners.interactive.html">apache_beam.runners.interactive package</a> &raquo;</li>
<li>apache_beam.runners.interactive.interactive_environment module</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/apache_beam.runners.interactive.interactive_environment.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-apache_beam.runners.interactive.interactive_environment">
<span id="apache-beam-runners-interactive-interactive-environment-module"></span><h1>apache_beam.runners.interactive.interactive_environment module<a class="headerlink" href="#module-apache_beam.runners.interactive.interactive_environment" title="Permalink to this headline"></a></h1>
<p>Module of the current Interactive Beam environment.</p>
<p>For internal use only; no backwards-compatibility guarantees.
Provides interfaces to interact with existing Interactive Beam environment.
External Interactive Beam users please use interactive_beam module in
application code or notebook.</p>
<dl class="function">
<dt id="apache_beam.runners.interactive.interactive_environment.current_env">
<code class="descclassname">apache_beam.runners.interactive.interactive_environment.</code><code class="descname">current_env</code><span class="sig-paren">(</span><em>cache_manager=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#current_env"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.current_env" title="Permalink to this definition"></a></dt>
<dd><p>Gets current Interactive Beam environment.</p>
</dd></dl>
<dl class="function">
<dt id="apache_beam.runners.interactive.interactive_environment.new_env">
<code class="descclassname">apache_beam.runners.interactive.interactive_environment.</code><code class="descname">new_env</code><span class="sig-paren">(</span><em>cache_manager=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#new_env"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.new_env" title="Permalink to this definition"></a></dt>
<dd><p>Creates a new Interactive Beam environment to replace current one.</p>
</dd></dl>
<dl class="class">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment">
<em class="property">class </em><code class="descclassname">apache_beam.runners.interactive.interactive_environment.</code><code class="descname">InteractiveEnvironment</code><span class="sig-paren">(</span><em>cache_manager=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/2/library/functions.html#object" title="(in Python v2.7)"><code class="xref py py-class docutils literal"><span class="pre">object</span></code></a></p>
<p>An interactive environment with cache and pipeline variable metadata.</p>
<p>Interactive Beam will use the watched variable information to determine if a
PCollection is assigned to a variable in user pipeline definition. When
executing the pipeline, interactivity is applied with implicit cache
mechanism for those PCollections if the pipeline is interactive. Users can
also visualize and introspect those PCollections in user code since they have
handles to the variables.</p>
<dl class="attribute">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_py_version_ready">
<code class="descname">is_py_version_ready</code><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_py_version_ready" title="Permalink to this definition"></a></dt>
<dd><p>If Python version is above the minimum requirement.</p>
</dd></dl>
<dl class="attribute">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_interactive_ready">
<code class="descname">is_interactive_ready</code><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_interactive_ready" title="Permalink to this definition"></a></dt>
<dd><p>If the [interactive] dependencies are installed.</p>
</dd></dl>
<dl class="attribute">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_in_ipython">
<code class="descname">is_in_ipython</code><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_in_ipython" title="Permalink to this definition"></a></dt>
<dd><p>If the runtime is within an IPython kernel.</p>
</dd></dl>
<dl class="attribute">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_in_notebook">
<code class="descname">is_in_notebook</code><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_in_notebook" title="Permalink to this definition"></a></dt>
<dd><p>If the kernel is connected to a notebook frontend.</p>
<p>If not, it could be that the user is using kernel in a terminal or a unit
test.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.cleanup">
<code class="descname">cleanup</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.cleanup"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.cleanup" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.watch">
<code class="descname">watch</code><span class="sig-paren">(</span><em>watchable</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.watch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.watch" title="Permalink to this definition"></a></dt>
<dd><p>Watches a watchable.</p>
<p>A watchable can be a dictionary of variable metadata such as locals(), a str
name of a module, a module object or an instance of a class. The variable
can come from any scope even local. Duplicated variable naming doesn’t
matter since they are different instances. Duplicated variables are also
allowed when watching.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.watching">
<code class="descname">watching</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.watching"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.watching" title="Permalink to this definition"></a></dt>
<dd><p>Analyzes and returns a list of pair lists referring to variable names and
values from watched scopes.</p>
<p>Each entry in the list represents the variable defined within a watched
watchable. Currently, each entry holds a list of pairs. The format might
change in the future to hold more metadata. Duplicated pairs are allowed.
And multiple paris can have the same variable name as the “first” while
having different variable values as the “second” since variables in
different scopes can have the same name.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.set_cache_manager">
<code class="descname">set_cache_manager</code><span class="sig-paren">(</span><em>cache_manager</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.set_cache_manager"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.set_cache_manager" title="Permalink to this definition"></a></dt>
<dd><p>Sets the cache manager held by current Interactive Environment.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.cache_manager">
<code class="descname">cache_manager</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.cache_manager"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.cache_manager" title="Permalink to this definition"></a></dt>
<dd><p>Gets the cache manager held by current Interactive Environment.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.set_pipeline_result">
<code class="descname">set_pipeline_result</code><span class="sig-paren">(</span><em>pipeline</em>, <em>result</em>, <em>is_main_job</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.set_pipeline_result"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.set_pipeline_result" title="Permalink to this definition"></a></dt>
<dd><p>Sets the pipeline run result. Adds one if absent. Otherwise, replace.</p>
<p>When is_main_job is True, set the result for the main job; otherwise, set
the result for the background caching job.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.evict_pipeline_result">
<code class="descname">evict_pipeline_result</code><span class="sig-paren">(</span><em>pipeline</em>, <em>is_main_job=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.evict_pipeline_result"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.evict_pipeline_result" title="Permalink to this definition"></a></dt>
<dd><p>Evicts the tracking of given pipeline run. Noop if absent.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.pipeline_result">
<code class="descname">pipeline_result</code><span class="sig-paren">(</span><em>pipeline</em>, <em>is_main_job=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.pipeline_result"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.pipeline_result" title="Permalink to this definition"></a></dt>
<dd><p>Gets the pipeline run result. None if absent.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_terminated">
<code class="descname">is_terminated</code><span class="sig-paren">(</span><em>pipeline</em>, <em>is_main_job=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.is_terminated"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.is_terminated" title="Permalink to this definition"></a></dt>
<dd><p>Queries if the most recent job (by executing the given pipeline) state
is in a terminal state. True if absent.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.set_cached_source_signature">
<code class="descname">set_cached_source_signature</code><span class="sig-paren">(</span><em>pipeline</em>, <em>signature</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.set_cached_source_signature"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.set_cached_source_signature" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.get_cached_source_signature">
<code class="descname">get_cached_source_signature</code><span class="sig-paren">(</span><em>pipeline</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.get_cached_source_signature"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.get_cached_source_signature" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.track_user_pipelines">
<code class="descname">track_user_pipelines</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/runners/interactive/interactive_environment.html#InteractiveEnvironment.track_user_pipelines"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.track_user_pipelines" title="Permalink to this definition"></a></dt>
<dd><p>Record references to all user-defined pipeline instances watched in
current environment.</p>
<p>Current static global singleton interactive environment holds references to
a set of pipeline instances defined by the user in the watched scope.
Interactive Beam features could use the references to determine if a given
pipeline is defined by user or implicitly created by Beam SDK or runners,
then handle them differently.</p>
<p>This is invoked every time a PTransform is to be applied if the current
code execution is under ipython due to the possibility that any user-defined
pipeline can be re-evaluated through notebook cell re-execution at any time.</p>
<p>Each time this is invoked, the tracked user pipelines are refreshed to
remove any pipeline instances that are no longer in watched scope. For
example, after a notebook cell re-execution re-evaluating a pipeline
creation, the last pipeline reference created by last evaluation will not be
in watched scope anymore.</p>
</dd></dl>
<dl class="attribute">
<dt id="apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.tracked_user_pipelines">
<code class="descname">tracked_user_pipelines</code><a class="headerlink" href="#apache_beam.runners.interactive.interactive_environment.InteractiveEnvironment.tracked_user_pipelines" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
</div>
</div>
<div class="articleComments">
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="apache_beam.runners.interactive.interactive_runner.html" class="btn btn-neutral float-right" title="apache_beam.runners.interactive.interactive_runner module" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="apache_beam.runners.interactive.interactive_beam.html" class="btn btn-neutral" title="apache_beam.runners.interactive.interactive_beam module" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright .
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
});
</script>
</body>
</html>