blob: 634b3c1ff57da3810c162289419aff650955e6ad [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>apache_beam.io.gcp.pubsub module &mdash; Apache Beam documentation</title>
<script type="text/javascript" src="_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/language_data.js"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="apache_beam.io.gcp.pubsub_it_pipeline module" href="apache_beam.io.gcp.pubsub_it_pipeline.html" />
<link rel="prev" title="apache_beam.io.gcp.gcsio_overrides module" href="apache_beam.io.gcp.gcsio_overrides.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="index.html" class="icon icon-home"> Apache Beam
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="apache_beam.coders.html">apache_beam.coders package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.internal.html">apache_beam.internal package</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="apache_beam.io.html">apache_beam.io package</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="apache_beam.io.html#subpackages">Subpackages</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.aws.html">apache_beam.io.aws package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.external.html">apache_beam.io.external package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.flink.html">apache_beam.io.flink package</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="apache_beam.io.gcp.html">apache_beam.io.gcp package</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="apache_beam.io.gcp.html#subpackages">Subpackages</a></li>
<li class="toctree-l4 current"><a class="reference internal" href="apache_beam.io.gcp.html#submodules">Submodules</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="apache_beam.io.html#submodules">Submodules</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.metrics.html">apache_beam.metrics package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.ml.html">apache_beam.ml package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.options.html">apache_beam.options package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.portability.html">apache_beam.portability package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.runners.html">apache_beam.runners package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.testing.html">apache_beam.testing package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.tools.html">apache_beam.tools package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.transforms.html">apache_beam.transforms package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.typehints.html">apache_beam.typehints package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.utils.html">apache_beam.utils package</a></li>
</ul>
<ul>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.error.html">apache_beam.error module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.version.html">apache_beam.version module</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Apache Beam</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li><a href="apache_beam.io.html">apache_beam.io package</a> &raquo;</li>
<li><a href="apache_beam.io.gcp.html">apache_beam.io.gcp package</a> &raquo;</li>
<li>apache_beam.io.gcp.pubsub module</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/apache_beam.io.gcp.pubsub.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-apache_beam.io.gcp.pubsub">
<span id="apache-beam-io-gcp-pubsub-module"></span><h1>apache_beam.io.gcp.pubsub module<a class="headerlink" href="#module-apache_beam.io.gcp.pubsub" title="Permalink to this headline"></a></h1>
<p>Google Cloud PubSub sources and sinks.</p>
<p>Cloud Pub/Sub sources and sinks are currently supported only in streaming
pipelines, during remote execution.</p>
<p>This API is currently under development and is subject to change.</p>
<dl class="class">
<dt id="apache_beam.io.gcp.pubsub.PubsubMessage">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.pubsub.</code><code class="descname">PubsubMessage</code><span class="sig-paren">(</span><em>data</em>, <em>attributes</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#PubsubMessage"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.PubsubMessage" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.8)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<p>Represents a Cloud Pub/Sub message.</p>
<p>Message payload includes the data and attributes fields. For the payload to be
valid, at least one of its fields must be non-empty.</p>
<p>This interface is experimental. No backwards compatibility guarantees.</p>
<dl class="attribute">
<dt id="apache_beam.io.gcp.pubsub.PubsubMessage.data">
<code class="descname">data</code><a class="headerlink" href="#apache_beam.io.gcp.pubsub.PubsubMessage.data" title="Permalink to this definition"></a></dt>
<dd><p>(bytes) Message data. May be None.</p>
</dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.pubsub.PubsubMessage.attributes">
<code class="descname">attributes</code><a class="headerlink" href="#apache_beam.io.gcp.pubsub.PubsubMessage.attributes" title="Permalink to this definition"></a></dt>
<dd><p>(dict) Key-value map of str to str, containing both user-defined
and service generated attributes (such as id_label and
timestamp_attribute). May be None.</p>
</dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.pubsub.ReadFromPubSub">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.pubsub.</code><code class="descname">ReadFromPubSub</code><span class="sig-paren">(</span><em>topic=None</em>, <em>subscription=None</em>, <em>id_label=None</em>, <em>with_attributes=False</em>, <em>timestamp_attribute=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#ReadFromPubSub"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.ReadFromPubSub" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="apache_beam.transforms.ptransform.html#apache_beam.transforms.ptransform.PTransform" title="apache_beam.transforms.ptransform.PTransform"><code class="xref py py-class docutils literal notranslate"><span class="pre">apache_beam.transforms.ptransform.PTransform</span></code></a></p>
<p>A <code class="docutils literal notranslate"><span class="pre">PTransform</span></code> for reading from Cloud Pub/Sub.</p>
<p>Initializes <code class="docutils literal notranslate"><span class="pre">ReadFromPubSub</span></code>.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>topic</strong> – Cloud Pub/Sub topic in the form
“projects/&lt;project&gt;/topics/&lt;topic&gt;”. If provided, subscription must be
None.</li>
<li><strong>subscription</strong> – Existing Cloud Pub/Sub subscription to use in the
form “projects/&lt;project&gt;/subscriptions/&lt;subscription&gt;”. If not
specified, a temporary subscription will be created from the specified
topic. If provided, topic must be None.</li>
<li><strong>id_label</strong> – The attribute on incoming Pub/Sub messages to use as a unique
record identifier. When specified, the value of this attribute (which
can be any string that uniquely identifies the record) will be used for
deduplication of messages. If not provided, we cannot guarantee
that no duplicate data will be delivered on the Pub/Sub stream. In this
case, deduplication of the stream will be strictly best effort.</li>
<li><strong>with_attributes</strong> – True - output elements will be <a class="reference internal" href="#apache_beam.io.gcp.pubsub.PubsubMessage" title="apache_beam.io.gcp.pubsub.PubsubMessage"><code class="xref py py-class docutils literal notranslate"><span class="pre">PubsubMessage</span></code></a> objects.
False - output elements will be of type <code class="docutils literal notranslate"><span class="pre">bytes</span></code> (message
data only).</li>
<li><strong>timestamp_attribute</strong><p>Message value to use as element timestamp. If None,
uses message publishing time as the timestamp.</p>
<p>Timestamp values should be in one of two formats:</p>
<ul>
<li>A numerical value representing the number of milliseconds since the
Unix epoch.</li>
<li>A string in RFC 3339 format, UTC timezone. Example:
<code class="docutils literal notranslate"><span class="pre">2015-10-29T23:41:41.123Z</span></code>. The sub-second component of the
timestamp is optional, and digits beyond the first three (i.e., time
units smaller than milliseconds) may be ignored.</li>
</ul>
</li>
</ul>
</td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="apache_beam.io.gcp.pubsub.ReadFromPubSub.expand">
<code class="descname">expand</code><span class="sig-paren">(</span><em>pvalue</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#ReadFromPubSub.expand"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.ReadFromPubSub.expand" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.pubsub.ReadFromPubSub.to_runner_api_parameter">
<code class="descname">to_runner_api_parameter</code><span class="sig-paren">(</span><em>context</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#ReadFromPubSub.to_runner_api_parameter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.ReadFromPubSub.to_runner_api_parameter" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.pubsub.ReadStringsFromPubSub">
<code class="descclassname">apache_beam.io.gcp.pubsub.</code><code class="descname">ReadStringsFromPubSub</code><span class="sig-paren">(</span><em>topic=None</em>, <em>subscription=None</em>, <em>id_label=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#ReadStringsFromPubSub"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.ReadStringsFromPubSub" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.pubsub.WriteStringsToPubSub">
<code class="descclassname">apache_beam.io.gcp.pubsub.</code><code class="descname">WriteStringsToPubSub</code><span class="sig-paren">(</span><em>topic</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#WriteStringsToPubSub"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.WriteStringsToPubSub" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.pubsub.WriteToPubSub">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.pubsub.</code><code class="descname">WriteToPubSub</code><span class="sig-paren">(</span><em>topic</em>, <em>with_attributes=False</em>, <em>id_label=None</em>, <em>timestamp_attribute=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#WriteToPubSub"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.WriteToPubSub" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="apache_beam.transforms.ptransform.html#apache_beam.transforms.ptransform.PTransform" title="apache_beam.transforms.ptransform.PTransform"><code class="xref py py-class docutils literal notranslate"><span class="pre">apache_beam.transforms.ptransform.PTransform</span></code></a></p>
<p>A <code class="docutils literal notranslate"><span class="pre">PTransform</span></code> for writing messages to Cloud Pub/Sub.</p>
<p>Initializes <code class="docutils literal notranslate"><span class="pre">WriteToPubSub</span></code>.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>topic</strong> – Cloud Pub/Sub topic in the form “/topics/&lt;project&gt;/&lt;topic&gt;”.</li>
<li><strong>with_attributes</strong> – True - input elements will be <a class="reference internal" href="#apache_beam.io.gcp.pubsub.PubsubMessage" title="apache_beam.io.gcp.pubsub.PubsubMessage"><code class="xref py py-class docutils literal notranslate"><span class="pre">PubsubMessage</span></code></a> objects.
False - input elements will be of type <code class="docutils literal notranslate"><span class="pre">bytes</span></code> (message
data only).</li>
<li><strong>id_label</strong> – If set, will set an attribute for each Cloud Pub/Sub message
with the given name and a unique value. This attribute can then be used
in a ReadFromPubSub PTransform to deduplicate messages.</li>
<li><strong>timestamp_attribute</strong> – If set, will set an attribute for each Cloud Pub/Sub
message with the given name and the message’s publish time as the value.</li>
</ul>
</td>
</tr>
</tbody>
</table>
<dl class="staticmethod">
<dt id="apache_beam.io.gcp.pubsub.WriteToPubSub.to_proto_str">
<em class="property">static </em><code class="descname">to_proto_str</code><span class="sig-paren">(</span><em>element</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#WriteToPubSub.to_proto_str"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.WriteToPubSub.to_proto_str" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.pubsub.WriteToPubSub.expand">
<code class="descname">expand</code><span class="sig-paren">(</span><em>pcoll</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#WriteToPubSub.expand"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.WriteToPubSub.expand" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.pubsub.WriteToPubSub.to_runner_api_parameter">
<code class="descname">to_runner_api_parameter</code><span class="sig-paren">(</span><em>context</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/pubsub.html#WriteToPubSub.to_runner_api_parameter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.pubsub.WriteToPubSub.to_runner_api_parameter" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="apache_beam.io.gcp.pubsub_it_pipeline.html" class="btn btn-neutral float-right" title="apache_beam.io.gcp.pubsub_it_pipeline module" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="apache_beam.io.gcp.gcsio_overrides.html" class="btn btn-neutral float-left" title="apache_beam.io.gcp.gcsio_overrides module" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>