Google Git
Sign in
apache / beam-site / refs/heads/updates_release_2.8.0 / . / javadoc / 2.8.0 / org / apache / beam / runners / dataflow / package-summary.html
blob: 6a7d2665ffadc2336844be14872f2dbfac43390e [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (1.8.0_161-google-v7) on Thu Oct 18 16:14:36 PDT 2018 -->
<title>org.apache.beam.runners.dataflow (Apache Beam 2.8.0-SNAPSHOT)</title>
<meta name="date" content="2018-10-18">
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="org.apache.beam.runners.dataflow (Apache Beam 2.8.0-SNAPSHOT)";
}
}
catch(err) {
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../overview-summary.html">Overview</a></li>
<li class="navBarCell1Rev">Package</li>
<li>Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../org/apache/beam/runners/apex/package-summary.html">Prev&nbsp;Package</a></li>
<li><a href="../../../../../org/apache/beam/runners/dataflow/options/package-summary.html">Next&nbsp;Package</a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../index.html?org/apache/beam/runners/dataflow/package-summary.html" target="_top">Frames</a></li>
<li><a href="package-summary.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<div class="header">
<h1 title="Package" class="title">Package&nbsp;org.apache.beam.runners.dataflow</h1>
<div class="docSummary">
<div class="block">Provides a Beam runner that executes pipelines on the Google Cloud Dataflow service.</div>
</div>
<p>See:&nbsp;<a href="#package.description">Description</a></p>
</div>
<div class="contentContainer">
<ul class="blockList">
<li class="blockList">
<table class="typeSummary" border="0" cellpadding="3" cellspacing="0" summary="Interface Summary table, listing interfaces, and an explanation">
<caption><span>Interface Summary</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Interface</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tbody>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/TestDataflowPipelineOptions.html" title="interface in org.apache.beam.runners.dataflow">TestDataflowPipelineOptions</a></td>
<td class="colLast">
<div class="block">A set of options used to configure the <a href="../../../../../org/apache/beam/sdk/testing/TestPipeline.html" title="class in org.apache.beam.sdk.testing"><code>TestPipeline</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/TransformTranslator.html" title="interface in org.apache.beam.runners.dataflow">TransformTranslator</a>&lt;TransformT extends <a href="../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a>&gt;</td>
<td class="colLast">
<div class="block">A <a href="../../../../../org/apache/beam/runners/dataflow/TransformTranslator.html" title="interface in org.apache.beam.runners.dataflow"><code>TransformTranslator</code></a> knows how to translate a particular subclass of <a href="../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms"><code>PTransform</code></a>
for the Cloud Dataflow service.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/TransformTranslator.StepTranslationContext.html" title="interface in org.apache.beam.runners.dataflow">TransformTranslator.StepTranslationContext</a></td>
<td class="colLast">
<div class="block">The interface for a <a href="../../../../../org/apache/beam/runners/dataflow/TransformTranslator.html" title="interface in org.apache.beam.runners.dataflow"><code>TransformTranslator</code></a> to build a Dataflow step.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/TransformTranslator.TranslationContext.html" title="interface in org.apache.beam.runners.dataflow">TransformTranslator.TranslationContext</a></td>
<td class="colLast">
<div class="block">The interface provided to registered callbacks for interacting with the <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a>,
including reading and writing the values of <a href="../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values"><code>PCollection</code></a>s and side inputs.</div>
</td>
</tr>
</tbody>
</table>
</li>
<li class="blockList">
<table class="typeSummary" border="0" cellpadding="3" cellspacing="0" summary="Class Summary table, listing classes, and an explanation">
<caption><span>Class Summary</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Class</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tbody>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.html" title="class in org.apache.beam.runners.dataflow">BatchStatefulParDoOverrides</a></td>
<td class="colLast">
<div class="block"><code>PTransformOverrideFactories</code> that expands to correctly implement
stateful <a href="../../../../../org/apache/beam/sdk/transforms/ParDo.html" title="class in org.apache.beam.sdk.transforms"><code>ParDo</code></a> using window-unaware <code>BatchViewOverrides.GroupByKeyAndSortValuesOnly</code> to linearize
processing per key.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/BatchStatefulParDoOverrides.BatchStatefulDoFn.html" title="class in org.apache.beam.runners.dataflow">BatchStatefulParDoOverrides.BatchStatefulDoFn</a>&lt;K,V,OutputT&gt;</td>
<td class="colLast">
<div class="block">A key-preserving <a href="../../../../../org/apache/beam/sdk/transforms/DoFn.html" title="class in org.apache.beam.sdk.transforms"><code>DoFn</code></a> that explodes an iterable that has been grouped by key and
window.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/CreateDataflowView.html" title="class in org.apache.beam.runners.dataflow">CreateDataflowView</a>&lt;ElemT,ViewT&gt;</td>
<td class="colLast">
<div class="block">A <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a> marker class for creating a <a href="../../../../../org/apache/beam/sdk/values/PCollectionView.html" title="interface in org.apache.beam.sdk.values"><code>PCollectionView</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowClient.html" title="class in org.apache.beam.runners.dataflow">DataflowClient</a></td>
<td class="colLast">
<div class="block">Wrapper around the generated <code>Dataflow</code> client to provide common functionality.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineJob.html" title="class in org.apache.beam.runners.dataflow">DataflowPipelineJob</a></td>
<td class="colLast">
<div class="block">A DataflowPipelineJob represents a job submitted to Dataflow using <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.html" title="class in org.apache.beam.runners.dataflow">DataflowPipelineRegistrar</a></td>
<td class="colLast">
<div class="block">Contains the <a href="../../../../../org/apache/beam/sdk/options/PipelineOptionsRegistrar.html" title="interface in org.apache.beam.sdk.options"><code>PipelineOptionsRegistrar</code></a> and <code>PipelineRunnerRegistrar</code> for the <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.Options.html" title="class in org.apache.beam.runners.dataflow">DataflowPipelineRegistrar.Options</a></td>
<td class="colLast">
<div class="block">Register the <a href="../../../../../org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.html" title="interface in org.apache.beam.runners.dataflow.options"><code>DataflowPipelineOptions</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.Runner.html" title="class in org.apache.beam.runners.dataflow">DataflowPipelineRegistrar.Runner</a></td>
<td class="colLast">
<div class="block">Register the <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineTranslator.html" title="class in org.apache.beam.runners.dataflow">DataflowPipelineTranslator</a></td>
<td class="colLast">
<div class="block"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineTranslator.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowPipelineTranslator</code></a> knows how to translate <a href="../../../../../org/apache/beam/sdk/Pipeline.html" title="class in org.apache.beam.sdk"><code>Pipeline</code></a> objects into Cloud
Dataflow Service API <code>Job</code>s.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineTranslator.JobSpecification.html" title="class in org.apache.beam.runners.dataflow">DataflowPipelineTranslator.JobSpecification</a></td>
<td class="colLast">
<div class="block">The result of a job translation.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow">DataflowRunner</a></td>
<td class="colLast">
<div class="block">A <a href="../../../../../org/apache/beam/sdk/PipelineRunner.html" title="class in org.apache.beam.sdk"><code>PipelineRunner</code></a> that executes the operations in the pipeline by first translating them
to the Dataflow representation using the <a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineTranslator.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowPipelineTranslator</code></a> and then submitting
them to a Dataflow service for execution.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.StreamingPCollectionViewWriterFn.html" title="class in org.apache.beam.runners.dataflow">DataflowRunner.StreamingPCollectionViewWriterFn</a>&lt;T&gt;</td>
<td class="colLast">
<div class="block">A marker <a href="../../../../../org/apache/beam/sdk/transforms/DoFn.html" title="class in org.apache.beam.sdk.transforms"><code>DoFn</code></a> for writing the contents of a <a href="../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values"><code>PCollection</code></a> to a streaming <a href="../../../../../org/apache/beam/sdk/values/PCollectionView.html" title="interface in org.apache.beam.sdk.values"><code>PCollectionView</code></a> backend implementation.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunnerHooks.html" title="class in org.apache.beam.runners.dataflow">DataflowRunnerHooks</a></td>
<td class="colLast">
<div class="block">An instance of this class can be passed to the <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a> to add user defined hooks
to be invoked at various times during pipeline execution.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunnerInfo.html" title="class in org.apache.beam.runners.dataflow">DataflowRunnerInfo</a></td>
<td class="colLast">
<div class="block">Populates versioning and other information for <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.html" title="class in org.apache.beam.runners.dataflow">PrimitiveParDoSingleFactory</a>&lt;InputT,OutputT&gt;</td>
<td class="colLast">
<div class="block">A <code>PTransformOverrideFactory</code> that produces <a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.ParDoSingle.html" title="class in org.apache.beam.runners.dataflow"><code>PrimitiveParDoSingleFactory.ParDoSingle</code></a> instances from <code>ParDo.SingleOutput</code> instances.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.ParDoSingle.html" title="class in org.apache.beam.runners.dataflow">PrimitiveParDoSingleFactory.ParDoSingle</a>&lt;InputT,OutputT&gt;</td>
<td class="colLast">
<div class="block">A single-output primitive <a href="../../../../../org/apache/beam/sdk/transforms/ParDo.html" title="class in org.apache.beam.sdk.transforms"><code>ParDo</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.PayloadTranslator.html" title="class in org.apache.beam.runners.dataflow">PrimitiveParDoSingleFactory.PayloadTranslator</a></td>
<td class="colLast">
<div class="block">A translator for <a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.ParDoSingle.html" title="class in org.apache.beam.runners.dataflow"><code>PrimitiveParDoSingleFactory.ParDoSingle</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.Registrar.html" title="class in org.apache.beam.runners.dataflow">PrimitiveParDoSingleFactory.Registrar</a></td>
<td class="colLast">
<div class="block">Registers <a href="../../../../../org/apache/beam/runners/dataflow/PrimitiveParDoSingleFactory.PayloadTranslator.html" title="class in org.apache.beam.runners.dataflow"><code>PrimitiveParDoSingleFactory.PayloadTranslator</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/TestDataflowRunner.html" title="class in org.apache.beam.runners.dataflow">TestDataflowRunner</a></td>
<td class="colLast">
<div class="block"><a href="../../../../../org/apache/beam/runners/dataflow/TestDataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>TestDataflowRunner</code></a> is a pipeline runner that wraps a <a href="../../../../../org/apache/beam/runners/dataflow/DataflowRunner.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowRunner</code></a> when running
tests against the <a href="../../../../../org/apache/beam/sdk/testing/TestPipeline.html" title="class in org.apache.beam.sdk.testing"><code>TestPipeline</code></a>.</div>
</td>
</tr>
</tbody>
</table>
</li>
<li class="blockList">
<table class="typeSummary" border="0" cellpadding="3" cellspacing="0" summary="Exception Summary table, listing exceptions, and an explanation">
<caption><span>Exception Summary</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Exception</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tbody>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowJobAlreadyExistsException.html" title="class in org.apache.beam.runners.dataflow">DataflowJobAlreadyExistsException</a></td>
<td class="colLast">
<div class="block">An exception that is thrown if the unique job name constraint of the Dataflow service is broken
because an existing job with the same job name is currently active.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowJobAlreadyUpdatedException.html" title="class in org.apache.beam.runners.dataflow">DataflowJobAlreadyUpdatedException</a></td>
<td class="colLast">
<div class="block">An exception that is thrown if the existing job has already been updated within the Dataflow
service and is no longer able to be updated.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowJobException.html" title="class in org.apache.beam.runners.dataflow">DataflowJobException</a></td>
<td class="colLast">
<div class="block">A <code>RuntimeException</code> that contains information about a <a href="../../../../../org/apache/beam/runners/dataflow/DataflowPipelineJob.html" title="class in org.apache.beam.runners.dataflow"><code>DataflowPipelineJob</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/beam/runners/dataflow/DataflowServiceException.html" title="class in org.apache.beam.runners.dataflow">DataflowServiceException</a></td>
<td class="colLast">
<div class="block">Signals there was an error retrieving information about a job from the Cloud Dataflow Service.</div>
</td>
</tr>
</tbody>
</table>
</li>
</ul>
<a name="package.description">
<!-- -->
</a>
<h2 title="Package org.apache.beam.runners.dataflow Description">Package org.apache.beam.runners.dataflow Description</h2>
<div class="block">Provides a Beam runner that executes pipelines on the Google Cloud Dataflow service.</div>
</div>
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../overview-summary.html">Overview</a></li>
<li class="navBarCell1Rev">Package</li>
<li>Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../org/apache/beam/runners/apex/package-summary.html">Prev&nbsp;Package</a></li>
<li><a href="../../../../../org/apache/beam/runners/dataflow/options/package-summary.html">Next&nbsp;Package</a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../index.html?org/apache/beam/runners/dataflow/package-summary.html" target="_top">Frames</a></li>
<li><a href="package-summary.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>