| <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> |
| <!-- NewPage --> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (1.8.0_181-google-v7) on Wed Aug 14 17:37:12 PDT 2019 --> |
| <title>CreateStream (Apache Beam 2.15.0-SNAPSHOT)</title> |
| <meta name="date" content="2019-08-14"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| <script type="text/javascript" src="../../../../../../script.js"></script> |
| </head> |
| <body> |
| <script type="text/javascript"><!-- |
| try { |
| if (location.href.indexOf('is-external=true') == -1) { |
| parent.document.title="CreateStream (Apache Beam 2.15.0-SNAPSHOT)"; |
| } |
| } |
| catch(err) { |
| } |
| //--> |
| var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":9,"i12":9}; |
| var tabs = {65535:["t0","All Methods"],1:["t1","Static Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]}; |
| var altColor = "altColor"; |
| var rowColor = "rowColor"; |
| var tableTab = "tableTab"; |
| var activeTableTab = "activeTableTab"; |
| </script> |
| <noscript> |
| <div>JavaScript is disabled on your browser.</div> |
| </noscript> |
| <!-- ========= START OF TOP NAVBAR ======= --> |
| <div class="topNav"><a name="navbar.top"> |
| <!-- --> |
| </a> |
| <div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div> |
| <a name="navbar.top.firstrow"> |
| <!-- --> |
| </a> |
| <ul class="navList" title="Navigation"> |
| <li><a href="../../../../../../overview-summary.html">Overview</a></li> |
| <li><a href="package-summary.html">Package</a></li> |
| <li class="navBarCell1Rev">Class</li> |
| <li><a href="package-tree.html">Tree</a></li> |
| <li><a href="../../../../../../deprecated-list.html">Deprecated</a></li> |
| <li><a href="../../../../../../index-all.html">Index</a></li> |
| <li><a href="../../../../../../help-doc.html">Help</a></li> |
| </ul> |
| </div> |
| <div class="subNav"> |
| <ul class="navList"> |
| <li><a href="../../../../../../org/apache/beam/runners/spark/io/ConsoleIO.Write.Unbound.html" title="class in org.apache.beam.runners.spark.io"><span class="typeNameLink">Prev Class</span></a></li> |
| <li><a href="../../../../../../org/apache/beam/runners/spark/io/EmptyCheckpointMark.html" title="class in org.apache.beam.runners.spark.io"><span class="typeNameLink">Next Class</span></a></li> |
| </ul> |
| <ul class="navList"> |
| <li><a href="../../../../../../index.html?org/apache/beam/runners/spark/io/CreateStream.html" target="_top">Frames</a></li> |
| <li><a href="CreateStream.html" target="_top">No Frames</a></li> |
| </ul> |
| <ul class="navList" id="allclasses_navbar_top"> |
| <li><a href="../../../../../../allclasses-noframe.html">All Classes</a></li> |
| </ul> |
| <div> |
| <script type="text/javascript"><!-- |
| allClassesLink = document.getElementById("allclasses_navbar_top"); |
| if(window==top) { |
| allClassesLink.style.display = "block"; |
| } |
| else { |
| allClassesLink.style.display = "none"; |
| } |
| //--> |
| </script> |
| </div> |
| <div> |
| <ul class="subNavList"> |
| <li>Summary: </li> |
| <li>Nested | </li> |
| <li><a href="#field.summary">Field</a> | </li> |
| <li>Constr | </li> |
| <li><a href="#method.summary">Method</a></li> |
| </ul> |
| <ul class="subNavList"> |
| <li>Detail: </li> |
| <li><a href="#field.detail">Field</a> | </li> |
| <li>Constr | </li> |
| <li><a href="#method.detail">Method</a></li> |
| </ul> |
| </div> |
| <a name="skip.navbar.top"> |
| <!-- --> |
| </a></div> |
| <!-- ========= END OF TOP NAVBAR ========= --> |
| <!-- ======== START OF CLASS DATA ======== --> |
| <div class="header"> |
| <div class="subTitle">org.apache.beam.runners.spark.io</div> |
| <h2 title="Class CreateStream" class="title">Class CreateStream<T></h2> |
| </div> |
| <div class="contentContainer"> |
| <ul class="inheritance"> |
| <li>java.lang.Object</li> |
| <li> |
| <ul class="inheritance"> |
| <li><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">org.apache.beam.sdk.transforms.PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><T>></li> |
| <li> |
| <ul class="inheritance"> |
| <li>org.apache.beam.runners.spark.io.CreateStream<T></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <div class="description"> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <dl> |
| <dt><span class="paramLabel">Type Parameters:</span></dt> |
| <dd><code>T</code> - The type of the element in this stream.</dd> |
| </dl> |
| <dl> |
| <dt>All Implemented Interfaces:</dt> |
| <dd>java.io.Serializable, <a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></dd> |
| </dl> |
| <hr> |
| <br> |
| <pre>public final class <span class="typeNameLabel">CreateStream<T></span> |
| extends <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><T>></pre> |
| <div class="block">Create an input stream from Queue. For SparkRunner tests only. |
| |
| <p>To properly compose a stream of micro-batches with their Watermarks, please keep in mind that |
| eventually there a two queues here - one for batches and another for Watermarks. |
| |
| <p>While both queues advance according to Spark's batch-interval, there is a slight difference in |
| how data is pushed into the stream compared to the advancement of Watermarks since Watermarks |
| advance onBatchCompleted hook call so if you'd want to set the watermark advance for a specific |
| batch it should be called before that batch. Also keep in mind that being a queue that is polled |
| per batch interval, if there is a need to "hold" the same Watermark without advancing it it |
| should be stated explicitly or the Watermark will advance as soon as it can (in the next batch |
| completed hook). |
| |
| <p>Example 1: |
| |
| <pre><code> |
| CreateStream.of(StringUtf8Coder.of(), batchDuration) |
| .nextBatch( |
| TimestampedValue.of("foo", endOfGlobalWindow), |
| TimestampedValue.of("bar", endOfGlobalWindow)) |
| .advanceNextBatchWatermarkToInfinity(); |
| </code></pre> |
| |
| The first batch will see the default start-of-time WM of <a href="../../../../../../org/apache/beam/sdk/transforms/windowing/BoundedWindow.html#TIMESTAMP_MIN_VALUE"><code>BoundedWindow.TIMESTAMP_MIN_VALUE</code></a> and any following batch will see the end-of-time WM <a href="../../../../../../org/apache/beam/sdk/transforms/windowing/BoundedWindow.html#TIMESTAMP_MAX_VALUE"><code>BoundedWindow.TIMESTAMP_MAX_VALUE</code></a>. |
| |
| <p>Example 2: |
| |
| <pre><code> |
| CreateStream.of(VarIntCoder.of(), batchDuration) |
| .nextBatch( |
| TimestampedValue.of(1, instant)) |
| .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(20))) |
| .nextBatch( |
| TimestampedValue.of(2, instant)) |
| .nextBatch( |
| TimestampedValue.of(3, instant)) |
| .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(30))) |
| </code></pre> |
| |
| <p>The first batch will see the start-of-time WM and the second will see the advanced (+20 min.) |
| WM. The third WM will see the WM advanced to +30 min, because this is the next advancement of the |
| WM regardless of where it ws called in the construction of CreateStream.</div> |
| <dl> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../serialized-form.html#org.apache.beam.runners.spark.io.CreateStream">Serialized Form</a></dd> |
| </dl> |
| </li> |
| </ul> |
| </div> |
| <div class="summary"> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <!-- =========== FIELD SUMMARY =========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="field.summary"> |
| <!-- --> |
| </a> |
| <h3>Field Summary</h3> |
| <table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Field Summary table, listing fields, and an explanation"> |
| <caption><span>Fields</span><span class="tabEnd"> </span></caption> |
| <tr> |
| <th class="colFirst" scope="col">Modifier and Type</th> |
| <th class="colLast" scope="col">Field and Description</th> |
| </tr> |
| <tr class="altColor"> |
| <td class="colFirst"><code>static java.lang.String</code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#TRANSFORM_URN">TRANSFORM_URN</a></span></code> </td> |
| </tr> |
| </table> |
| <ul class="blockList"> |
| <li class="blockList"><a name="fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform"> |
| <!-- --> |
| </a> |
| <h3>Fields inherited from class org.apache.beam.sdk.transforms.<a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a></h3> |
| <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#name">name</a></code></li> |
| </ul> |
| </li> |
| </ul> |
| <!-- ========== METHOD SUMMARY =========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="method.summary"> |
| <!-- --> |
| </a> |
| <h3>Method Summary</h3> |
| <table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation"> |
| <caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd"> </span></span><span id="t1" class="tableTab"><span><a href="javascript:show(1);">Static Methods</a></span><span class="tabEnd"> </span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd"> </span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd"> </span></span></caption> |
| <tr> |
| <th class="colFirst" scope="col">Modifier and Type</th> |
| <th class="colLast" scope="col">Method and Description</th> |
| </tr> |
| <tr id="i0" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#advanceNextBatchWatermarkToInfinity--">advanceNextBatchWatermarkToInfinity</a></span>()</code> |
| <div class="block">Advances the watermark in the next batch to the end-of-time.</div> |
| </td> |
| </tr> |
| <tr id="i1" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#advanceWatermarkForNextBatch-org.joda.time.Instant-">advanceWatermarkForNextBatch</a></span>(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a> newWatermark)</code> |
| <div class="block">Advances the watermark in the next batch.</div> |
| </td> |
| </tr> |
| <tr id="i2" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#emptyBatch--">emptyBatch</a></span>()</code> |
| <div class="block">Adds an empty batch.</div> |
| </td> |
| </tr> |
| <tr id="i3" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#expand-org.apache.beam.sdk.values.PBegin-">expand</a></span>(<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a> input)</code> |
| <div class="block">Override this method to specify how this <code>PTransform</code> should be expanded on the given |
| <code>InputT</code>.</div> |
| </td> |
| </tr> |
| <tr id="i4" class="altColor"> |
| <td class="colFirst"><code>long</code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#getBatchDuration--">getBatchDuration</a></span>()</code> </td> |
| </tr> |
| <tr id="i5" class="rowColor"> |
| <td class="colFirst"><code>java.util.Queue<java.lang.Iterable<<a href="../../../../../../org/apache/beam/sdk/values/TimestampedValue.html" title="class in org.apache.beam.sdk.values">TimestampedValue</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>>>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#getBatches--">getBatches</a></span>()</code> |
| <div class="block">Get the underlying queue representing the mock stream of micro-batches.</div> |
| </td> |
| </tr> |
| <tr id="i6" class="altColor"> |
| <td class="colFirst"><code>java.util.Queue<<a href="../../../../../../org/apache/beam/runners/spark/util/GlobalWatermarkHolder.SparkWatermarks.html" title="class in org.apache.beam.runners.spark.util">GlobalWatermarkHolder.SparkWatermarks</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#getTimes--">getTimes</a></span>()</code> |
| <div class="block">Get times so they can be pushed into the <a href="../../../../../../org/apache/beam/runners/spark/util/GlobalWatermarkHolder.html" title="class in org.apache.beam.runners.spark.util"><code>GlobalWatermarkHolder</code></a>.</div> |
| </td> |
| </tr> |
| <tr id="i7" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#initialSystemTimeAt-org.joda.time.Instant-">initialSystemTimeAt</a></span>(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a> initialSystemTime)</code> |
| <div class="block">Set the initial synchronized processing time.</div> |
| </td> |
| </tr> |
| <tr id="i8" class="altColor"> |
| <td class="colFirst"><code>boolean</code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#isForceWatermarkSync--">isForceWatermarkSync</a></span>()</code> </td> |
| </tr> |
| <tr id="i9" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#nextBatch-T...-">nextBatch</a></span>(<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>... batchElements)</code> |
| <div class="block">For non-timestamped elements.</div> |
| </td> |
| </tr> |
| <tr id="i10" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#nextBatch-org.apache.beam.sdk.values.TimestampedValue...-">nextBatch</a></span>(<a href="../../../../../../org/apache/beam/sdk/values/TimestampedValue.html" title="class in org.apache.beam.sdk.values">TimestampedValue</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>>... batchElements)</code> |
| <div class="block">Enqueue next micro-batch elements.</div> |
| </td> |
| </tr> |
| <tr id="i11" class="rowColor"> |
| <td class="colFirst"><code>static <T> <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><T></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#of-org.apache.beam.sdk.coders.Coder-org.joda.time.Duration-">of</a></span>(<a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><T> coder, |
| <a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> batchDuration)</code> |
| <div class="block">Creates a new Spark based stream without forced watermark sync, intended for test purposes.</div> |
| </td> |
| </tr> |
| <tr id="i12" class="altColor"> |
| <td class="colFirst"><code>static <T> <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><T></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#of-org.apache.beam.sdk.coders.Coder-org.joda.time.Duration-boolean-">of</a></span>(<a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><T> coder, |
| <a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> batchDuration, |
| boolean forceWatermarkSync)</code> |
| <div class="block">Creates a new Spark based stream intended for test purposes.</div> |
| </td> |
| </tr> |
| </table> |
| <ul class="blockList"> |
| <li class="blockList"><a name="methods.inherited.from.class.org.apache.beam.sdk.transforms.PTransform"> |
| <!-- --> |
| </a> |
| <h3>Methods inherited from class org.apache.beam.sdk.transforms.<a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a></h3> |
| <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#compose-org.apache.beam.sdk.transforms.SerializableFunction-">compose</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#compose-java.lang.String-org.apache.beam.sdk.transforms.SerializableFunction-">compose</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getAdditionalInputs--">getAdditionalInputs</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder--">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder-InputT-">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder-InputT-org.apache.beam.sdk.values.PCollection-">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getKindString--">getKindString</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getName--">getName</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#toString--">toString</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#validate-org.apache.beam.sdk.options.PipelineOptions-">validate</a></code></li> |
| </ul> |
| <ul class="blockList"> |
| <li class="blockList"><a name="methods.inherited.from.class.java.lang.Object"> |
| <!-- --> |
| </a> |
| <h3>Methods inherited from class java.lang.Object</h3> |
| <code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait</code></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| <div class="details"> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <!-- ============ FIELD DETAIL =========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="field.detail"> |
| <!-- --> |
| </a> |
| <h3>Field Detail</h3> |
| <a name="TRANSFORM_URN"> |
| <!-- --> |
| </a> |
| <ul class="blockListLast"> |
| <li class="blockList"> |
| <h4>TRANSFORM_URN</h4> |
| <pre>public static final java.lang.String TRANSFORM_URN</pre> |
| <dl> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../constant-values.html#org.apache.beam.runners.spark.io.CreateStream.TRANSFORM_URN">Constant Field Values</a></dd> |
| </dl> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <!-- ============ METHOD DETAIL ========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="method.detail"> |
| <!-- --> |
| </a> |
| <h3>Method Detail</h3> |
| <a name="of-org.apache.beam.sdk.coders.Coder-org.joda.time.Duration-boolean-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>of</h4> |
| <pre>public static <T> <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><T> of(<a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><T> coder, |
| <a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> batchDuration, |
| boolean forceWatermarkSync)</pre> |
| <div class="block">Creates a new Spark based stream intended for test purposes.</div> |
| <dl> |
| <dt><span class="paramLabel">Parameters:</span></dt> |
| <dd><code>batchDuration</code> - the batch duration (interval) to be used for creating this stream.</dd> |
| <dd><code>coder</code> - the coder to be used for this stream.</dd> |
| <dd><code>forceWatermarkSync</code> - whether this stream should be synced with the advancement of the |
| watermark maintained by the <a href="../../../../../../org/apache/beam/runners/spark/util/GlobalWatermarkHolder.html" title="class in org.apache.beam.runners.spark.util"><code>GlobalWatermarkHolder</code></a>.</dd> |
| </dl> |
| </li> |
| </ul> |
| <a name="of-org.apache.beam.sdk.coders.Coder-org.joda.time.Duration-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>of</h4> |
| <pre>public static <T> <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><T> of(<a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><T> coder, |
| <a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> batchDuration)</pre> |
| <div class="block">Creates a new Spark based stream without forced watermark sync, intended for test purposes. See |
| also <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html#of-org.apache.beam.sdk.coders.Coder-org.joda.time.Duration-boolean-"><code>of(Coder, Duration, boolean)</code></a>.</div> |
| </li> |
| </ul> |
| <a name="nextBatch-org.apache.beam.sdk.values.TimestampedValue...-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>nextBatch</h4> |
| <pre>@SafeVarargs |
| public final <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> nextBatch(<a href="../../../../../../org/apache/beam/sdk/values/TimestampedValue.html" title="class in org.apache.beam.sdk.values">TimestampedValue</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>>... batchElements)</pre> |
| <div class="block">Enqueue next micro-batch elements. This is backed by a <code>Queue</code> so stream input order |
| would keep the population order (FIFO).</div> |
| </li> |
| </ul> |
| <a name="nextBatch-java.lang.Object:A-"> |
| <!-- --> |
| </a><a name="nextBatch-T...-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>nextBatch</h4> |
| <pre>@SafeVarargs |
| public final <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> nextBatch(<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>... batchElements)</pre> |
| <div class="block">For non-timestamped elements.</div> |
| </li> |
| </ul> |
| <a name="emptyBatch--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>emptyBatch</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> emptyBatch()</pre> |
| <div class="block">Adds an empty batch.</div> |
| </li> |
| </ul> |
| <a name="initialSystemTimeAt-org.joda.time.Instant-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>initialSystemTimeAt</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> initialSystemTimeAt(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a> initialSystemTime)</pre> |
| <div class="block">Set the initial synchronized processing time.</div> |
| </li> |
| </ul> |
| <a name="advanceWatermarkForNextBatch-org.joda.time.Instant-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>advanceWatermarkForNextBatch</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> advanceWatermarkForNextBatch(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.1/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a> newWatermark)</pre> |
| <div class="block">Advances the watermark in the next batch.</div> |
| </li> |
| </ul> |
| <a name="advanceNextBatchWatermarkToInfinity--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>advanceNextBatchWatermarkToInfinity</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="class in org.apache.beam.runners.spark.io">CreateStream</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> advanceNextBatchWatermarkToInfinity()</pre> |
| <div class="block">Advances the watermark in the next batch to the end-of-time.</div> |
| </li> |
| </ul> |
| <a name="getBatchDuration--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>getBatchDuration</h4> |
| <pre>public long getBatchDuration()</pre> |
| </li> |
| </ul> |
| <a name="getBatches--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>getBatches</h4> |
| <pre>public java.util.Queue<java.lang.Iterable<<a href="../../../../../../org/apache/beam/sdk/values/TimestampedValue.html" title="class in org.apache.beam.sdk.values">TimestampedValue</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>>>> getBatches()</pre> |
| <div class="block">Get the underlying queue representing the mock stream of micro-batches.</div> |
| </li> |
| </ul> |
| <a name="getTimes--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>getTimes</h4> |
| <pre>public java.util.Queue<<a href="../../../../../../org/apache/beam/runners/spark/util/GlobalWatermarkHolder.SparkWatermarks.html" title="class in org.apache.beam.runners.spark.util">GlobalWatermarkHolder.SparkWatermarks</a>> getTimes()</pre> |
| <div class="block">Get times so they can be pushed into the <a href="../../../../../../org/apache/beam/runners/spark/util/GlobalWatermarkHolder.html" title="class in org.apache.beam.runners.spark.util"><code>GlobalWatermarkHolder</code></a>.</div> |
| </li> |
| </ul> |
| <a name="isForceWatermarkSync--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>isForceWatermarkSync</h4> |
| <pre>public boolean isForceWatermarkSync()</pre> |
| </li> |
| </ul> |
| <a name="expand-org.apache.beam.sdk.values.PBegin-"> |
| <!-- --> |
| </a> |
| <ul class="blockListLast"> |
| <li class="blockList"> |
| <h4>expand</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>> expand(<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a> input)</pre> |
| <div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#expand-InputT-">PTransform</a></code></span></div> |
| <div class="block">Override this method to specify how this <code>PTransform</code> should be expanded on the given |
| <code>InputT</code>. |
| |
| <p>NOTE: This method should not be called directly. Instead apply the <code>PTransform</code> should |
| be applied to the <code>InputT</code> using the <code>apply</code> method. |
| |
| <p>Composite transforms, which are defined in terms of other transforms, should return the |
| output of one of the composed transforms. Non-composite transforms, which do not apply any |
| transforms internally, should return a new unbound output and register evaluators (via |
| backend-specific registration methods).</div> |
| <dl> |
| <dt><span class="overrideSpecifyLabel">Specified by:</span></dt> |
| <dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#expand-InputT-">expand</a></code> in class <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/runners/spark/io/CreateStream.html" title="type parameter in CreateStream">T</a>>></code></dd> |
| </dl> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <!-- ========= END OF CLASS DATA ========= --> |
| <!-- ======= START OF BOTTOM NAVBAR ====== --> |
| <div class="bottomNav"><a name="navbar.bottom"> |
| <!-- --> |
| </a> |
| <div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div> |
| <a name="navbar.bottom.firstrow"> |
| <!-- --> |
| </a> |
| <ul class="navList" title="Navigation"> |
| <li><a href="../../../../../../overview-summary.html">Overview</a></li> |
| <li><a href="package-summary.html">Package</a></li> |
| <li class="navBarCell1Rev">Class</li> |
| <li><a href="package-tree.html">Tree</a></li> |
| <li><a href="../../../../../../deprecated-list.html">Deprecated</a></li> |
| <li><a href="../../../../../../index-all.html">Index</a></li> |
| <li><a href="../../../../../../help-doc.html">Help</a></li> |
| </ul> |
| </div> |
| <div class="subNav"> |
| <ul class="navList"> |
| <li><a href="../../../../../../org/apache/beam/runners/spark/io/ConsoleIO.Write.Unbound.html" title="class in org.apache.beam.runners.spark.io"><span class="typeNameLink">Prev Class</span></a></li> |
| <li><a href="../../../../../../org/apache/beam/runners/spark/io/EmptyCheckpointMark.html" title="class in org.apache.beam.runners.spark.io"><span class="typeNameLink">Next Class</span></a></li> |
| </ul> |
| <ul class="navList"> |
| <li><a href="../../../../../../index.html?org/apache/beam/runners/spark/io/CreateStream.html" target="_top">Frames</a></li> |
| <li><a href="CreateStream.html" target="_top">No Frames</a></li> |
| </ul> |
| <ul class="navList" id="allclasses_navbar_bottom"> |
| <li><a href="../../../../../../allclasses-noframe.html">All Classes</a></li> |
| </ul> |
| <div> |
| <script type="text/javascript"><!-- |
| allClassesLink = document.getElementById("allclasses_navbar_bottom"); |
| if(window==top) { |
| allClassesLink.style.display = "block"; |
| } |
| else { |
| allClassesLink.style.display = "none"; |
| } |
| //--> |
| </script> |
| </div> |
| <div> |
| <ul class="subNavList"> |
| <li>Summary: </li> |
| <li>Nested | </li> |
| <li><a href="#field.summary">Field</a> | </li> |
| <li>Constr | </li> |
| <li><a href="#method.summary">Method</a></li> |
| </ul> |
| <ul class="subNavList"> |
| <li>Detail: </li> |
| <li><a href="#field.detail">Field</a> | </li> |
| <li>Constr | </li> |
| <li><a href="#method.detail">Method</a></li> |
| </ul> |
| </div> |
| <a name="skip.navbar.bottom"> |
| <!-- --> |
| </a></div> |
| <!-- ======== END OF BOTTOM NAVBAR ======= --> |
| </body> |
| </html> |
