| <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> |
| <!-- NewPage --> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (1.8.0_181-google-v7) on Mon Jan 27 16:42:28 PST 2020 --> |
| <title>KafkaIO.Read (Apache Beam 2.20.0-SNAPSHOT)</title> |
| <meta name="date" content="2020-01-27"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| <script type="text/javascript" src="../../../../../../script.js"></script> |
| </head> |
| <body> |
| <script type="text/javascript"><!-- |
| try { |
| if (location.href.indexOf('is-external=true') == -1) { |
| parent.document.title="KafkaIO.Read (Apache Beam 2.20.0-SNAPSHOT)"; |
| } |
| } |
| catch(err) { |
| } |
| //--> |
| var methods = {"i0":10,"i1":10,"i2":10,"i3":42,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":42,"i19":42,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":42,"i27":42}; |
| var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"],32:["t6","Deprecated Methods"]}; |
| var altColor = "altColor"; |
| var rowColor = "rowColor"; |
| var tableTab = "tableTab"; |
| var activeTableTab = "activeTableTab"; |
| </script> |
| <noscript> |
| <div>JavaScript is disabled on your browser.</div> |
| </noscript> |
| <!-- ========= START OF TOP NAVBAR ======= --> |
| <div class="topNav"><a name="navbar.top"> |
| <!-- --> |
| </a> |
| <div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div> |
| <a name="navbar.top.firstrow"> |
| <!-- --> |
| </a> |
| <ul class="navList" title="Navigation"> |
| <li><a href="../../../../../../overview-summary.html">Overview</a></li> |
| <li><a href="package-summary.html">Package</a></li> |
| <li class="navBarCell1Rev">Class</li> |
| <li><a href="package-tree.html">Tree</a></li> |
| <li><a href="../../../../../../deprecated-list.html">Deprecated</a></li> |
| <li><a href="../../../../../../index-all.html">Index</a></li> |
| <li><a href="../../../../../../help-doc.html">Help</a></li> |
| </ul> |
| </div> |
| <div class="subNav"> |
| <ul class="navList"> |
| <li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html" title="class in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Prev Class</span></a></li> |
| <li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.External.html" title="class in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Next Class</span></a></li> |
| </ul> |
| <ul class="navList"> |
| <li><a href="../../../../../../index.html?org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" target="_top">Frames</a></li> |
| <li><a href="KafkaIO.Read.html" target="_top">No Frames</a></li> |
| </ul> |
| <ul class="navList" id="allclasses_navbar_top"> |
| <li><a href="../../../../../../allclasses-noframe.html">All Classes</a></li> |
| </ul> |
| <div> |
| <script type="text/javascript"><!-- |
| allClassesLink = document.getElementById("allclasses_navbar_top"); |
| if(window==top) { |
| allClassesLink.style.display = "block"; |
| } |
| else { |
| allClassesLink.style.display = "none"; |
| } |
| //--> |
| </script> |
| </div> |
| <div> |
| <ul class="subNavList"> |
| <li>Summary: </li> |
| <li><a href="#nested.class.summary">Nested</a> | </li> |
| <li><a href="#fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform">Field</a> | </li> |
| <li><a href="#constructor.summary">Constr</a> | </li> |
| <li><a href="#method.summary">Method</a></li> |
| </ul> |
| <ul class="subNavList"> |
| <li>Detail: </li> |
| <li>Field | </li> |
| <li><a href="#constructor.detail">Constr</a> | </li> |
| <li><a href="#method.detail">Method</a></li> |
| </ul> |
| </div> |
| <a name="skip.navbar.top"> |
| <!-- --> |
| </a></div> |
| <!-- ========= END OF TOP NAVBAR ========= --> |
| <!-- ======== START OF CLASS DATA ======== --> |
| <div class="header"> |
| <div class="subTitle">org.apache.beam.sdk.io.kafka</div> |
| <h2 title="Class KafkaIO.Read" class="title">Class KafkaIO.Read<K,V></h2> |
| </div> |
| <div class="contentContainer"> |
| <ul class="inheritance"> |
| <li>java.lang.Object</li> |
| <li> |
| <ul class="inheritance"> |
| <li><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">org.apache.beam.sdk.transforms.PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><K,V>>></li> |
| <li> |
| <ul class="inheritance"> |
| <li>org.apache.beam.sdk.io.kafka.KafkaIO.Read<K,V></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <div class="description"> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <dl> |
| <dt>All Implemented Interfaces:</dt> |
| <dd>java.io.Serializable, <a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></dd> |
| </dl> |
| <dl> |
| <dt>Enclosing class:</dt> |
| <dd><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO</a></dd> |
| </dl> |
| <hr> |
| <br> |
| <pre>public abstract static class <span class="typeNameLabel">KafkaIO.Read<K,V></span> |
| extends <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><K,V>>></pre> |
| <div class="block">A <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms"><code>PTransform</code></a> to read from Kafka topics. See <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html" title="class in org.apache.beam.sdk.io.kafka"><code>KafkaIO</code></a> for more information on |
| usage and configuration.</div> |
| <dl> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../serialized-form.html#org.apache.beam.sdk.io.kafka.KafkaIO.Read">Serialized Form</a></dd> |
| </dl> |
| </li> |
| </ul> |
| </div> |
| <div class="summary"> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <!-- ======== NESTED CLASS SUMMARY ======== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="nested.class.summary"> |
| <!-- --> |
| </a> |
| <h3>Nested Class Summary</h3> |
| <table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Nested Class Summary table, listing nested classes, and an explanation"> |
| <caption><span>Nested Classes</span><span class="tabEnd"> </span></caption> |
| <tr> |
| <th class="colFirst" scope="col">Modifier and Type</th> |
| <th class="colLast" scope="col">Class and Description</th> |
| </tr> |
| <tr class="altColor"> |
| <td class="colFirst"><code>static class </code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.External.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read.External</a></span></code> |
| <div class="block">Exposes <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.TypedWithoutMetadata.html" title="class in org.apache.beam.sdk.io.kafka"><code>KafkaIO.TypedWithoutMetadata</code></a> as an external transform for cross-language |
| usage.</div> |
| </td> |
| </tr> |
| </table> |
| </li> |
| </ul> |
| <!-- =========== FIELD SUMMARY =========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="field.summary"> |
| <!-- --> |
| </a> |
| <h3>Field Summary</h3> |
| <ul class="blockList"> |
| <li class="blockList"><a name="fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform"> |
| <!-- --> |
| </a> |
| <h3>Fields inherited from class org.apache.beam.sdk.transforms.<a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a></h3> |
| <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#name">name</a></code></li> |
| </ul> |
| </li> |
| </ul> |
| <!-- ======== CONSTRUCTOR SUMMARY ======== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="constructor.summary"> |
| <!-- --> |
| </a> |
| <h3>Constructor Summary</h3> |
| <table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation"> |
| <caption><span>Constructors</span><span class="tabEnd"> </span></caption> |
| <tr> |
| <th class="colOne" scope="col">Constructor and Description</th> |
| </tr> |
| <tr class="altColor"> |
| <td class="colOne"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#Read--">Read</a></span>()</code> </td> |
| </tr> |
| </table> |
| </li> |
| </ul> |
| <!-- ========== METHOD SUMMARY =========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="method.summary"> |
| <!-- --> |
| </a> |
| <h3>Method Summary</h3> |
| <table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation"> |
| <caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd"> </span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd"> </span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd"> </span></span><span id="t6" class="tableTab"><span><a href="javascript:show(32);">Deprecated Methods</a></span><span class="tabEnd"> </span></span></caption> |
| <tr> |
| <th class="colFirst" scope="col">Modifier and Type</th> |
| <th class="colLast" scope="col">Method and Description</th> |
| </tr> |
| <tr id="i0" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#commitOffsetsInFinalize--">commitOffsetsInFinalize</a></span>()</code> |
| <div class="block">Finalized offsets are committed to Kafka.</div> |
| </td> |
| </tr> |
| <tr id="i1" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#expand-org.apache.beam.sdk.values.PBegin-">expand</a></span>(<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a> input)</code> |
| <div class="block">Override this method to specify how this <code>PTransform</code> should be expanded on the given |
| <code>InputT</code>.</div> |
| </td> |
| </tr> |
| <tr id="i2" class="altColor"> |
| <td class="colFirst"><code>void</code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/display/DisplayData.Builder.html" title="interface in org.apache.beam.sdk.transforms.display">DisplayData.Builder</a> builder)</code> |
| <div class="block">Register display data for the given transform or component.</div> |
| </td> |
| </tr> |
| <tr id="i3" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#updateConsumerProperties-java.util.Map-">updateConsumerProperties</a></span>(java.util.Map<java.lang.String,java.lang.Object> configUpdates)</code> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> |
| <div class="block"><span class="deprecationComment">as of version 2.13. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withConsumerConfigUpdates-java.util.Map-"><code>withConsumerConfigUpdates(Map)</code></a> instead</span></div> |
| </div> |
| </td> |
| </tr> |
| <tr id="i4" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withBootstrapServers-java.lang.String-">withBootstrapServers</a></span>(java.lang.String bootstrapServers)</code> |
| <div class="block">Sets the bootstrap servers for the Kafka consumer.</div> |
| </td> |
| </tr> |
| <tr id="i5" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withConsumerConfigUpdates-java.util.Map-">withConsumerConfigUpdates</a></span>(java.util.Map<java.lang.String,java.lang.Object> configUpdates)</code> |
| <div class="block">Update configuration for the backend main consumer.</div> |
| </td> |
| </tr> |
| <tr id="i6" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withConsumerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-">withConsumerFactoryFn</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><java.util.Map<java.lang.String,java.lang.Object>,<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/Consumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer">Consumer</a><byte[],byte[]>> consumerFactoryFn)</code> |
| <div class="block">A factory to create Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/Consumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer"><code>Consumer</code></a> from consumer configuration.</div> |
| </td> |
| </tr> |
| <tr id="i7" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withCreateTime-org.joda.time.Duration-">withCreateTime</a></span>(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> maxDelay)</code> |
| <div class="block">Sets the timestamps policy based on <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaTimestampType.html#CREATE_TIME"><code>KafkaTimestampType.CREATE_TIME</code></a> timestamp of the |
| records.</div> |
| </td> |
| </tr> |
| <tr id="i8" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withKeyDeserializer-java.lang.Class-">withKeyDeserializer</a></span>(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>>> keyDeserializer)</code> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> to interpret key bytes read from Kafka.</div> |
| </td> |
| </tr> |
| <tr id="i9" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withKeyDeserializerAndCoder-java.lang.Class-org.apache.beam.sdk.coders.Coder-">withKeyDeserializerAndCoder</a></span>(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>>> keyDeserializer, |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>> keyCoder)</code> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> for interpreting key bytes read from Kafka along with a |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders"><code>Coder</code></a> for helping the Beam runner materialize key objects at runtime if necessary.</div> |
| </td> |
| </tr> |
| <tr id="i10" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withLogAppendTime--">withLogAppendTime</a></span>()</code> |
| <div class="block">Sets <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicy</code></a> to <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.LogAppendTimePolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicyFactory.LogAppendTimePolicy</code></a>.</div> |
| </td> |
| </tr> |
| <tr id="i11" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withMaxNumRecords-long-">withMaxNumRecords</a></span>(long maxNumRecords)</code> |
| <div class="block">Similar to <a href="../../../../../../org/apache/beam/sdk/io/Read.Unbounded.html#withMaxNumRecords-long-"><code>Read.Unbounded.withMaxNumRecords(long)</code></a>.</div> |
| </td> |
| </tr> |
| <tr id="i12" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withMaxReadTime-org.joda.time.Duration-">withMaxReadTime</a></span>(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> maxReadTime)</code> |
| <div class="block">Similar to <a href="../../../../../../org/apache/beam/sdk/io/Read.Unbounded.html#withMaxReadTime-org.joda.time.Duration-"><code>Read.Unbounded.withMaxReadTime(Duration)</code></a>.</div> |
| </td> |
| </tr> |
| <tr id="i13" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withOffsetConsumerConfigOverrides-java.util.Map-">withOffsetConsumerConfigOverrides</a></span>(java.util.Map<java.lang.String,java.lang.Object> offsetConsumerConfig)</code> |
| <div class="block">Set additional configuration for the backend offset consumer.</div> |
| </td> |
| </tr> |
| <tr id="i14" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values">KV</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withoutMetadata--">withoutMetadata</a></span>()</code> |
| <div class="block">Returns a <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms"><code>PTransform</code></a> for PCollection of <a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values"><code>KV</code></a>, dropping Kafka metatdata.</div> |
| </td> |
| </tr> |
| <tr id="i15" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withProcessingTime--">withProcessingTime</a></span>()</code> |
| <div class="block">Sets <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicy</code></a> to <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.ProcessingTimePolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicyFactory.ProcessingTimePolicy</code></a>.</div> |
| </td> |
| </tr> |
| <tr id="i16" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withReadCommitted--">withReadCommitted</a></span>()</code> |
| <div class="block">Sets "isolation_level" to "read_committed" in Kafka consumer configuration.</div> |
| </td> |
| </tr> |
| <tr id="i17" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withStartReadTime-org.joda.time.Instant-">withStartReadTime</a></span>(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a> startReadTime)</code> |
| <div class="block">Use timestamp to set up start offset.</div> |
| </td> |
| </tr> |
| <tr id="i18" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampFn-org.apache.beam.sdk.transforms.SerializableFunction-">withTimestampFn</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values">KV</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> timestampFn)</code> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> |
| <div class="block"><span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| </div> |
| </td> |
| </tr> |
| <tr id="i19" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampFn2-org.apache.beam.sdk.transforms.SerializableFunction-">withTimestampFn2</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> timestampFn)</code> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> |
| <div class="block"><span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| </div> |
| </td> |
| </tr> |
| <tr id="i20" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-">withTimestampPolicyFactory</a></span>(<a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.html" title="interface in org.apache.beam.sdk.io.kafka">TimestampPolicyFactory</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> timestampPolicyFactory)</code> |
| <div class="block">Provide custom <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.html" title="interface in org.apache.beam.sdk.io.kafka"><code>TimestampPolicyFactory</code></a> to set event times and watermark for each |
| partition.</div> |
| </td> |
| </tr> |
| <tr id="i21" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTopic-java.lang.String-">withTopic</a></span>(java.lang.String topic)</code> |
| <div class="block">Sets the topic to read from.</div> |
| </td> |
| </tr> |
| <tr id="i22" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTopicPartitions-java.util.List-">withTopicPartitions</a></span>(java.util.List<<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/TopicPartition.html?is-external=true" title="class or interface in org.apache.kafka.common">TopicPartition</a>> topicPartitions)</code> |
| <div class="block">Sets a list of partitions to read from.</div> |
| </td> |
| </tr> |
| <tr id="i23" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTopics-java.util.List-">withTopics</a></span>(java.util.List<java.lang.String> topics)</code> |
| <div class="block">Sets a list of topics to read from.</div> |
| </td> |
| </tr> |
| <tr id="i24" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withValueDeserializer-java.lang.Class-">withValueDeserializer</a></span>(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>> valueDeserializer)</code> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> to interpret value bytes read from Kafka.</div> |
| </td> |
| </tr> |
| <tr id="i25" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withValueDeserializerAndCoder-java.lang.Class-org.apache.beam.sdk.coders.Coder-">withValueDeserializerAndCoder</a></span>(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>> valueDeserializer, |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> valueCoder)</code> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> for interpreting value bytes read from Kafka along with a |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders"><code>Coder</code></a> for helping the Beam runner materialize value objects at runtime if necessary.</div> |
| </td> |
| </tr> |
| <tr id="i26" class="altColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withWatermarkFn-org.apache.beam.sdk.transforms.SerializableFunction-">withWatermarkFn</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values">KV</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> watermarkFn)</code> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> |
| <div class="block"><span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| </div> |
| </td> |
| </tr> |
| <tr id="i27" class="rowColor"> |
| <td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>></code></td> |
| <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withWatermarkFn2-org.apache.beam.sdk.transforms.SerializableFunction-">withWatermarkFn2</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> watermarkFn)</code> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> |
| <div class="block"><span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| </div> |
| </td> |
| </tr> |
| </table> |
| <ul class="blockList"> |
| <li class="blockList"><a name="methods.inherited.from.class.org.apache.beam.sdk.transforms.PTransform"> |
| <!-- --> |
| </a> |
| <h3>Methods inherited from class org.apache.beam.sdk.transforms.<a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a></h3> |
| <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#compose-org.apache.beam.sdk.transforms.SerializableFunction-">compose</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#compose-java.lang.String-org.apache.beam.sdk.transforms.SerializableFunction-">compose</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getAdditionalInputs--">getAdditionalInputs</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder--">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder-InputT-">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder-InputT-org.apache.beam.sdk.values.PCollection-">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getKindString--">getKindString</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getName--">getName</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#toString--">toString</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#validate-org.apache.beam.sdk.options.PipelineOptions-">validate</a></code></li> |
| </ul> |
| <ul class="blockList"> |
| <li class="blockList"><a name="methods.inherited.from.class.java.lang.Object"> |
| <!-- --> |
| </a> |
| <h3>Methods inherited from class java.lang.Object</h3> |
| <code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait</code></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| <div class="details"> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <!-- ========= CONSTRUCTOR DETAIL ======== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="constructor.detail"> |
| <!-- --> |
| </a> |
| <h3>Constructor Detail</h3> |
| <a name="Read--"> |
| <!-- --> |
| </a> |
| <ul class="blockListLast"> |
| <li class="blockList"> |
| <h4>Read</h4> |
| <pre>public Read()</pre> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <!-- ============ METHOD DETAIL ========== --> |
| <ul class="blockList"> |
| <li class="blockList"><a name="method.detail"> |
| <!-- --> |
| </a> |
| <h3>Method Detail</h3> |
| <a name="withBootstrapServers-java.lang.String-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withBootstrapServers</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withBootstrapServers(java.lang.String bootstrapServers)</pre> |
| <div class="block">Sets the bootstrap servers for the Kafka consumer.</div> |
| </li> |
| </ul> |
| <a name="withTopic-java.lang.String-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withTopic</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withTopic(java.lang.String topic)</pre> |
| <div class="block">Sets the topic to read from. |
| |
| <p>See <code>KafkaUnboundedSource.split(int, PipelineOptions)</code> for description of how the |
| partitions are distributed among the splits.</div> |
| </li> |
| </ul> |
| <a name="withTopics-java.util.List-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withTopics</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withTopics(java.util.List<java.lang.String> topics)</pre> |
| <div class="block">Sets a list of topics to read from. All the partitions from each of the topics are read. |
| |
| <p>See <code>KafkaUnboundedSource.split(int, PipelineOptions)</code> for description of how the |
| partitions are distributed among the splits.</div> |
| </li> |
| </ul> |
| <a name="withTopicPartitions-java.util.List-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withTopicPartitions</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withTopicPartitions(java.util.List<<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/TopicPartition.html?is-external=true" title="class or interface in org.apache.kafka.common">TopicPartition</a>> topicPartitions)</pre> |
| <div class="block">Sets a list of partitions to read from. This allows reading only a subset of partitions for |
| one or more topics when (if ever) needed. |
| |
| <p>See <code>KafkaUnboundedSource.split(int, PipelineOptions)</code> for description of how the |
| partitions are distributed among the splits.</div> |
| </li> |
| </ul> |
| <a name="withKeyDeserializer-java.lang.Class-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withKeyDeserializer</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withKeyDeserializer(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>>> keyDeserializer)</pre> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> to interpret key bytes read from Kafka. |
| |
| <p>In addition, Beam also needs a <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders"><code>Coder</code></a> to serialize and deserialize key objects at |
| runtime. KafkaIO tries to infer a coder for the key based on the <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> class, |
| however in case that fails, you can use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withKeyDeserializerAndCoder-java.lang.Class-org.apache.beam.sdk.coders.Coder-"><code>withKeyDeserializerAndCoder(Class, Coder)</code></a> to |
| provide the key coder explicitly.</div> |
| </li> |
| </ul> |
| <a name="withKeyDeserializerAndCoder-java.lang.Class-org.apache.beam.sdk.coders.Coder-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withKeyDeserializerAndCoder</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withKeyDeserializerAndCoder(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>>> keyDeserializer, |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>> keyCoder)</pre> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> for interpreting key bytes read from Kafka along with a |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders"><code>Coder</code></a> for helping the Beam runner materialize key objects at runtime if necessary. |
| |
| <p>Use this method only if your pipeline doesn't work with plain <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withKeyDeserializer-java.lang.Class-"><code>withKeyDeserializer(Class)</code></a>.</div> |
| </li> |
| </ul> |
| <a name="withValueDeserializer-java.lang.Class-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withValueDeserializer</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withValueDeserializer(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>> valueDeserializer)</pre> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> to interpret value bytes read from Kafka. |
| |
| <p>In addition, Beam also needs a <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders"><code>Coder</code></a> to serialize and deserialize value objects at |
| runtime. KafkaIO tries to infer a coder for the value based on the <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> |
| class, however in case that fails, you can use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withValueDeserializerAndCoder-java.lang.Class-org.apache.beam.sdk.coders.Coder-"><code>withValueDeserializerAndCoder(Class, |
| Coder)</code></a> to provide the value coder explicitly.</div> |
| </li> |
| </ul> |
| <a name="withValueDeserializerAndCoder-java.lang.Class-org.apache.beam.sdk.coders.Coder-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withValueDeserializerAndCoder</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withValueDeserializerAndCoder(java.lang.Class<? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Deserializer</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>> valueDeserializer, |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders">Coder</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> valueCoder)</pre> |
| <div class="block">Sets a Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/common/serialization/Deserializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Deserializer</code></a> for interpreting value bytes read from Kafka along with a |
| <a href="../../../../../../org/apache/beam/sdk/coders/Coder.html" title="class in org.apache.beam.sdk.coders"><code>Coder</code></a> for helping the Beam runner materialize value objects at runtime if necessary. |
| |
| <p>Use this method only if your pipeline doesn't work with plain <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withValueDeserializer-java.lang.Class-"><code>withValueDeserializer(Class)</code></a>.</div> |
| </li> |
| </ul> |
| <a name="withConsumerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withConsumerFactoryFn</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withConsumerFactoryFn(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><java.util.Map<java.lang.String,java.lang.Object>,<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/Consumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer">Consumer</a><byte[],byte[]>> consumerFactoryFn)</pre> |
| <div class="block">A factory to create Kafka <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/Consumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer"><code>Consumer</code></a> from consumer configuration. This is useful for |
| supporting another version of Kafka consumer. Default is <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/KafkaConsumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer"><code>KafkaConsumer</code></a>.</div> |
| </li> |
| </ul> |
| <a name="updateConsumerProperties-java.util.Map-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>updateConsumerProperties</h4> |
| <pre>@Deprecated |
| public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> updateConsumerProperties(java.util.Map<java.lang.String,java.lang.Object> configUpdates)</pre> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> <span class="deprecationComment">as of version 2.13. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withConsumerConfigUpdates-java.util.Map-"><code>withConsumerConfigUpdates(Map)</code></a> instead</span></div> |
| <div class="block">Update consumer configuration with new properties.</div> |
| </li> |
| </ul> |
| <a name="withMaxNumRecords-long-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withMaxNumRecords</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withMaxNumRecords(long maxNumRecords)</pre> |
| <div class="block">Similar to <a href="../../../../../../org/apache/beam/sdk/io/Read.Unbounded.html#withMaxNumRecords-long-"><code>Read.Unbounded.withMaxNumRecords(long)</code></a>. Mainly used |
| for tests and demo applications.</div> |
| </li> |
| </ul> |
| <a name="withStartReadTime-org.joda.time.Instant-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withStartReadTime</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withStartReadTime(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a> startReadTime)</pre> |
| <div class="block">Use timestamp to set up start offset. It is only supported by Kafka Client 0.10.1.0 onwards |
| and the message format version after 0.10.0. |
| |
| <p>Note that this take priority over start offset configuration <code>ConsumerConfig.AUTO_OFFSET_RESET_CONFIG</code> and any auto committed offsets. |
| |
| <p>This results in hard failures in either of the following two cases : 1. If one of more |
| partitions do not contain any messages with timestamp larger than or equal to desired |
| timestamp. 2. If the message format version in a partition is before 0.10.0, i.e. the |
| messages do not have timestamps.</div> |
| </li> |
| </ul> |
| <a name="withMaxReadTime-org.joda.time.Duration-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withMaxReadTime</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withMaxReadTime(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> maxReadTime)</pre> |
| <div class="block">Similar to <a href="../../../../../../org/apache/beam/sdk/io/Read.Unbounded.html#withMaxReadTime-org.joda.time.Duration-"><code>Read.Unbounded.withMaxReadTime(Duration)</code></a>. Mainly |
| used for tests and demo applications.</div> |
| </li> |
| </ul> |
| <a name="withLogAppendTime--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withLogAppendTime</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withLogAppendTime()</pre> |
| <div class="block">Sets <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicy</code></a> to <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.LogAppendTimePolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicyFactory.LogAppendTimePolicy</code></a>. The |
| policy assigns Kafka's log append time (server side ingestion time) to each record. The |
| watermark for each Kafka partition is the timestamp of the last record read. If a partition |
| is idle, the watermark advances to couple of seconds behind wall time. Every record consumed |
| from Kafka is expected to have its timestamp type set to 'LOG_APPEND_TIME'. |
| |
| <p>In Kafka, log append time needs to be enabled for each topic, and all the subsequent |
| records wil have their timestamp set to log append time. If a record does not have its |
| timestamp type set to 'LOG_APPEND_TIME' for any reason, it's timestamp is set to previous |
| record timestamp or latest watermark, whichever is larger. |
| |
| <p>The watermark for the entire source is the oldest of each partition's watermark. If one of |
| the readers falls behind possibly due to uneven distribution of records among Kafka |
| partitions, it ends up holding the watermark for the entire source.</div> |
| </li> |
| </ul> |
| <a name="withProcessingTime--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withProcessingTime</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withProcessingTime()</pre> |
| <div class="block">Sets <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicy</code></a> to <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.ProcessingTimePolicy.html" title="class in org.apache.beam.sdk.io.kafka"><code>TimestampPolicyFactory.ProcessingTimePolicy</code></a>. This is |
| the default timestamp policy. It assigns processing time to each record. Specifically, this |
| is the timestamp when the record becomes 'current' in the reader. The watermark aways |
| advances to current time. If server side time (log append time) is enabled in Kafka, <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withLogAppendTime--"><code>withLogAppendTime()</code></a> is recommended over this.</div> |
| </li> |
| </ul> |
| <a name="withCreateTime-org.joda.time.Duration-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withCreateTime</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withCreateTime(<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Duration.html?is-external=true" title="class or interface in org.joda.time">Duration</a> maxDelay)</pre> |
| <div class="block">Sets the timestamps policy based on <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaTimestampType.html#CREATE_TIME"><code>KafkaTimestampType.CREATE_TIME</code></a> timestamp of the |
| records. It is an error if a record's timestamp type is not <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaTimestampType.html#CREATE_TIME"><code>KafkaTimestampType.CREATE_TIME</code></a>. The timestamps within a partition are expected to be roughly |
| monotonically increasing with a cap on out of order delays (e.g. 'max delay' of 1 minute). |
| The watermark at any time is '(<code>Min(now(), Max(event timestamp so far)) - max delay</code>)'. |
| However, watermark is never set in future and capped to 'now - max delay'. In addition, |
| watermark advanced to 'now - max delay' when a partition is idle.</div> |
| <dl> |
| <dt><span class="paramLabel">Parameters:</span></dt> |
| <dd><code>maxDelay</code> - For any record in the Kafka partition, the timestamp of any subsequent record |
| is expected to be after <code>current record timestamp - maxDelay</code>.</dd> |
| </dl> |
| </li> |
| </ul> |
| <a name="withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withTimestampPolicyFactory</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withTimestampPolicyFactory(<a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.html" title="interface in org.apache.beam.sdk.io.kafka">TimestampPolicyFactory</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> timestampPolicyFactory)</pre> |
| <div class="block">Provide custom <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.html" title="interface in org.apache.beam.sdk.io.kafka"><code>TimestampPolicyFactory</code></a> to set event times and watermark for each |
| partition. <a href="../../../../../../org/apache/beam/sdk/io/kafka/TimestampPolicyFactory.html#createTimestampPolicy-org.apache.kafka.common.TopicPartition-java.util.Optional-"><code>TimestampPolicyFactory.createTimestampPolicy(TopicPartition, Optional)</code></a> is |
| invoked for each partition when the reader starts.</div> |
| <dl> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withLogAppendTime--"><code>withLogAppendTime()</code></a>, |
| <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withCreateTime-org.joda.time.Duration-"><code>withCreateTime(Duration)</code></a>, |
| <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withProcessingTime--"><code>withProcessingTime()</code></a></dd> |
| </dl> |
| </li> |
| </ul> |
| <a name="withTimestampFn2-org.apache.beam.sdk.transforms.SerializableFunction-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withTimestampFn2</h4> |
| <pre>@Deprecated |
| public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withTimestampFn2(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> timestampFn)</pre> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> <span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| <div class="block">A function to assign a timestamp to a record. Default is processing timestamp.</div> |
| </li> |
| </ul> |
| <a name="withWatermarkFn2-org.apache.beam.sdk.transforms.SerializableFunction-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withWatermarkFn2</h4> |
| <pre>@Deprecated |
| public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withWatermarkFn2(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> watermarkFn)</pre> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> <span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| <div class="block">A function to calculate watermark after a record. Default is last record timestamp.</div> |
| <dl> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampFn-org.apache.beam.sdk.transforms.SerializableFunction-"><code>withTimestampFn(SerializableFunction)</code></a></dd> |
| </dl> |
| </li> |
| </ul> |
| <a name="withTimestampFn-org.apache.beam.sdk.transforms.SerializableFunction-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withTimestampFn</h4> |
| <pre>@Deprecated |
| public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withTimestampFn(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values">KV</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> timestampFn)</pre> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> <span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| <div class="block">A function to assign a timestamp to a record. Default is processing timestamp.</div> |
| </li> |
| </ul> |
| <a name="withWatermarkFn-org.apache.beam.sdk.transforms.SerializableFunction-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withWatermarkFn</h4> |
| <pre>@Deprecated |
| public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withWatermarkFn(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a><<a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values">KV</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>,<a href="https://static.javadoc.io/joda-time/joda-time/2.10.3/org/joda/time/Instant.html?is-external=true" title="class or interface in org.joda.time">Instant</a>> watermarkFn)</pre> |
| <div class="block"><span class="deprecatedLabel">Deprecated.</span> <span class="deprecationComment">as of version 2.4. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampPolicyFactory-org.apache.beam.sdk.io.kafka.TimestampPolicyFactory-"><code>withTimestampPolicyFactory(TimestampPolicyFactory)</code></a> instead.</span></div> |
| <div class="block">A function to calculate watermark after a record. Default is last record timestamp.</div> |
| <dl> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withTimestampFn-org.apache.beam.sdk.transforms.SerializableFunction-"><code>withTimestampFn(SerializableFunction)</code></a></dd> |
| </dl> |
| </li> |
| </ul> |
| <a name="withReadCommitted--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withReadCommitted</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withReadCommitted()</pre> |
| <div class="block">Sets "isolation_level" to "read_committed" in Kafka consumer configuration. This is ensures |
| that the consumer does not read uncommitted messages. Kafka version 0.11 introduced |
| transactional writes. Applications requiring end-to-end exactly-once semantics should only |
| read committed messages. See JavaDoc for <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/KafkaConsumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer"><code>KafkaConsumer</code></a> for more description.</div> |
| </li> |
| </ul> |
| <a name="commitOffsetsInFinalize--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>commitOffsetsInFinalize</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> commitOffsetsInFinalize()</pre> |
| <div class="block">Finalized offsets are committed to Kafka. See <a href="../../../../../../org/apache/beam/sdk/io/UnboundedSource.CheckpointMark.html#finalizeCheckpoint--"><code>UnboundedSource.CheckpointMark.finalizeCheckpoint()</code></a>. It |
| helps with minimizing gaps or duplicate processing of records while restarting a pipeline |
| from scratch. But it does not provide hard processing guarantees. There could be a short |
| delay to commit after <a href="../../../../../../org/apache/beam/sdk/io/UnboundedSource.CheckpointMark.html#finalizeCheckpoint--"><code>UnboundedSource.CheckpointMark.finalizeCheckpoint()</code></a> is invoked, as reader might |
| be blocked on reading from Kafka. Note that it is independent of 'AUTO_COMMIT' Kafka consumer |
| configuration. Usually either this or AUTO_COMMIT in Kafka consumer is enabled, but not both.</div> |
| </li> |
| </ul> |
| <a name="withOffsetConsumerConfigOverrides-java.util.Map-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withOffsetConsumerConfigOverrides</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withOffsetConsumerConfigOverrides(java.util.Map<java.lang.String,java.lang.Object> offsetConsumerConfig)</pre> |
| <div class="block">Set additional configuration for the backend offset consumer. It may be required for a |
| secured Kafka cluster, especially when you see similar WARN log message 'exception while |
| fetching latest offset for partition {}. will be retried'. |
| |
| <p>In <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html#read--"><code>KafkaIO.read()</code></a>, there're two consumers running in the backend actually:<br> |
| 1. the main consumer, which reads data from kafka;<br> |
| 2. the secondary offset consumer, which is used to estimate backlog, by fetching latest |
| offset;<br> |
| |
| <p>By default, offset consumer inherits the configuration from main consumer, with an |
| auto-generated <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/1.0.0/org/apache/kafka/clients/consumer/ConsumerConfig.html?is-external=true#GROUP_ID_CONFIG" title="class or interface in org.apache.kafka.clients.consumer"><code>ConsumerConfig.GROUP_ID_CONFIG</code></a>. This may not work in a secured Kafka |
| which requires more configurations.</div> |
| </li> |
| </ul> |
| <a name="withConsumerConfigUpdates-java.util.Map-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withConsumerConfigUpdates</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.Read</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>> withConsumerConfigUpdates(java.util.Map<java.lang.String,java.lang.Object> configUpdates)</pre> |
| <div class="block">Update configuration for the backend main consumer. Note that the default consumer properties |
| will not be completely overridden. This method only updates the value which has the same key. |
| |
| <p>In <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html#read--"><code>KafkaIO.read()</code></a>, there're two consumers running in the backend actually:<br> |
| 1. the main consumer, which reads data from kafka;<br> |
| 2. the secondary offset consumer, which is used to estimate backlog, by fetching latest |
| offset;<br> |
| |
| <p>By default, main consumer uses the configuration from <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#DEFAULT_CONSUMER_PROPERTIES"><code>DEFAULT_CONSUMER_PROPERTIES</code></a>.</div> |
| </li> |
| </ul> |
| <a name="withoutMetadata--"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>withoutMetadata</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values">KV</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>>> withoutMetadata()</pre> |
| <div class="block">Returns a <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms"><code>PTransform</code></a> for PCollection of <a href="../../../../../../org/apache/beam/sdk/values/KV.html" title="class in org.apache.beam.sdk.values"><code>KV</code></a>, dropping Kafka metatdata.</div> |
| </li> |
| </ul> |
| <a name="expand-org.apache.beam.sdk.values.PBegin-"> |
| <!-- --> |
| </a> |
| <ul class="blockList"> |
| <li class="blockList"> |
| <h4>expand</h4> |
| <pre>public <a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>> expand(<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a> input)</pre> |
| <div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#expand-InputT-">PTransform</a></code></span></div> |
| <div class="block">Override this method to specify how this <code>PTransform</code> should be expanded on the given |
| <code>InputT</code>. |
| |
| <p>NOTE: This method should not be called directly. Instead apply the <code>PTransform</code> should |
| be applied to the <code>InputT</code> using the <code>apply</code> method. |
| |
| <p>Composite transforms, which are defined in terms of other transforms, should return the |
| output of one of the composed transforms. Non-composite transforms, which do not apply any |
| transforms internally, should return a new unbound output and register evaluators (via |
| backend-specific registration methods).</div> |
| <dl> |
| <dt><span class="overrideSpecifyLabel">Specified by:</span></dt> |
| <dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#expand-InputT-">expand</a></code> in class <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>>></code></dd> |
| </dl> |
| </li> |
| </ul> |
| <a name="populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-"> |
| <!-- --> |
| </a> |
| <ul class="blockListLast"> |
| <li class="blockList"> |
| <h4>populateDisplayData</h4> |
| <pre>public void populateDisplayData(<a href="../../../../../../org/apache/beam/sdk/transforms/display/DisplayData.Builder.html" title="interface in org.apache.beam.sdk.transforms.display">DisplayData.Builder</a> builder)</pre> |
| <div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">PTransform</a></code></span></div> |
| <div class="block">Register display data for the given transform or component. |
| |
| <p><code>populateDisplayData(DisplayData.Builder)</code> is invoked by Pipeline runners to collect |
| display data via <a href="../../../../../../org/apache/beam/sdk/transforms/display/DisplayData.html#from-org.apache.beam.sdk.transforms.display.HasDisplayData-"><code>DisplayData.from(HasDisplayData)</code></a>. Implementations may call <code>super.populateDisplayData(builder)</code> in order to register display data in the current namespace, |
| but should otherwise use <code>subcomponent.populateDisplayData(builder)</code> to use the namespace |
| of the subcomponent. |
| |
| <p>By default, does not register any display data. Implementors may override this method to |
| provide their own display data.</div> |
| <dl> |
| <dt><span class="overrideSpecifyLabel">Specified by:</span></dt> |
| <dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></code> in interface <code><a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></code></dd> |
| <dt><span class="overrideSpecifyLabel">Overrides:</span></dt> |
| <dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></code> in class <code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a><<a href="../../../../../../org/apache/beam/sdk/values/PBegin.html" title="class in org.apache.beam.sdk.values">PBegin</a>,<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaRecord.html" title="class in org.apache.beam.sdk.io.kafka">KafkaRecord</a><<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" title="type parameter in KafkaIO.Read">V</a>>>></code></dd> |
| <dt><span class="paramLabel">Parameters:</span></dt> |
| <dd><code>builder</code> - The builder to populate with display data.</dd> |
| <dt><span class="seeLabel">See Also:</span></dt> |
| <dd><a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display"><code>HasDisplayData</code></a></dd> |
| </dl> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <!-- ========= END OF CLASS DATA ========= --> |
| <!-- ======= START OF BOTTOM NAVBAR ====== --> |
| <div class="bottomNav"><a name="navbar.bottom"> |
| <!-- --> |
| </a> |
| <div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div> |
| <a name="navbar.bottom.firstrow"> |
| <!-- --> |
| </a> |
| <ul class="navList" title="Navigation"> |
| <li><a href="../../../../../../overview-summary.html">Overview</a></li> |
| <li><a href="package-summary.html">Package</a></li> |
| <li class="navBarCell1Rev">Class</li> |
| <li><a href="package-tree.html">Tree</a></li> |
| <li><a href="../../../../../../deprecated-list.html">Deprecated</a></li> |
| <li><a href="../../../../../../index-all.html">Index</a></li> |
| <li><a href="../../../../../../help-doc.html">Help</a></li> |
| </ul> |
| </div> |
| <div class="subNav"> |
| <ul class="navList"> |
| <li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html" title="class in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Prev Class</span></a></li> |
| <li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.External.html" title="class in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Next Class</span></a></li> |
| </ul> |
| <ul class="navList"> |
| <li><a href="../../../../../../index.html?org/apache/beam/sdk/io/kafka/KafkaIO.Read.html" target="_top">Frames</a></li> |
| <li><a href="KafkaIO.Read.html" target="_top">No Frames</a></li> |
| </ul> |
| <ul class="navList" id="allclasses_navbar_bottom"> |
| <li><a href="../../../../../../allclasses-noframe.html">All Classes</a></li> |
| </ul> |
| <div> |
| <script type="text/javascript"><!-- |
| allClassesLink = document.getElementById("allclasses_navbar_bottom"); |
| if(window==top) { |
| allClassesLink.style.display = "block"; |
| } |
| else { |
| allClassesLink.style.display = "none"; |
| } |
| //--> |
| </script> |
| </div> |
| <div> |
| <ul class="subNavList"> |
| <li>Summary: </li> |
| <li><a href="#nested.class.summary">Nested</a> | </li> |
| <li><a href="#fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform">Field</a> | </li> |
| <li><a href="#constructor.summary">Constr</a> | </li> |
| <li><a href="#method.summary">Method</a></li> |
| </ul> |
| <ul class="subNavList"> |
| <li>Detail: </li> |
| <li>Field | </li> |
| <li><a href="#constructor.detail">Constr</a> | </li> |
| <li><a href="#method.detail">Method</a></li> |
| </ul> |
| </div> |
| <a name="skip.navbar.bottom"> |
| <!-- --> |
| </a></div> |
| <!-- ======== END OF BOTTOM NAVBAR ======= --> |
| </body> |
| </html> |
