blob: ef983621cad6ce3d803cfd2dbac574bb68ce8539 [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc -->
<title>KafkaIO.WriteRecords (Apache Beam 2.38.0-SNAPSHOT)</title>
<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="KafkaIO.WriteRecords (Apache Beam 2.38.0-SNAPSHOT)";
}
}
catch(err) {
}
//-->
var methods = {"i0":10,"i1":10,"i2":42,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":42,"i12":10,"i13":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"],32:["t6","Deprecated Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Write.External.Configuration.html" title="class in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaPublishTimestampFunction.html" title="interface in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../../index.html?org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" target="_top">Frames</a></li>
<li><a href="KafkaIO.WriteRecords.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li><a href="#fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform">Field</a>&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">org.apache.beam.sdk.io.kafka</div>
<h2 title="Class KafkaIO.WriteRecords" class="title">Class KafkaIO.WriteRecords&lt;K,V&gt;</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">org.apache.beam.sdk.transforms.PTransform</a>&lt;<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;K,V&gt;&gt;,<a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a>&gt;</li>
<li>
<ul class="inheritance">
<li>org.apache.beam.sdk.io.kafka.KafkaIO.WriteRecords&lt;K,V&gt;</li>
</ul>
</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt>All Implemented Interfaces:</dt>
<dd>java.io.Serializable, <a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></dd>
</dl>
<dl>
<dt>Enclosing class:</dt>
<dd><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO</a></dd>
</dl>
<hr>
<br>
<pre>public abstract static class <span class="typeNameLabel">KafkaIO.WriteRecords&lt;K,V&gt;</span>
extends <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a>&lt;<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;K,V&gt;&gt;,<a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a>&gt;</pre>
<div class="block">A <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms"><code>PTransform</code></a> to write to a Kafka topic with ProducerRecord's. See <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.html" title="class in org.apache.beam.sdk.io.kafka"><code>KafkaIO</code></a> for
more information on usage and configuration.</div>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../../serialized-form.html#org.apache.beam.sdk.io.kafka.KafkaIO.WriteRecords">Serialized Form</a></dd>
</dl>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- =========== FIELD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="field.summary">
<!-- -->
</a>
<h3>Field Summary</h3>
<ul class="blockList">
<li class="blockList"><a name="fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform">
<!-- -->
</a>
<h3>Fields inherited from class&nbsp;org.apache.beam.sdk.transforms.<a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a></h3>
<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#name">name</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#resourceHints">resourceHints</a></code></li>
</ul>
</li>
</ul>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colOne" scope="col">Constructor and Description</th>
</tr>
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#WriteRecords--">WriteRecords</a></span>()</code>&nbsp;</td>
</tr>
</table>
</li>
</ul>
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t6" class="tableTab"><span><a href="javascript:show(32);">Deprecated Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#expand-org.apache.beam.sdk.values.PCollection-">expand</a></span>(<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;input)</code>
<div class="block">Override this method to specify how this <code>PTransform</code> should be expanded on the given
<code>InputT</code>.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/display/DisplayData.Builder.html" title="interface in org.apache.beam.sdk.transforms.display">DisplayData.Builder</a>&nbsp;builder)</code>
<div class="block">Register display data for the given transform or component.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#updateProducerProperties-java.util.Map-">updateProducerProperties</a></span>(java.util.Map&lt;java.lang.String,java.lang.Object&gt;&nbsp;configUpdates)</code>
<div class="block"><span class="deprecatedLabel">Deprecated.</span>&nbsp;
<div class="block"><span class="deprecationComment">as of version 2.13. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withProducerConfigUpdates-java.util.Map-"><code>withProducerConfigUpdates(Map)</code></a> instead.</span></div>
</div>
</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#validate-org.apache.beam.sdk.options.PipelineOptions-">validate</a></span>(<a href="../../../../../../org/apache/beam/sdk/options/PipelineOptions.html" title="interface in org.apache.beam.sdk.options">PipelineOptions</a>&nbsp;options)</code>
<div class="block">Called before running the Pipeline to verify this transform is fully and correctly specified.</div>
</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withBootstrapServers-java.lang.String-">withBootstrapServers</a></span>(java.lang.String&nbsp;bootstrapServers)</code>
<div class="block">Returns a new <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Write.html" title="class in org.apache.beam.sdk.io.kafka"><code>KafkaIO.Write</code></a> transform with Kafka producer pointing to <code>bootstrapServers</code>.</div>
</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withConsumerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-">withConsumerFactoryFn</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a>&lt;java.util.Map&lt;java.lang.String,java.lang.Object&gt;,? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/consumer/Consumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer">Consumer</a>&lt;?,?&gt;&gt;&nbsp;consumerFactoryFn)</code>
<div class="block">When exactly-once semantics are enabled (see <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withEOS-int-java.lang.String-"><code>withEOS(int, String)</code></a>), the sink needs
to fetch previously stored state with Kafka topic.</div>
</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withEOS-int-java.lang.String-">withEOS</a></span>(int&nbsp;numShards,
java.lang.String&nbsp;sinkGroupId)</code>
<div class="block">Provides exactly-once semantics while writing to Kafka, which enables applications with
end-to-end exactly-once guarantees on top of exactly-once semantics <i>within</i> Beam
pipelines.</div>
</td>
</tr>
<tr id="i7" class="rowColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withInputTimestamp--">withInputTimestamp</a></span>()</code>
<div class="block">The timestamp for each record being published is set to timestamp of the element in the
pipeline.</div>
</td>
</tr>
<tr id="i8" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withKeySerializer-java.lang.Class-">withKeySerializer</a></span>(java.lang.Class&lt;? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Serializer</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>&gt;&gt;&nbsp;keySerializer)</code>
<div class="block">Sets a <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Serializer</code></a> for serializing key (if any) to bytes.</div>
</td>
</tr>
<tr id="i9" class="rowColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withProducerConfigUpdates-java.util.Map-">withProducerConfigUpdates</a></span>(java.util.Map&lt;java.lang.String,java.lang.Object&gt;&nbsp;configUpdates)</code>
<div class="block">Update configuration for the producer.</div>
</td>
</tr>
<tr id="i10" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withProducerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-">withProducerFactoryFn</a></span>(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a>&lt;java.util.Map&lt;java.lang.String,java.lang.Object&gt;,<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/Producer.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">Producer</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;producerFactoryFn)</code>
<div class="block">Sets a custom function to create Kafka producer.</div>
</td>
</tr>
<tr id="i11" class="rowColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withPublishTimestampFunction-org.apache.beam.sdk.io.kafka.KafkaPublishTimestampFunction-">withPublishTimestampFunction</a></span>(<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaPublishTimestampFunction.html" title="interface in org.apache.beam.sdk.io.kafka">KafkaPublishTimestampFunction</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;timestampFunction)</code>
<div class="block"><span class="deprecatedLabel">Deprecated.</span>&nbsp;
<div class="block"><span class="deprecationComment">use <code>ProducerRecords</code> to set publish timestamp.</span></div>
</div>
</td>
</tr>
<tr id="i12" class="altColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withTopic-java.lang.String-">withTopic</a></span>(java.lang.String&nbsp;topic)</code>
<div class="block">Sets the default Kafka topic to write to.</div>
</td>
</tr>
<tr id="i13" class="rowColor">
<td class="colFirst"><code><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withValueSerializer-java.lang.Class-">withValueSerializer</a></span>(java.lang.Class&lt;? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Serializer</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;valueSerializer)</code>
<div class="block">Sets a <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Serializer</code></a> for serializing value to bytes.</div>
</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.org.apache.beam.sdk.transforms.PTransform">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;org.apache.beam.sdk.transforms.<a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a></h3>
<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#compose-org.apache.beam.sdk.transforms.SerializableFunction-">compose</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#compose-java.lang.String-org.apache.beam.sdk.transforms.SerializableFunction-">compose</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getAdditionalInputs--">getAdditionalInputs</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder--">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder-InputT-">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getDefaultOutputCoder-InputT-org.apache.beam.sdk.values.PCollection-">getDefaultOutputCoder</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getKindString--">getKindString</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getName--">getName</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#getResourceHints--">getResourceHints</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#setResourceHints-org.apache.beam.sdk.transforms.resourcehints.ResourceHints-">setResourceHints</a>, <a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#toString--">toString</a></code></li>
</ul>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a name="WriteRecords--">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>WriteRecords</h4>
<pre>public&nbsp;WriteRecords()</pre>
</li>
</ul>
</li>
</ul>
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="withBootstrapServers-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withBootstrapServers</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withBootstrapServers(java.lang.String&nbsp;bootstrapServers)</pre>
<div class="block">Returns a new <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Write.html" title="class in org.apache.beam.sdk.io.kafka"><code>KafkaIO.Write</code></a> transform with Kafka producer pointing to <code>bootstrapServers</code>.</div>
</li>
</ul>
<a name="withTopic-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withTopic</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withTopic(java.lang.String&nbsp;topic)</pre>
<div class="block">Sets the default Kafka topic to write to. Use <code>ProducerRecords</code> to set topic name per
published record.</div>
</li>
</ul>
<a name="withKeySerializer-java.lang.Class-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withKeySerializer</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withKeySerializer(java.lang.Class&lt;? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Serializer</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>&gt;&gt;&nbsp;keySerializer)</pre>
<div class="block">Sets a <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Serializer</code></a> for serializing key (if any) to bytes.
<p>A key is optional while writing to Kafka. Note when a key is set, its hash is used to
determine partition in Kafka (see <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer"><code>ProducerRecord</code></a> for more details).</div>
</li>
</ul>
<a name="withValueSerializer-java.lang.Class-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withValueSerializer</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withValueSerializer(java.lang.Class&lt;? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization">Serializer</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;valueSerializer)</pre>
<div class="block">Sets a <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/common/serialization/Serializer.html?is-external=true" title="class or interface in org.apache.kafka.common.serialization"><code>Serializer</code></a> for serializing value to bytes.</div>
</li>
</ul>
<a name="updateProducerProperties-java.util.Map-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>updateProducerProperties</h4>
<pre>@Deprecated
public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;updateProducerProperties(java.util.Map&lt;java.lang.String,java.lang.Object&gt;&nbsp;configUpdates)</pre>
<div class="block"><span class="deprecatedLabel">Deprecated.</span>&nbsp;<span class="deprecationComment">as of version 2.13. Use <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withProducerConfigUpdates-java.util.Map-"><code>withProducerConfigUpdates(Map)</code></a> instead.</span></div>
<div class="block">Adds the given producer properties, overriding old values of properties with the same key.</div>
</li>
</ul>
<a name="withProducerConfigUpdates-java.util.Map-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withProducerConfigUpdates</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withProducerConfigUpdates(java.util.Map&lt;java.lang.String,java.lang.Object&gt;&nbsp;configUpdates)</pre>
<div class="block">Update configuration for the producer. Note that the default producer properties will not be
completely overridden. This method only updates the value which has the same key.
<p>By default, the producer uses the configuration from <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#DEFAULT_PRODUCER_PROPERTIES"><code>DEFAULT_PRODUCER_PROPERTIES</code></a>.</div>
</li>
</ul>
<a name="withProducerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withProducerFactoryFn</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withProducerFactoryFn(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a>&lt;java.util.Map&lt;java.lang.String,java.lang.Object&gt;,<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/Producer.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">Producer</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;producerFactoryFn)</pre>
<div class="block">Sets a custom function to create Kafka producer. Primarily used for tests. Default is <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/KafkaProducer.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer"><code>KafkaProducer</code></a></div>
</li>
</ul>
<a name="withInputTimestamp--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withInputTimestamp</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withInputTimestamp()</pre>
<div class="block">The timestamp for each record being published is set to timestamp of the element in the
pipeline. This is equivalent to <code>withPublishTimestampFunction((e, ts) -&gt; ts)</code>. <br>
NOTE: Kafka's retention policies are based on message timestamps. If the pipeline is
processing messages from the past, they might be deleted immediately by Kafka after being
published if the timestamps are older than Kafka cluster's <code>log.retention.hours</code>.</div>
</li>
</ul>
<a name="withPublishTimestampFunction-org.apache.beam.sdk.io.kafka.KafkaPublishTimestampFunction-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withPublishTimestampFunction</h4>
<pre>@Deprecated
public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withPublishTimestampFunction(<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaPublishTimestampFunction.html" title="interface in org.apache.beam.sdk.io.kafka">KafkaPublishTimestampFunction</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;timestampFunction)</pre>
<div class="block"><span class="deprecatedLabel">Deprecated.</span>&nbsp;<span class="deprecationComment">use <code>ProducerRecords</code> to set publish timestamp.</span></div>
<div class="block">A function to provide timestamp for records being published. <br>
NOTE: Kafka's retention policies are based on message timestamps. If the pipeline is
processing messages from the past, they might be deleted immediately by Kafka after being
published if the timestamps are older than Kafka cluster's <code>log.retention.hours</code>.</div>
</li>
</ul>
<a name="withEOS-int-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withEOS</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withEOS(int&nbsp;numShards,
java.lang.String&nbsp;sinkGroupId)</pre>
<div class="block">Provides exactly-once semantics while writing to Kafka, which enables applications with
end-to-end exactly-once guarantees on top of exactly-once semantics <i>within</i> Beam
pipelines. It ensures that records written to sink are committed on Kafka exactly once, even
in the case of retries during pipeline execution even when some processing is retried.
Retries typically occur when workers restart (as in failure recovery), or when the work is
redistributed (as in an autoscaling event).
<p>Beam runners typically provide exactly-once semantics for results of a pipeline, but not
for side effects from user code in transform. If a transform such as Kafka sink writes to an
external system, those writes might occur more than once. When EOS is enabled here, the sink
transform ties checkpointing semantics in compatible Beam runners and transactions in Kafka
(version 0.11+) to ensure a record is written only once. As the implementation relies on
runners checkpoint semantics, not all the runners are compatible. The sink throws an
exception during initialization if the runner is not explicitly allowed. The Dataflow, Flink,
and Spark runners are compatible.
<p>Note on performance: Exactly-once sink involves two shuffles of the records. In addition
to cost of shuffling the records among workers, the records go through 2
serialization-deserialization cycles. Depending on volume and cost of serialization, the CPU
cost might be noticeable. The CPU cost can be reduced by writing byte arrays (i.e.
serializing them to byte before writing to Kafka sink).</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>numShards</code> - Sets sink parallelism. The state metadata stored on Kafka is stored across
this many virtual partitions using <code>sinkGroupId</code>. A good rule of thumb is to set
this to be around number of partitions in Kafka topic.</dd>
<dd><code>sinkGroupId</code> - The <i>group id</i> used to store small amount of state as metadata on
Kafka. It is similar to <i>consumer group id</i> used with a <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/consumer/KafkaConsumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer"><code>KafkaConsumer</code></a>. Each
job should use a unique group id so that restarts/updates of job preserve the state to
ensure exactly-once semantics. The state is committed atomically with sink transactions
on Kafka. See <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/KafkaProducer.html?is-external=true#sendOffsetsToTransaction-java.util.Map-java.lang.String-" title="class or interface in org.apache.kafka.clients.producer"><code>KafkaProducer.sendOffsetsToTransaction(Map, String)</code></a> for more
information. The sink performs multiple sanity checks during initialization to catch
common mistakes so that it does not end up using state that does not <i>seem</i> to be
written by the same job.</dd>
</dl>
</li>
</ul>
<a name="withConsumerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withConsumerFactoryFn</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="class in org.apache.beam.sdk.io.kafka">KafkaIO.WriteRecords</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&nbsp;withConsumerFactoryFn(<a href="../../../../../../org/apache/beam/sdk/transforms/SerializableFunction.html" title="interface in org.apache.beam.sdk.transforms">SerializableFunction</a>&lt;java.util.Map&lt;java.lang.String,java.lang.Object&gt;,? extends <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/consumer/Consumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer">Consumer</a>&lt;?,?&gt;&gt;&nbsp;consumerFactoryFn)</pre>
<div class="block">When exactly-once semantics are enabled (see <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html#withEOS-int-java.lang.String-"><code>withEOS(int, String)</code></a>), the sink needs
to fetch previously stored state with Kafka topic. Fetching the metadata requires a consumer.
Similar to <a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Read.html#withConsumerFactoryFn-org.apache.beam.sdk.transforms.SerializableFunction-"><code>KafkaIO.Read.withConsumerFactoryFn(SerializableFunction)</code></a>, a factory function can
be supplied if required in a specific case. The default is <a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/consumer/KafkaConsumer.html?is-external=true" title="class or interface in org.apache.kafka.clients.consumer"><code>KafkaConsumer</code></a>.</div>
</li>
</ul>
<a name="expand-org.apache.beam.sdk.values.PCollection-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>expand</h4>
<pre>public&nbsp;<a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a>&nbsp;expand(<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;&nbsp;input)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#expand-InputT-">PTransform</a></code></span></div>
<div class="block">Override this method to specify how this <code>PTransform</code> should be expanded on the given
<code>InputT</code>.
<p>NOTE: This method should not be called directly. Instead apply the <code>PTransform</code> should
be applied to the <code>InputT</code> using the <code>apply</code> method.
<p>Composite transforms, which are defined in terms of other transforms, should return the
output of one of the composed transforms. Non-composite transforms, which do not apply any
transforms internally, should return a new unbound output and register evaluators (via
backend-specific registration methods).</div>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#expand-InputT-">expand</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a>&lt;<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;,<a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a>&gt;</code></dd>
</dl>
</li>
</ul>
<a name="validate-org.apache.beam.sdk.options.PipelineOptions-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>validate</h4>
<pre>public&nbsp;void&nbsp;validate(<a href="../../../../../../org/apache/beam/sdk/options/PipelineOptions.html" title="interface in org.apache.beam.sdk.options">PipelineOptions</a>&nbsp;options)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#validate-org.apache.beam.sdk.options.PipelineOptions-">PTransform</a></code></span></div>
<div class="block">Called before running the Pipeline to verify this transform is fully and correctly specified.
<p>By default, does nothing.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#validate-org.apache.beam.sdk.options.PipelineOptions-">validate</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a>&lt;<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;,<a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a>&gt;</code></dd>
</dl>
</li>
</ul>
<a name="populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>populateDisplayData</h4>
<pre>public&nbsp;void&nbsp;populateDisplayData(<a href="../../../../../../org/apache/beam/sdk/transforms/display/DisplayData.Builder.html" title="interface in org.apache.beam.sdk.transforms.display">DisplayData.Builder</a>&nbsp;builder)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">PTransform</a></code></span></div>
<div class="block">Register display data for the given transform or component.
<p><code>populateDisplayData(DisplayData.Builder)</code> is invoked by Pipeline runners to collect
display data via <a href="../../../../../../org/apache/beam/sdk/transforms/display/DisplayData.html#from-org.apache.beam.sdk.transforms.display.HasDisplayData-"><code>DisplayData.from(HasDisplayData)</code></a>. Implementations may call <code>super.populateDisplayData(builder)</code> in order to register display data in the current namespace,
but should otherwise use <code>subcomponent.populateDisplayData(builder)</code> to use the namespace
of the subcomponent.
<p>By default, does not register any display data. Implementors may override this method to
provide their own display data.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></code></dd>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/beam/sdk/transforms/PTransform.html" title="class in org.apache.beam.sdk.transforms">PTransform</a>&lt;<a href="../../../../../../org/apache/beam/sdk/values/PCollection.html" title="class in org.apache.beam.sdk.values">PCollection</a>&lt;<a href="https://static.javadoc.io/org.apache.kafka/kafka-clients/2.4.1/org/apache/kafka/clients/producer/ProducerRecord.html?is-external=true" title="class or interface in org.apache.kafka.clients.producer">ProducerRecord</a>&lt;<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">K</a>,<a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" title="type parameter in KafkaIO.WriteRecords">V</a>&gt;&gt;,<a href="../../../../../../org/apache/beam/sdk/values/PDone.html" title="class in org.apache.beam.sdk.values">PDone</a>&gt;</code></dd>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>builder</code> - The builder to populate with display data.</dd>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display"><code>HasDisplayData</code></a></dd>
</dl>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaIO.Write.External.Configuration.html" title="class in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../../org/apache/beam/sdk/io/kafka/KafkaPublishTimestampFunction.html" title="interface in org.apache.beam.sdk.io.kafka"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../../index.html?org/apache/beam/sdk/io/kafka/KafkaIO.WriteRecords.html" target="_top">Frames</a></li>
<li><a href="KafkaIO.WriteRecords.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li><a href="#fields.inherited.from.class.org.apache.beam.sdk.transforms.PTransform">Field</a>&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>