yamldoc/2.55.1/index.html - beam-site - Git at Google


             <html>
               <head>
                 <title>Beam YAML Transform Index</title>
                 <style>
                 pre { line-height: 125%; }
 td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
 span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
 td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
 span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
 .codehilite .hll { background-color: #ffffcc }
 .codehilite { background: #f8f8f8; }
 .codehilite .c { color: #3D7B7B; font-style: italic } /* Comment */
 .codehilite .err { border: 1px solid #FF0000 } /* Error */
 .codehilite .k { color: #008000; font-weight: bold } /* Keyword */
 .codehilite .o { color: #666666 } /* Operator */
 .codehilite .ch { color: #3D7B7B; font-style: italic } /* Comment.Hashbang */
 .codehilite .cm { color: #3D7B7B; font-style: italic } /* Comment.Multiline */
 .codehilite .cp { color: #9C6500 } /* Comment.Preproc */
 .codehilite .cpf { color: #3D7B7B; font-style: italic } /* Comment.PreprocFile */
 .codehilite .c1 { color: #3D7B7B; font-style: italic } /* Comment.Single */
 .codehilite .cs { color: #3D7B7B; font-style: italic } /* Comment.Special */
 .codehilite .gd { color: #A00000 } /* Generic.Deleted */
 .codehilite .ge { font-style: italic } /* Generic.Emph */
 .codehilite .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
 .codehilite .gr { color: #E40000 } /* Generic.Error */
 .codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */
 .codehilite .gi { color: #008400 } /* Generic.Inserted */
 .codehilite .go { color: #717171 } /* Generic.Output */
 .codehilite .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
 .codehilite .gs { font-weight: bold } /* Generic.Strong */
 .codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
 .codehilite .gt { color: #0044DD } /* Generic.Traceback */
 .codehilite .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
 .codehilite .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
 .codehilite .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
 .codehilite .kp { color: #008000 } /* Keyword.Pseudo */
 .codehilite .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
 .codehilite .kt { color: #B00040 } /* Keyword.Type */
 .codehilite .m { color: #666666 } /* Literal.Number */
 .codehilite .s { color: #BA2121 } /* Literal.String */
 .codehilite .na { color: #687822 } /* Name.Attribute */
 .codehilite .nb { color: #008000 } /* Name.Builtin */
 .codehilite .nc { color: #0000FF; font-weight: bold } /* Name.Class */
 .codehilite .no { color: #880000 } /* Name.Constant */
 .codehilite .nd { color: #AA22FF } /* Name.Decorator */
 .codehilite .ni { color: #717171; font-weight: bold } /* Name.Entity */
 .codehilite .ne { color: #CB3F38; font-weight: bold } /* Name.Exception */
 .codehilite .nf { color: #0000FF } /* Name.Function */
 .codehilite .nl { color: #767600 } /* Name.Label */
 .codehilite .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
 .codehilite .nt { color: #008000; font-weight: bold } /* Name.Tag */
 .codehilite .nv { color: #19177C } /* Name.Variable */
 .codehilite .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
 .codehilite .w { color: #bbbbbb } /* Text.Whitespace */
 .codehilite .mb { color: #666666 } /* Literal.Number.Bin */
 .codehilite .mf { color: #666666 } /* Literal.Number.Float */
 .codehilite .mh { color: #666666 } /* Literal.Number.Hex */
 .codehilite .mi { color: #666666 } /* Literal.Number.Integer */
 .codehilite .mo { color: #666666 } /* Literal.Number.Oct */
 .codehilite .sa { color: #BA2121 } /* Literal.String.Affix */
 .codehilite .sb { color: #BA2121 } /* Literal.String.Backtick */
 .codehilite .sc { color: #BA2121 } /* Literal.String.Char */
 .codehilite .dl { color: #BA2121 } /* Literal.String.Delimiter */
 .codehilite .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
 .codehilite .s2 { color: #BA2121 } /* Literal.String.Double */
 .codehilite .se { color: #AA5D1F; font-weight: bold } /* Literal.String.Escape */
 .codehilite .sh { color: #BA2121 } /* Literal.String.Heredoc */
 .codehilite .si { color: #A45A77; font-weight: bold } /* Literal.String.Interpol */
 .codehilite .sx { color: #008000 } /* Literal.String.Other */
 .codehilite .sr { color: #A45A77 } /* Literal.String.Regex */
 .codehilite .s1 { color: #BA2121 } /* Literal.String.Single */
 .codehilite .ss { color: #19177C } /* Literal.String.Symbol */
 .codehilite .bp { color: #008000 } /* Name.Builtin.Pseudo */
 .codehilite .fm { color: #0000FF } /* Name.Function.Magic */
 .codehilite .vc { color: #19177C } /* Name.Variable.Class */
 .codehilite .vg { color: #19177C } /* Name.Variable.Global */
 .codehilite .vi { color: #19177C } /* Name.Variable.Instance */
 .codehilite .vm { color: #19177C } /* Name.Variable.Magic */
 .codehilite .il { color: #666666 } /* Literal.Number.Integer.Long */

           .nav {
             height: 100%;
             width: 12em;
             position: fixed;
             top: 0;
             left: 0;
             overflow-x: hidden;
           }
           .nav a {
             color: #333;
             padding: .2em;
             display: block;
             text-decoration: none;
           }
           .nav a:hover {
             color: #888;
           }
           .nav li {
             list-style-type: none;
             margin: 0;
             padding: 0;
           }
           .content {
             margin-left: 12em;
           }
           h2 {
             margin-top: 2em;
           }

                 </style>
               </head>
               <body>
                 <div class="nav">
                   <div class="toc">
 <ul>
 <li><a href="#assigntimestamps">AssignTimestamps</a></li>
 <li><a href="#combine">Combine</a></li>
 <li><a href="#create">Create</a></li>
 <li><a href="#explode">Explode</a></li>
 <li><a href="#filter">Filter</a></li>
 <li><a href="#flatten">Flatten</a></li>
 <li><a href="#logfortesting">LogForTesting</a></li>
 <li><a href="#maptofields">MapToFields</a></li>
 <li><a href="#pytransform">PyTransform</a></li>
 <li><a href="#sql">Sql</a></li>
 <li><a href="#windowinto">WindowInto</a></li>
 <li><a href="#readfromavro">ReadFromAvro</a></li>
 <li><a href="#writetoavro">WriteToAvro</a></li>
 <li><a href="#readfrombigquery">ReadFromBigQuery</a></li>
 <li><a href="#writetobigquery">WriteToBigQuery</a></li>
 <li><a href="#readfromcsv">ReadFromCsv</a></li>
 <li><a href="#writetocsv">WriteToCsv</a></li>
 <li><a href="#readfromjdbc">ReadFromJdbc</a></li>
 <li><a href="#writetojdbc">WriteToJdbc</a></li>
 <li><a href="#readfromjson">ReadFromJson</a></li>
 <li><a href="#writetojson">WriteToJson</a></li>
 <li><a href="#readfromkafka">ReadFromKafka</a></li>
 <li><a href="#writetokafka">WriteToKafka</a></li>
 <li><a href="#readfrommysql">ReadFromMySql</a></li>
 <li><a href="#writetomysql">WriteToMySql</a></li>
 <li><a href="#readfromoracle">ReadFromOracle</a></li>
 <li><a href="#writetooracle">WriteToOracle</a></li>
 <li><a href="#readfromparquet">ReadFromParquet</a></li>
 <li><a href="#writetoparquet">WriteToParquet</a></li>
 <li><a href="#readfrompostgres">ReadFromPostgres</a></li>
 <li><a href="#writetopostgres">WriteToPostgres</a></li>
 <li><a href="#readfrompubsub">ReadFromPubSub</a></li>
 <li><a href="#writetopubsub">WriteToPubSub</a></li>
 <li><a href="#readfrompubsublite">ReadFromPubSubLite</a></li>
 <li><a href="#writetopubsublite">WriteToPubSubLite</a></li>
 <li><a href="#readfromsqlserver">ReadFromSqlServer</a></li>
 <li><a href="#writetosqlserver">WriteToSqlServer</a></li>
 <li><a href="#readfromtext">ReadFromText</a></li>
 <li><a href="#writetotext">WriteToText</a></li>
 </ul>
 </div>

                 </div>
                 <div class="content">
                   <h1>Beam YAML Transform Index</h1>
                   <h2 id="assigntimestamps">AssignTimestamps</h2>
 <p>Assigns a new timestamp each element of its input.</p>
 <p>This can be useful when reading records that have the timestamp embedded
 in them, for example with various file types or other sources that by default
 set all timestamps to the infinite past.</p>
 <p>Note that the timestamp should only be set forward, as setting it backwards
 may not cause it to hold back an already advanced watermark and the data
 could become droppably late.</p>
 <p>Supported languages: generic, javascript, python</p>
 <h3 id="configuration">Configuration</h3>
 <ul>
 <li>
 <p><strong>timestamp</strong> <code>?</code> (Optional)  : A field, callable, or expression giving the new timestamp.</p>
 </li>
 <li>
 <p><strong>language</strong> <code>string</code> (Optional)  : The language of the timestamp expression.</p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code>  : Whether and how to handle errors during iteration.    </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 </ul>
 <h3 id="usage">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">AssignTimestamps</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">timestamp</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">timestamp</span>
 <span class="w">  </span><span class="nt">language</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;language&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 </code></pre></div>

 <h2 id="combine">Combine</h2>
 <p>Groups and combines records sharing common fields.</p>
 <p>Built-in combine functions are <code>sum</code>, <code>max</code>, <code>min</code>, <code>all</code>, <code>any</code>, <code>mean</code>, <code>count</code>, <code>group</code>, <code>concat</code>
 but custom aggregation functions can be used as well.</p>
 <p>See also the documentation on
 <a href="https://beam.apache.org/documentation/sdks/yaml-combine/">YAML Aggregation</a>.</p>
 <p>Supported languages: calcite, generic, javascript, python, sql</p>
 <h3 id="configuration_1">Configuration</h3>
 <ul>
 <li>
 <p><strong>group_by</strong> <code>Array[string]</code>  </p>
 </li>
 <li>
 <p><strong>combine</strong> <code>Map[string, Map[string, ?]]</code>  </p>
 </li>
 <li>
 <p><strong>language</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_1">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Combine</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">group_by</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;group_by&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">combine</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">a</span><span class="p">:</span>
 <span class="w">      </span><span class="nt">a</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">combine_value_a_value_a</span>
 <span class="w">      </span><span class="nt">b</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">combine_value_a_value_b</span>
 <span class="w">      </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">    </span><span class="nt">b</span><span class="p">:</span>
 <span class="w">      </span><span class="nt">a</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">combine_value_b_value_a</span>
 <span class="w">      </span><span class="nt">b</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">combine_value_b_value_b</span>
 <span class="w">      </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">    </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">language</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;language&quot;</span>
 </code></pre></div>

 <h2 id="create">Create</h2>
 <p>Creates a collection containing a specified set of elements.</p>
 <p>This transform always produces schema'd data. For example</p>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Create</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">elements</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">1</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">2</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">3</span><span class="p p-Indicator">]</span>
 </code></pre></div>

 <p>will result in an output with three elements with a schema of
 Row(element=int) whereas YAML/JSON-style mappings will be interpreted
 directly as Beam rows, e.g.</p>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Create</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">elements</span><span class="p">:</span>
 <span class="w">     </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="p p-Indicator">{</span><span class="nt">first</span><span class="p">:</span><span class="w"> </span><span class="nv">0</span><span class="p p-Indicator">,</span><span class="nt"> second</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">{</span><span class="nt">str</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;foo&quot;</span><span class="p p-Indicator">,</span><span class="nt"> values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">1</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">2</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">3</span><span class="p p-Indicator">]}}</span>
 <span class="w">     </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="p p-Indicator">{</span><span class="nt">first</span><span class="p">:</span><span class="w"> </span><span class="nv">1</span><span class="p p-Indicator">,</span><span class="nt"> second</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">{</span><span class="nt">str</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;bar&quot;</span><span class="p p-Indicator">,</span><span class="nt"> values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">4</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">5</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">6</span><span class="p p-Indicator">]}}</span>
 </code></pre></div>

 <p>will result in a schema of the form (int, Row(string, List[int])).</p>
 <p>This can also be expressed as YAML</p>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Create</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">elements</span><span class="p">:</span>
 <span class="w">    </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">first</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">0</span>
 <span class="w">      </span><span class="nt">second</span><span class="p">:</span>
 <span class="w">        </span><span class="nt">str</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;foo&quot;</span>
 <span class="w">         </span><span class="nt">values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">1</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">2</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">3</span><span class="p p-Indicator">]</span>
 <span class="w">    </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">first</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
 <span class="w">      </span><span class="nt">second</span><span class="p">:</span>
 <span class="w">        </span><span class="nt">str</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;bar&quot;</span>
 <span class="w">         </span><span class="nt">values</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">4</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">5</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="nv">6</span><span class="p p-Indicator">]</span>
 </code></pre></div>

 <h3 id="configuration_2">Configuration</h3>
 <ul>
 <li>
 <p><strong>elements</strong> <code>Array[?]</code>  : The set of elements that should belong to the PCollection.
     YAML/JSON-style mappings will be interpreted as Beam rows.
     Primitives will be mapped to rows with a single "element" field.</p>
 </li>
 <li>
 <p><strong>reshuffle</strong> <code>boolean</code> (Optional)  : (optional) Whether to introduce a reshuffle (to possibly
     redistribute the work) if there is more than one element in the
     collection. Defaults to True.</p>
 </li>
 </ul>
 <h3 id="usage_2">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Create</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">elements</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">elements</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">reshuffle</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true|false</span>
 </code></pre></div>

 <h2 id="explode">Explode</h2>
 <p>Explodes (aka unnest/flatten) one or more fields producing multiple rows.</p>
 <p>Given one or more fields of iterable type, produces multiple rows, one for
 each value of that field. For example, a row of the form <code>('a', [1, 2, 3])</code>
 would expand to <code>('a', 1)</code>, <code>('a', 2')</code>, and <code>('a', 3)</code> when exploded on
 the second field.</p>
 <p>This is akin to a <code>FlatMap</code> when paired with the MapToFields transform.</p>
 <p>See more complete documentation on
 <a href="https://beam.apache.org/documentation/sdks/yaml-udf/#flatmap">YAML Mapping Functions</a>.</p>
 <h3 id="configuration_3">Configuration</h3>
 <ul>
 <li>
 <p><strong>fields</strong> <code>?</code> (Optional)  : The list of fields to expand.</p>
 </li>
 <li>
 <p><strong>cross_product</strong> <code>boolean</code> (Optional)  : If multiple fields are specified, indicates whether the
     full cross-product of combinations should be produced, or if the
     first element of the first field corresponds to the first element
     of the second field, etc. For example, the row
     <code>(['a', 'b'], [1, 2])</code> would expand to the four rows
     <code>('a', 1)</code>, <code>('a', 2)</code>, <code>('b', 1)</code>, and <code>('b', 2)</code> when
     <code>cross_product</code> is set to <code>true</code> but only the two rows
     <code>('a', 1)</code> and <code>('b', 2)</code> when it is set to <code>false</code>.
     Only meaningful (and required) if multiple rows are specified.</p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Map[string, ?]</code> (Optional)  : Whether and how to handle errors during iteration.</p>
 </li>
 </ul>
 <h3 id="usage_3">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Explode</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">fields</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">fields</span>
 <span class="w">  </span><span class="nt">cross_product</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true|false</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">a</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">error_handling_value_a</span>
 <span class="w">    </span><span class="nt">b</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">error_handling_value_b</span>
 <span class="w">    </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="filter">Filter</h2>
 <p>Keeps only records that satisfy the given criteria.</p>
 <p>See more complete documentation on
 <a href="https://beam.apache.org/documentation/sdks/yaml-udf/#filtering">YAML Filtering</a>.</p>
 <p>Supported languages: calcite, java, javascript, python, sql</p>
 <h3 id="configuration_4">Configuration</h3>
 <ul>
 <li>
 <p><strong>language</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>keep</strong> <code>Row</code>      </p>
 <p>Row fields:</p>
 <ul>
 <li>
 <p><strong>callable</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>expression</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>path</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 </ul>
 <h3 id="usage_4">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Filter</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">language</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;language&quot;</span>
 <span class="w">  </span><span class="nt">keep</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">callable</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;callable&quot;</span>
 <span class="w">    </span><span class="nt">expression</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;expression&quot;</span>
 <span class="w">    </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;name&quot;</span>
 <span class="w">    </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 </code></pre></div>

 <h2 id="flatten">Flatten</h2>
 <p>Flattens multiple PCollections into a single PCollection.</p>
 <p>The elements of the resulting PCollection will be the (disjoint) union of
 all the elements of all the inputs.</p>
 <p>Note that in YAML transforms can always take a list of inputs which will
 be implicitly flattened.</p>
 <h3 id="configuration_5">Configuration</h3>
 <p>No configuration parameters.</p>
 <h3 id="usage_5">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Flatten</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="logfortesting">LogForTesting</h2>
 <p>Logs each element of its input PCollection.</p>
 <p>The output of this transform is a copy of its input for ease of use in
 chain-style pipelines.</p>
 <h3 id="configuration_6">Configuration</h3>
 <ul>
 <li>
 <p><strong>level</strong> <code>string</code> (Optional)  : one of ERROR, INFO, or DEBUG, mapped to a corresponding
     language-specific logging level</p>
 </li>
 <li>
 <p><strong>prefix</strong> <code>string</code> (Optional)  : an optional identifier that will get prepended to the element
     being logged</p>
 </li>
 </ul>
 <h3 id="usage_6">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">LogForTesting</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">level</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;level&quot;</span>
 <span class="w">  </span><span class="nt">prefix</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;prefix&quot;</span>
 </code></pre></div>

 <h2 id="maptofields">MapToFields</h2>
 <p>Creates records with new fields defined in terms of the input fields.</p>
 <p>See more complete documentation on
 <a href="https://beam.apache.org/documentation/sdks/yaml-udf/#mapping-functions">YAML Mapping Functions</a>.</p>
 <p>Supported languages: calcite, generic, java, javascript, python, sql</p>
 <h3 id="configuration_7">Configuration</h3>
 <ul>
 <li>
 <p><strong>language</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>append</strong> <code>boolean</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>drop</strong> <code>Array[string]</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>fields</strong> <code>Map[string, Row]</code>  </p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 </ul>
 <h3 id="usage_7">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">MapToFields</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">language</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;language&quot;</span>
 <span class="w">  </span><span class="nt">append</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true|false</span>
 <span class="w">  </span><span class="nt">drop</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;drop&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">fields</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">a</span><span class="p">:</span>
 <span class="w">      </span><span class="nt">callable</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;callable&quot;</span>
 <span class="w">      </span><span class="nt">expression</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;expression&quot;</span>
 <span class="w">      </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;name&quot;</span>
 <span class="w">      </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 <span class="w">    </span><span class="nt">b</span><span class="p">:</span>
 <span class="w">      </span><span class="nt">callable</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;callable&quot;</span>
 <span class="w">      </span><span class="nt">expression</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;expression&quot;</span>
 <span class="w">      </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;name&quot;</span>
 <span class="w">      </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 <span class="w">    </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 </code></pre></div>

 <h2 id="pytransform">PyTransform</h2>
 <p>A Python PTransform identified by fully qualified name.</p>
 <p>This allows one to import, construct, and apply any Beam Python transform.
 This can be useful for using transforms that have not yet been exposed
 via a YAML interface. Note, however, that conversion may be required if this
 transform does not accept or produce Beam Rows.</p>
 <p>For example</p>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">PyTransform</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">   </span><span class="nt">constructor</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">apache_beam.pkg.mod.SomeClass</span>
 <span class="w">   </span><span class="nt">args</span><span class="p">:</span><span class="w"> </span><span class="p p-Indicator">[</span><span class="nv">1</span><span class="p p-Indicator">,</span><span class="w"> </span><span class="s">&#39;foo&#39;</span><span class="p p-Indicator">]</span>
 <span class="w">   </span><span class="nt">kwargs</span><span class="p">:</span>
 <span class="w">     </span><span class="nt">baz</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">3</span>
 </code></pre></div>

 <p>can be used to access the transform
 <code>apache_beam.pkg.mod.SomeClass(1, 'foo', baz=3)</code>.</p>
 <p>See also the documentation on
 <a href="https://beam.apache.org/documentation/sdks/yaml-inline-python/">Inlining
 Python</a>.</p>
 <h3 id="configuration_8">Configuration</h3>
 <ul>
 <li>
 <p><strong>constructor</strong> <code>string</code>  : Fully qualified name of a callable used to construct the
     transform.  Often this is a class such as
     <code>apache_beam.pkg.mod.SomeClass</code> but it can also be a function or
     any other callable that returns a PTransform.</p>
 </li>
 <li>
 <p><strong>args</strong> <code>Array[?]</code> (Optional)  : A list of parameters to pass to the callable as positional
     arguments.</p>
 </li>
 <li>
 <p><strong>kwargs</strong> <code>Map[string, ?]</code> (Optional)  : A list of parameters to pass to the callable as keyword
     arguments.</p>
 </li>
 </ul>
 <h3 id="usage_8">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">PyTransform</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">constructor</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;constructor&quot;</span>
 <span class="w">  </span><span class="nt">args</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">args</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">kwargs</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">a</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">kwargs_value_a</span>
 <span class="w">    </span><span class="nt">b</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">kwargs_value_b</span>
 <span class="w">    </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="sql">Sql</h2>
 <h3 id="configuration_9">Configuration</h3>
 <h3 id="usage_9">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Sql</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="windowinto">WindowInto</h2>
 <p>A window transform assigning windows to each element of a PCollection.</p>
 <p>The assigned windows will affect all downstream aggregating operations,
 which will aggregate by window as well as by key.</p>
 <p>See <a href="https://beam.apache.org/documentation/programming-guide/#windowing">the Beam documentation on windowing</a>
 for more details.</p>
 <p>Sizes, offsets, periods and gaps (where applicable) must be defined using
 a time unit suffix 'ms', 's', 'm', 'h' or 'd' for milliseconds, seconds,
 minutes, hours or days, respectively. If a time unit is not specified, it
 will default to 's'.</p>
 <p>For example</p>
 <div class="codehilite"><pre><span></span><code><span class="nt">windowing</span><span class="p">:</span>
 <span class="w">   </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">fixed</span>
 <span class="w">   </span><span class="nt">size</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">30s</span>
 </code></pre></div>

 <p>Note that any Yaml transform can have a
 <a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/yaml/README.md#windowing">windowing parameter</a>,
 which is applied to its inputs (if any) or outputs (if there are no inputs)
 which means that explicit WindowInto operations are not typically needed.</p>
 <h3 id="configuration_10">Configuration</h3>
 <ul>
 <li><strong>windowing</strong> <code>?</code> (Optional)  : the type and parameters of the windowing to perform</li>
 </ul>
 <h3 id="usage_10">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WindowInto</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">windowing</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">windowing</span>
 </code></pre></div>

 <h2 id="readfromavro">ReadFromAvro</h2>
 <p>A <code>PTransform</code> for reading records from avro files.</p>
 <p>Each record of the resulting PCollection will contain
 a single record read from a source. Records that are of simple types will be
 mapped to beam Rows with a single <code>record</code> field containing the records
 value. Records that are of Avro type <code>RECORD</code> will be mapped to Beam rows
 that comply with the schema contained in the Avro file that contains those
 records.</p>
 <h3 id="configuration_11">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>?</code> (Optional)</li>
 </ul>
 <h3 id="usage_11">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromAvro</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">path</span>
 </code></pre></div>

 <h2 id="writetoavro">WriteToAvro</h2>
 <p>A <code>PTransform</code> for writing avro files.</p>
 <p>If the input has a schema, a corresponding avro schema will be automatically
 generated and used to write the output records.</p>
 <h3 id="configuration_12">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>?</code> (Optional)</li>
 </ul>
 <h3 id="usage_12">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToAvro</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">path</span>
 </code></pre></div>

 <h2 id="readfrombigquery">ReadFromBigQuery</h2>
 <p>Reads data from BigQuery.</p>
 <p>Exactly one of table or query must be set.
 If query is set, neither row_restriction nor fields should be set.</p>
 <h3 id="configuration_13">Configuration</h3>
 <ul>
 <li>
 <p><strong>query</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>fields</strong> <code>Array[string]</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>row_restriction</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_13">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromBigQuery</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">query</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;query&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">fields</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;fields&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">row_restriction</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;row_restriction&quot;</span>
 </code></pre></div>

 <h2 id="writetobigquery">WriteToBigQuery</h2>
 <p>Writes data to BigQuery using the Storage Write API (https://cloud.google.com/bigquery/docs/write-api).</p>
 <p>This expects a single PCollection of Beam Rows and outputs two dead-letter queues (DLQ) that contain failed rows. The first DLQ has tag [FailedRows] and contains the failed rows. The second DLQ has tag [FailedRowsWithErrors] and contains failed rows and along with their respective errors.</p>
 <h3 id="configuration_14">Configuration</h3>
 <ul>
 <li>
 <p><strong>table</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>create_disposition</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>write_disposition</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 <li>
 <p><strong>num_streams</strong> <code>int32</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_14">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToBigQuery</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">create_disposition</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;create_disposition&quot;</span>
 <span class="w">  </span><span class="nt">write_disposition</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;write_disposition&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 <span class="w">  </span><span class="nt">num_streams</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">num_streams</span>
 </code></pre></div>

 <h2 id="readfromcsv">ReadFromCsv</h2>
 <p>A PTransform for reading comma-separated values (csv) files into a
 PCollection.</p>
 <h3 id="configuration_15">Configuration</h3>
 <ul>
 <li>
 <p><strong>path</strong> <code>string</code>  : The file path to read from.  The path can contain glob
     characters such as <code>*</code> and <code>?</code>.</p>
 </li>
 <li>
 <p><strong>delimiter</strong> <code>?</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>comment</strong> <code>?</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_15">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromCsv</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 <span class="w">  </span><span class="nt">delimiter</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">delimiter</span>
 <span class="w">  </span><span class="nt">comment</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">comment</span>
 </code></pre></div>

 <h2 id="writetocsv">WriteToCsv</h2>
 <p>A PTransform for writing a schema'd PCollection as a (set of)
 comma-separated values (csv) files.</p>
 <h3 id="configuration_16">Configuration</h3>
 <ul>
 <li>
 <p><strong>path</strong> <code>string</code>  : The file path to write to. The files written will
     begin with this prefix, followed by a shard identifier (see
     <code>num_shards</code>) according to the <code>file_naming</code> parameter.</p>
 </li>
 <li>
 <p><strong>delimiter</strong> <code>?</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_16">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToCsv</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">delimiter</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;delimiter&quot;</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 </code></pre></div>

 <h2 id="readfromjdbc">ReadFromJdbc</h2>
 <h3 id="configuration_17">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>type</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>query</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_17">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromJdbc</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;type&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">query</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;query&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="writetojdbc">WriteToJdbc</h2>
 <h3 id="configuration_18">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>type</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_18">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToJdbc</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;type&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="readfromjson">ReadFromJson</h2>
 <p>A PTransform for reading json values from files into a PCollection.</p>
 <h3 id="configuration_19">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>string</code>  : The file path to read from.  The path can contain glob
     characters such as <code>*</code> and <code>?</code>.</li>
 </ul>
 <h3 id="usage_19">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromJson</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 </code></pre></div>

 <h2 id="writetojson">WriteToJson</h2>
 <p>A PTransform for writing a PCollection as json values to files.</p>
 <h3 id="configuration_20">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>string</code>  : The file path to write to. The files written will
     begin with this prefix, followed by a shard identifier (see
     <code>num_shards</code>) according to the <code>file_naming</code> parameter.</li>
 </ul>
 <h3 id="usage_20">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToJson</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 </code></pre></div>

 <h2 id="readfromkafka">ReadFromKafka</h2>
 <h3 id="configuration_21">Configuration</h3>
 <ul>
 <li>
 <p><strong>schema</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>consumer_config</strong> <code>Map[string, string]</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>format</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>topic</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>bootstrap_servers</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>confluent_schema_registry_url</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>confluent_schema_registry_subject</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>auto_offset_reset_config</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 <li>
 <p><strong>file_descriptor_path</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>message_name</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_21">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromKafka</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">schema</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;schema&quot;</span>
 <span class="w">  </span><span class="nt">consumer_config</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">a</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;consumer_config_value_a&quot;</span>
 <span class="w">    </span><span class="nt">b</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;consumer_config_value_b&quot;</span>
 <span class="w">    </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;format&quot;</span>
 <span class="w">  </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;topic&quot;</span>
 <span class="w">  </span><span class="nt">bootstrap_servers</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;bootstrap_servers&quot;</span>
 <span class="w">  </span><span class="nt">confluent_schema_registry_url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;confluent_schema_registry_url&quot;</span>
 <span class="w">  </span><span class="nt">confluent_schema_registry_subject</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;confluent_schema_registry_subject&quot;</span>
 <span class="w">  </span><span class="nt">auto_offset_reset_config</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;auto_offset_reset_config&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 <span class="w">  </span><span class="nt">file_descriptor_path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;file_descriptor_path&quot;</span>
 <span class="w">  </span><span class="nt">message_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;message_name&quot;</span>
 </code></pre></div>

 <h2 id="writetokafka">WriteToKafka</h2>
 <h3 id="configuration_22">Configuration</h3>
 <ul>
 <li>
 <p><strong>format</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>topic</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>bootstrap_servers</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>producer_config_updates</strong> <code>Map[string, string]</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 <li>
 <p><strong>file_descriptor_path</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>message_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>schema</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_22">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToKafka</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;format&quot;</span>
 <span class="w">  </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;topic&quot;</span>
 <span class="w">  </span><span class="nt">bootstrap_servers</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;bootstrap_servers&quot;</span>
 <span class="w">  </span><span class="nt">producer_config_updates</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">a</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;producer_config_updates_value_a&quot;</span>
 <span class="w">    </span><span class="nt">b</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;producer_config_updates_value_b&quot;</span>
 <span class="w">    </span><span class="nt">c</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 <span class="w">  </span><span class="nt">file_descriptor_path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;file_descriptor_path&quot;</span>
 <span class="w">  </span><span class="nt">message_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;message_name&quot;</span>
 <span class="w">  </span><span class="nt">schema</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;schema&quot;</span>
 </code></pre></div>

 <h2 id="readfrommysql">ReadFromMySql</h2>
 <h3 id="configuration_23">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>query</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_23">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromMySql</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">query</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;query&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="writetomysql">WriteToMySql</h2>
 <h3 id="configuration_24">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_24">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToMySql</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="readfromoracle">ReadFromOracle</h2>
 <h3 id="configuration_25">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>query</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_25">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromOracle</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">query</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;query&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="writetooracle">WriteToOracle</h2>
 <h3 id="configuration_26">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_26">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToOracle</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="readfromparquet">ReadFromParquet</h2>
 <p>A <code>PTransform</code> for reading Parquet files.</p>
 <h3 id="configuration_27">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>?</code> (Optional)</li>
 </ul>
 <h3 id="usage_27">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromParquet</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">path</span>
 </code></pre></div>

 <h2 id="writetoparquet">WriteToParquet</h2>
 <p>A <code>PTransform</code> for writing parquet files.</p>
 <h3 id="configuration_28">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>?</code> (Optional)</li>
 </ul>
 <h3 id="usage_28">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToParquet</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">path</span>
 </code></pre></div>

 <h2 id="readfrompostgres">ReadFromPostgres</h2>
 <h3 id="configuration_29">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>query</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_29">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromPostgres</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">query</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;query&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="writetopostgres">WriteToPostgres</h2>
 <h3 id="configuration_30">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_30">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToPostgres</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="readfrompubsub">ReadFromPubSub</h2>
 <p>Reads messages from Cloud Pub/Sub.</p>
 <h3 id="configuration_31">Configuration</h3>
 <ul>
 <li>
 <p><strong>topic</strong> <code>string</code> (Optional)  : Cloud Pub/Sub topic in the form
     "projects/<project>/topics/<topic>". If provided, subscription must be
     None.</p>
 </li>
 <li>
 <p><strong>subscription</strong> <code>string</code> (Optional)  : Existing Cloud Pub/Sub subscription to use in the
     form "projects/<project>/subscriptions/<subscription>". If not
     specified, a temporary subscription will be created from the specified
     topic. If provided, topic must be None.</p>
 </li>
 <li>
 <p><strong>format</strong> <code>string</code>  : The expected format of the message payload.  Currently suported
     formats are</p>
 <ul>
 <li>raw: Produces records with a single <code>payload</code> field whose contents
     are the raw bytes of the pubsub message.</li>
 <li>avro: Parses records with a given avro schema.</li>
 <li>json: Parses records with a given json schema.</li>
 </ul>
 </li>
 <li>
 <p><strong>schema</strong> <code>?</code> (Optional)  : Schema specification for the given format.</p>
 </li>
 <li>
 <p><strong>attributes</strong> <code>Array[string]</code> (Optional)  : List of attribute keys whose values will be flattened into the
     output message as additional fields.  For example, if the format is <code>raw</code>
     and attributes is <code>["a", "b"]</code> then this read will produce elements of
     the form <code>Row(payload=..., a=..., b=...)</code>.</p>
 </li>
 <li>
 <p><strong>attributes_map</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>id_attribute</strong> <code>string</code> (Optional)  : The attribute on incoming Pub/Sub messages to use as a unique
     record identifier. When specified, the value of this attribute (which
     can be any string that uniquely identifies the record) will be used for
     deduplication of messages. If not provided, we cannot guarantee
     that no duplicate data will be delivered on the Pub/Sub stream. In this
     case, deduplication of the stream will be strictly best effort.</p>
 </li>
 <li>
 <p><strong>timestamp_attribute</strong> <code>string</code> (Optional)  : Message value to use as element timestamp. If None,
     uses message publishing time as the timestamp.</p>
 <p>Timestamp values should be in one of two formats:</p>
 <ul>
 <li>A numerical value representing the number of milliseconds since the
   Unix epoch.</li>
 <li>A string in RFC 3339 format, UTC timezone. Example:
   <code>2015-10-29T23:41:41.123Z</code>. The sub-second component of the
   timestamp is optional, and digits beyond the first three (i.e., time
   units smaller than milliseconds) may be ignored.</li>
 </ul>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code>      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 </ul>
 <h3 id="usage_31">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromPubSub</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;topic&quot;</span>
 <span class="w">  </span><span class="nt">subscription</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;subscription&quot;</span>
 <span class="w">  </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;format&quot;</span>
 <span class="w">  </span><span class="nt">schema</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">schema</span>
 <span class="w">  </span><span class="nt">attributes</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;attributes&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">attributes_map</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;attributes_map&quot;</span>
 <span class="w">  </span><span class="nt">id_attribute</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;id_attribute&quot;</span>
 <span class="w">  </span><span class="nt">timestamp_attribute</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;timestamp_attribute&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 </code></pre></div>

 <h2 id="writetopubsub">WriteToPubSub</h2>
 <p>Writes messages to Cloud Pub/Sub.</p>
 <h3 id="configuration_32">Configuration</h3>
 <ul>
 <li>
 <p><strong>topic</strong> <code>string</code>  : Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".</p>
 </li>
 <li>
 <p><strong>format</strong> <code>string</code>  : How to format the message payload.  Currently suported
     formats are</p>
 <ul>
 <li>raw: Expects a message with a single field (excluding
     attribute-related fields) whose contents are used as the raw bytes
     of the pubsub message.</li>
 <li>avro: Encodes records with a given avro schema, which may be inferred
     from the input PCollection schema.</li>
 <li>json: Formats records with a given json schema, which may be inferred
     from the input PCollection schema.</li>
 </ul>
 </li>
 <li>
 <p><strong>schema</strong> <code>?</code> (Optional)  : Schema specification for the given format.</p>
 </li>
 <li>
 <p><strong>attributes</strong> <code>Array[string]</code> (Optional)  : List of attribute keys whose values will be pulled out as
     PubSub message attributes.  For example, if the format is <code>raw</code>
     and attributes is <code>["a", "b"]</code> then elements of the form
     <code>Row(any_field=..., a=..., b=...)</code> will result in PubSub messages whose
     payload has the contents of any_field and whose attribute will be
     populated with the values of <code>a</code> and <code>b</code>.</p>
 </li>
 <li>
 <p><strong>attributes_map</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>id_attribute</strong> <code>string</code> (Optional)  : If set, will set an attribute for each Cloud Pub/Sub message
     with the given name and a unique value. This attribute can then be used
     in a ReadFromPubSub PTransform to deduplicate messages.</p>
 </li>
 <li>
 <p><strong>timestamp_attribute</strong> <code>string</code> (Optional)  : If set, will set an attribute for each Cloud Pub/Sub
     message with the given name and the message's publish time as the value.</p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code>      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 </ul>
 <h3 id="usage_32">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToPubSub</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">topic</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;topic&quot;</span>
 <span class="w">  </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;format&quot;</span>
 <span class="w">  </span><span class="nt">schema</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">schema</span>
 <span class="w">  </span><span class="nt">attributes</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;attributes&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">attributes_map</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;attributes_map&quot;</span>
 <span class="w">  </span><span class="nt">id_attribute</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;id_attribute&quot;</span>
 <span class="w">  </span><span class="nt">timestamp_attribute</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;timestamp_attribute&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 </code></pre></div>

 <h2 id="readfrompubsublite">ReadFromPubSubLite</h2>
 <h3 id="configuration_33">Configuration</h3>
 <ul>
 <li>
 <p><strong>project</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>schema</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>format</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>subscription_name</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>location</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>attributes</strong> <code>Array[string]</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>attribute_map</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>attribute_id</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 <li>
 <p><strong>file_descriptor_path</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>message_name</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_33">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromPubSubLite</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">project</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;project&quot;</span>
 <span class="w">  </span><span class="nt">schema</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;schema&quot;</span>
 <span class="w">  </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;format&quot;</span>
 <span class="w">  </span><span class="nt">subscription_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;subscription_name&quot;</span>
 <span class="w">  </span><span class="nt">location</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;location&quot;</span>
 <span class="w">  </span><span class="nt">attributes</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;attributes&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">attribute_map</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;attribute_map&quot;</span>
 <span class="w">  </span><span class="nt">attribute_id</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;attribute_id&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 <span class="w">  </span><span class="nt">file_descriptor_path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;file_descriptor_path&quot;</span>
 <span class="w">  </span><span class="nt">message_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;message_name&quot;</span>
 </code></pre></div>

 <h2 id="writetopubsublite">WriteToPubSubLite</h2>
 <h3 id="configuration_34">Configuration</h3>
 <ul>
 <li>
 <p><strong>project</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>format</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>topic_name</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>location</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>attributes</strong> <code>Array[string]</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>attribute_id</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>error_handling</strong> <code>Row</code> (Optional)      </p>
 <p>Row fields:</p>
 <ul>
 <li><strong>output</strong> <code>string</code></li>
 </ul>
 </li>
 <li>
 <p><strong>file_descriptor_path</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>message_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>schema</strong> <code>string</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_34">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToPubSubLite</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">project</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;project&quot;</span>
 <span class="w">  </span><span class="nt">format</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;format&quot;</span>
 <span class="w">  </span><span class="nt">topic_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;topic_name&quot;</span>
 <span class="w">  </span><span class="nt">location</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;location&quot;</span>
 <span class="w">  </span><span class="nt">attributes</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;attributes&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="w">  </span><span class="nt">attribute_id</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;attribute_id&quot;</span>
 <span class="w">  </span><span class="nt">error_handling</span><span class="p">:</span>
 <span class="w">    </span><span class="nt">output</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;output&quot;</span>
 <span class="w">  </span><span class="nt">file_descriptor_path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;file_descriptor_path&quot;</span>
 <span class="w">  </span><span class="nt">message_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;message_name&quot;</span>
 <span class="w">  </span><span class="nt">schema</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;schema&quot;</span>
 </code></pre></div>

 <h2 id="readfromsqlserver">ReadFromSqlServer</h2>
 <h3 id="configuration_35">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>query</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_35">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromSqlServer</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">query</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;query&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="writetosqlserver">WriteToSqlServer</h2>
 <h3 id="configuration_36">Configuration</h3>
 <ul>
 <li>
 <p><strong>driver_class_name</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>url</strong> <code>string</code>  </p>
 </li>
 <li>
 <p><strong>username</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>password</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>table</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>driver_jars</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_properties</strong> <code>string</code> (Optional)  </p>
 </li>
 <li>
 <p><strong>connection_init_sql</strong> <code>Array[string]</code> (Optional)</p>
 </li>
 </ul>
 <h3 id="usage_36">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToSqlServer</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">driver_class_name</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_class_name&quot;</span>
 <span class="w">  </span><span class="nt">url</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;url&quot;</span>
 <span class="w">  </span><span class="nt">username</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;username&quot;</span>
 <span class="w">  </span><span class="nt">password</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;password&quot;</span>
 <span class="w">  </span><span class="nt">table</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;table&quot;</span>
 <span class="w">  </span><span class="nt">driver_jars</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;driver_jars&quot;</span>
 <span class="w">  </span><span class="nt">connection_properties</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;connection_properties&quot;</span>
 <span class="w">  </span><span class="nt">connection_init_sql</span><span class="p">:</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="s">&quot;connection_init_sql&quot;</span>
 <span class="w">  </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 </code></pre></div>

 <h2 id="readfromtext">ReadFromText</h2>
 <p>Reads lines from a text files.</p>
 <p>The resulting PCollection consists of rows with a single string filed named
 "line."</p>
 <h3 id="configuration_37">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>string</code>  : The file path to read from.  The path can contain glob
     characters such as <code>*</code> and <code>?</code>.</li>
 </ul>
 <h3 id="usage_37">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">ReadFromText</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 </code></pre></div>

 <h2 id="writetotext">WriteToText</h2>
 <p>Writes a PCollection to a (set of) text files(s).</p>
 <p>The input must be a PCollection whose schema has exactly one field.</p>
 <h3 id="configuration_38">Configuration</h3>
 <ul>
 <li><strong>path</strong> <code>string</code>  : The file path to write to. The files written will
     begin with this prefix, followed by a shard identifier.</li>
 </ul>
 <h3 id="usage_38">Usage</h3>
 <div class="codehilite"><pre><span></span><code><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">WriteToText</span>
 <span class="nt">input</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
 <span class="nt">config</span><span class="p">:</span>
 <span class="w">  </span><span class="nt">path</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;path&quot;</span>
 </code></pre></div>
                 </div>
               </body>
             </html>