website/www/site/layouts/partials/section-menu/en/documentation.html

{{/*
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
   http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. See accompanying LICENSE file.
*/}}

<li><span class="section-nav-list-main-title">Documentation</span></li>
<li><a href="/documentation">Using the Documentation</a></li>
<li class="section-nav-item--collapsible">
  <span class="section-nav-list-title">Concepts</span>

  <ul class="section-nav-list">
    <li><a href="/documentation/basics/">Basics of the Beam model</a></li>
    <li><a href="/documentation/runtime/model/">How Beam executes a pipeline</a></li>
  </ul>
</li>
<li class="section-nav-item--collapsible">
  <span class="section-nav-list-title">Beam programming guide</span>

  <ul class="section-nav-list">
    <li><a href="/documentation/programming-guide/">Overview</a></li>
    <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">PCollections</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
        <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Transforms</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
        <li>
          <span class="section-nav-list-title">Core Beam transforms</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
            <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
            <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
            <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
            <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
            <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
          </ul>
        </li>

        <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
        <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
        <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
        <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Pipeline I/O</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
        <li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>

        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Built-in I/O connector guides</span>
          <ul class="section-nav-list">
            <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
            </a></li>
            <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
            <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
            <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
            <li><a href="/documentation/io/built-in/snowflake/">Snowflake I/O connector</a></li>
          </ul>
      </li>


        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Developing new I/O connectors</span>

          <ul class="section-nav-list">
          <li><a href="/documentation/io/developing-io-overview/">Overview: Developing connectors</a></li>
          <li><a href="/documentation/io/developing-io-java/">Developing connectors (Java)</a></li>
          <li><a href="/documentation/io/developing-io-python/">Developing connectors (Python)</a></li>
          </ul>
        </li>

        <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Schemas</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#what-is-a-schema">What is a schema</a></li>
        <li><a href="/documentation/programming-guide/#schemas-for-pl-types">Schemas for programming language types</a></li>
        <li><a href="/documentation/programming-guide/#schema-definition">Schema definition</a></li>
        <li><a href="/documentation/programming-guide/#logical-types">Logical types</a></li>
        <li><a href="/documentation/programming-guide/#creating-schemas">Creating schemas</a></li>
        <li><a href="/documentation/programming-guide/#using-schemas">Using schemas</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Data encoding and type safety</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
        <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
        <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Windowing</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
        <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
        <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
        <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
        <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Triggers</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
        <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
        <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
        <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
        <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
        <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Metrics</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#metrics">Metrics basics</a></li>
        <li><a href="/documentation/programming-guide/#types-of-metrics">Types of metrics</a></li>
        <li><a href="/documentation/programming-guide/#querying-metrics">Querying metrics</a></li>
        <li><a href="/documentation/programming-guide/#using-metrics">Using metrics in pipeline</a></li>
        <li><a href="/documentation/programming-guide/#export-metrics">Export metrics</a></li>
      </ul>
    </li>
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">State and Timers</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#types-of-state">Types of state</a></li>
        <li><a href="/documentation/programming-guide/#deferred-state-reads">Deferred state reads</a></li>
        <li><a href="/documentation/programming-guide/#timers">Timers</a></li>
        <li><a href="/documentation/programming-guide/#garbage-collecting-state">Garbage collecting state</a></li>
        <li><a href="/documentation/programming-guide/#state-timers-examples">State and timers examples</a></li>
      </ul>
    </li>

    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Splittable DoFns</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#sdf-basics">Basics</a></li>
        <li><a href="/documentation/programming-guide/#sizing-and-progress">Sizing and progress</a></li>
        <li><a href="/documentation/programming-guide/#user-initiated-checkpoint">User-initiated checkpoint</a></li>
        <li><a href="/documentation/programming-guide/#runner-initiated-split">Runner initiated split</a></li>
        <li><a href="/documentation/programming-guide/#watermark-estimation">Watermark estimation</a></li>
        <li><a href="/documentation/programming-guide/#truncating-during-drain">Truncating during drain</a></li>
        <li><a href="/documentation/programming-guide/#bundle-finalization">Bundle finalization</a></li>
      </ul>
    </li>

    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Multi-language Pipelines</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/programming-guide/#create-x-lang-transforms">Creating cross-language transforms</a></li>
        <li><a href="/documentation/programming-guide/#use-x-lang-transforms">Using cross-language transforms</a></li>
        <li><a href="/documentation/programming-guide/#x-lang-transform-runner-support">Runner Support</a></li>
      </ul>
    </li>
  </ul>
</li>
<li class="section-nav-item--collapsible">
  <span class="section-nav-list-title">Pipeline development lifecycle</span>
  <ul class="section-nav-list">
    <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
    <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
    <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
  </ul>
</li>
<li class="section-nav-item--collapsible">
  <span class="section-nav-list-title">Common pipeline patterns</span>

  <ul class="section-nav-list">
    <li><a href="/documentation/patterns/overview/">Overview</a></li>
    <li><a href="/documentation/patterns/file-processing/">File processing</a></li>
    <li><a href="/documentation/patterns/side-inputs/">Side inputs</a></li>
    <li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
    <li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
    <li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
    <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
    <li><a href="/documentation/patterns/ai-platform/">AI Platform</a></li>
    <li><a href="/documentation/patterns/schema/">Schema</a></li>
    <li><a href="/documentation/patterns/bqml/">BigQuery ML</a></li>
    <li><a href="/documentation/patterns/cross-language/">Cross-language transforms</a></li>
  </ul>
</li>
<li class="section-nav-item--collapsible">
  <span class="section-nav-list-title">Runtime systems</span>

  <ul class="section-nav-list">
    <li><a href="/documentation/runtime/environments/">Container environments</a></li>
    <li><a href="/documentation/runtime/resource-hints/">Resource hints</a></li>
    <li><a href="/documentation/runtime/sdk-harness-config/">SDK Harness Configuration</a></li>
  </ul>
</li>
<li class="section-nav-item--collapsible">
  <span class="section-nav-list-title">Transform catalog</span>

  <ul class="section-nav-list">
    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Python</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/transforms/python/overview/">Overview</a></li>
        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Element-wise</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/transforms/python/elementwise/filter/">Filter</a></li>
            <li><a href="/documentation/transforms/python/elementwise/flatmap/">FlatMap</a></li>
            <li><a href="/documentation/transforms/python/elementwise/keys/">Keys</a></li>
            <li><a href="/documentation/transforms/python/elementwise/kvswap/">KvSwap</a></li>
            <li><a href="/documentation/transforms/python/elementwise/map/">Map</a></li>
            <li><a href="/documentation/transforms/python/elementwise/pardo/">ParDo</a></li>
            <li><a href="/documentation/transforms/python/elementwise/partition/">Partition</a></li>
            <li><a href="/documentation/transforms/python/elementwise/regex/">Regex</a></li>
            <li><a href="/documentation/transforms/python/elementwise/reify/">Reify</a></li>
            <li><a href="/documentation/transforms/python/elementwise/tostring/">ToString</a></li>
            <li><a href="/documentation/transforms/python/elementwise/values/">Values</a></li>
            <li><a href="/documentation/transforms/python/elementwise/withtimestamps/">WithTimestamps</a></li>
          </ul>
        </li>
        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Aggregation</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
            <li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
            <li><a href="/documentation/transforms/python/aggregation/combineperkey/">CombinePerKey</a></li>
            <li><a href="/documentation/transforms/python/aggregation/combinevalues/">CombineValues</a></li>
            <li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
            <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
            <li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
            <li><a href="/documentation/transforms/python/aggregation/groupby/">GroupBy</a></li>
            <li><a href="/documentation/transforms/python/aggregation/groupintobatches/">GroupIntoBatches</a></li>
            <li><a href="/documentation/transforms/python/aggregation/latest/">Latest</a></li>
            <li><a href="/documentation/transforms/python/aggregation/max/">Max</a></li>
            <li><a href="/documentation/transforms/python/aggregation/min/">Min</a></li>
            <li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
            <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
            <li><a href="/documentation/transforms/python/aggregation/sum/">Sum</a></li>
            <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
          </ul>
        </li>
        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Other</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/transforms/python/other/create/">Create</a></li>
            <li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
            <li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
            <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
          </ul>
        </li>
      </ul>
    </li>

    <li class="section-nav-item--collapsible">
      <span class="section-nav-list-title">Java</span>

      <ul class="section-nav-list">
        <li><a href="/documentation/transforms/java/overview/">Overview</a></li>
        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Element-wise</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/transforms/java/elementwise/filter/">Filter</a></li>
            <li><a href="/documentation/transforms/java/elementwise/flatmapelements/">FlatMapElements</a></li>
            <li><a href="/documentation/transforms/java/elementwise/keys/">Keys</a></li>
            <li><a href="/documentation/transforms/java/elementwise/kvswap/">KvSwap</a></li>
            <li><a href="/documentation/transforms/java/elementwise/mapelements/">MapElements</a></li>
            <li><a href="/documentation/transforms/java/elementwise/pardo/">ParDo</a></li>
            <li><a href="/documentation/transforms/java/elementwise/partition/">Partition</a></li>
            <li><a href="/documentation/transforms/java/elementwise/regex/">Regex</a></li>
            <li><a href="/documentation/transforms/java/elementwise/reify/">Reify</a></li>
            <li><a href="/documentation/transforms/java/elementwise/values/">Values</a></li>
            <li><a href="/documentation/transforms/java/elementwise/tostring/">ToString</a></li>
            <li><a href="/documentation/transforms/java/elementwise/withkeys/">WithKeys</a></li>
            <li><a href="/documentation/transforms/java/elementwise/withtimestamps/">WithTimestamps</a></li>
          </ul>
        </li>
        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Aggregation</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/transforms/java/aggregation/approximatequantiles/">ApproximateQuantiles</a></li>
            <li><a href="/documentation/transforms/java/aggregation/approximateunique/">ApproximateUnique</a></li>
            <li><a href="/documentation/transforms/java/aggregation/cogroupbykey/">CoGroupByKey</a></li>
            <li><a href="/documentation/transforms/java/aggregation/combine/">Combine</a></li>
            <li><a href="/documentation/transforms/java/aggregation/combinewithcontext/">CombineWithContext</a></li>
            <li><a href="/documentation/transforms/java/aggregation/count/">Count</a></li>
            <li><a href="/documentation/transforms/java/aggregation/distinct/">Distinct</a></li>
            <li><a href="/documentation/transforms/java/aggregation/groupbykey/">GroupByKey</a></li>
            <li><a href="/documentation/transforms/java/aggregation/groupintobatches/">GroupIntoBatches</a></li>
            <li><a href="/documentation/transforms/java/aggregation/hllcount/">HllCount</a></li>
            <li><a href="/documentation/transforms/java/aggregation/latest/">Latest</a></li>
            <li><a href="/documentation/transforms/java/aggregation/max/">Max</a></li>
            <li><a href="/documentation/transforms/java/aggregation/mean/">Mean</a></li>
            <li><a href="/documentation/transforms/java/aggregation/min/">Min</a></li>
            <li><a href="/documentation/transforms/java/aggregation/sample/">Sample</a></li>
            <li><a href="/documentation/transforms/java/aggregation/sum/">Sum</a></li>
            <li><a href="/documentation/transforms/java/aggregation/top/">Top</a></li>
          </ul>
        </li>
        <li class="section-nav-item--collapsible">
          <span class="section-nav-list-title">Other</span>

          <ul class="section-nav-list">
            <li><a href="/documentation/transforms/java/other/create/">Create</a></li>
            <li><a href="/documentation/transforms/java/other/flatten/">Flatten</a></li>
            <li><a href="/documentation/transforms/java/other/passert/">PAssert</a></li>
            <li><a href="/documentation/transforms/java/other/view/">View</a></li>
            <li><a href="/documentation/transforms/java/other/window/">Window</a></li>
          </ul>
        </li>
      </ul>
    </li>
  </ul>
</li>
<li><a href="/documentation/glossary/">Glossary</a></li>
<li><a href="https://cwiki.apache.org/confluence/display/BEAM/Apache+Beam">Beam Wiki <img src="/images/external-link-icon.png" width="14" height="14" alt="External link."></a></li>