| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| # NOTE: This file is autogenerated and should not be edited by hand. |
| # Configs are generated based on the expansion service |
| # configuration in /sdks/standard_expansion_services.yaml. |
| # Refer to gen_xlang_wrappers.py for more info. |
| # |
| # Last updated on: 2026-08-15 |
| |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: '' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: The Datadog API key. |
| name: api_key |
| nullable: false |
| type: str |
| - description: The number of events to batch together for each write. |
| name: batch_count |
| nullable: true |
| type: int32 |
| - description: Specifies how to handle errors. |
| name: error_handling |
| nullable: true |
| type: Row(output=<class 'str'>) |
| - description: The maximum buffer size in bytes. |
| name: max_buffer_size |
| nullable: true |
| type: int64 |
| - description: The minimum number of events to batch together for each write. |
| name: min_batch_count |
| nullable: true |
| type: int32 |
| - description: The degree of parallelism for writing. |
| name: parallelism |
| nullable: true |
| type: int32 |
| - description: The Datadog API URL. |
| name: url |
| nullable: false |
| type: str |
| identifier: beam:schematransform:org.apache.beam:datadog_write:v1 |
| name: DatadogWrite |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: 'Outputs a PCollection of Beam Rows, each containing a single INT64 |
| number called "value". The count is produced from the given "start" value and |
| either up to the given "end" or until 2^63 - 1. |
| |
| To produce an unbounded PCollection, simply do not specify an "end" value. Unbounded |
| sequences can specify a "rate" for output elements. |
| |
| In all cases, the sequence of numbers is generated in parallel, so there is no |
| inherent ordering between the generated values' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: The maximum number to generate (exclusive). Will be an unbounded |
| sequence if left unspecified. |
| name: end |
| nullable: true |
| type: int64 |
| - description: Specifies the rate to generate a given number of elements per a given |
| number of seconds. Applicable only to unbounded sequences. |
| name: rate |
| nullable: true |
| type: Row(elements=<class 'int64'>, seconds=typing.Optional[int64]) |
| - description: The minimum number to generate (inclusive). |
| name: start |
| nullable: false |
| type: int64 |
| identifier: beam:schematransform:org.apache.beam:generate_sequence:v1 |
| name: GenerateSequence |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: '' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: The MongoDB collection to read from. |
| name: collection |
| nullable: false |
| type: str |
| - description: The MongoDB database to read from. |
| name: database |
| nullable: false |
| type: str |
| - description: This option specifies whether and where to output rows that failed |
| to be read. |
| name: error_handling |
| nullable: true |
| type: Row(output=<class 'str'>) |
| - description: An optional BSON filter to apply to the read. This should be a valid |
| JSON string. |
| name: filter |
| nullable: true |
| type: str |
| - description: The schema in which the data is encoded, defined with JSON-schema |
| syntax (https://json-schema.org/). |
| name: schema |
| nullable: false |
| type: str |
| - description: The connection URI for the MongoDB server. |
| name: uri |
| nullable: false |
| type: str |
| identifier: beam:schematransform:org.apache.beam:mongodb_read:v1 |
| name: MongodbRead |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: '' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: The number of documents to include in each batch write. |
| name: batch_size |
| nullable: true |
| type: int64 |
| - description: The MongoDB collection to write to. |
| name: collection |
| nullable: false |
| type: str |
| - description: The MongoDB database to write to. |
| name: database |
| nullable: false |
| type: str |
| - description: 'This option specifies whether and where to output unwritable rows. |
| Note: Error handling is currently limited to data conversion failures before |
| sending to the MongoDB driver, as the underlying MongoDbIO does not yet support |
| dead-letter queues for write failures.' |
| name: error_handling |
| nullable: true |
| type: Row(output=<class 'str'>) |
| - description: The connection URI for the MongoDB server. |
| name: uri |
| nullable: false |
| type: str |
| identifier: beam:schematransform:org.apache.beam:mongodb_write:v1 |
| name: MongodbWrite |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: Reads records from an Amazon Kinesis stream and outputs Beam Rows |
| with `data` (bytes) plus stream metadata fields. |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: AWS access key id. |
| name: aws_access_key |
| nullable: false |
| type: str |
| - description: AWS secret access key. |
| name: aws_secret_key |
| nullable: false |
| type: str |
| - description: 'Where to start reading in the stream: LATEST, TRIM_HORIZON, or AT_TIMESTAMP.' |
| name: initial_position_in_stream |
| nullable: true |
| type: str |
| - description: Epoch millis timestamp used when initial_position_in_stream is AT_TIMESTAMP. |
| name: initial_timestamp_in_stream |
| nullable: true |
| type: int64 |
| - description: Maximum number of records to hold in memory per shard. |
| name: max_capacity_per_shard |
| nullable: true |
| type: int64 |
| - description: Maximum number of records to read. When set, the resulting PCollection |
| is bounded. |
| name: max_num_records |
| nullable: true |
| type: int64 |
| - description: Maximum read time in milliseconds. When set, the resulting PCollection |
| is bounded. |
| name: max_read_time |
| nullable: true |
| type: int64 |
| - description: Fixed delay between GetRecords calls in milliseconds. |
| name: rate_limit |
| nullable: true |
| type: int64 |
| - description: AWS region, for example us-east-1. |
| name: region |
| nullable: false |
| type: str |
| - description: Max records returned by a single GetRecords call (1-10000). |
| name: request_records_limit |
| nullable: true |
| type: int64 |
| - description: Optional custom Kinesis service endpoint URI. |
| name: service_endpoint |
| nullable: true |
| type: str |
| - description: Kinesis stream name to read from. |
| name: stream_name |
| nullable: false |
| type: str |
| - description: Threshold duration in milliseconds after which a shard is considered |
| up to date. |
| name: up_to_date_threshold |
| nullable: true |
| type: int64 |
| - description: Whether to verify TLS certificates. Defaults to true. Never disable |
| in production. |
| name: verify_certificate |
| nullable: true |
| type: boolean |
| - description: Idle duration threshold in milliseconds for ARRIVAL_TIME watermark |
| policy. |
| name: watermark_idle_duration_threshold |
| nullable: true |
| type: int64 |
| - description: 'Watermark policy: ARRIVAL_TIME or PROCESSING_TIME.' |
| name: watermark_policy |
| nullable: true |
| type: str |
| identifier: beam:schematransform:org.apache.beam:kinesis_read:v1 |
| name: KinesisRead |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: Writes Beam Rows to an Amazon Kinesis stream. Each input row must include |
| a `data` (bytes) field, or a `payload` string/bytes field. Optional per-row `partition_key` |
| overrides the configured default partition key. |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: Enable KPL-compatible record aggregation. |
| name: aggregation_enabled |
| nullable: true |
| type: boolean |
| - description: Max time in milliseconds to buffer records when aggregation is enabled. |
| name: aggregation_max_buffered_time |
| nullable: true |
| type: int64 |
| - description: Max aggregated record size in bytes when aggregation is enabled. |
| name: aggregation_max_bytes |
| nullable: true |
| type: int64 |
| - description: Shard map refresh interval in minutes when aggregation is enabled. |
| name: aggregation_shard_refresh_interval |
| nullable: true |
| type: int64 |
| - description: AWS access key id. |
| name: aws_access_key |
| nullable: false |
| type: str |
| - description: AWS secret access key. |
| name: aws_secret_key |
| nullable: false |
| type: str |
| - description: Default partition key used when the input row does not contain a |
| partition_key field. |
| name: partition_key |
| nullable: false |
| type: str |
| - description: AWS region, for example us-east-1. |
| name: region |
| nullable: false |
| type: str |
| - description: Optional custom Kinesis service endpoint URI. |
| name: service_endpoint |
| nullable: true |
| type: str |
| - description: Kinesis stream name to write to. |
| name: stream_name |
| nullable: false |
| type: str |
| - description: Whether to verify TLS certificates. Defaults to true. Never disable |
| in production. |
| name: verify_certificate |
| nullable: true |
| type: boolean |
| identifier: beam:schematransform:org.apache.beam:kinesis_write:v1 |
| name: KinesisWrite |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: '' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: Decompression type to use when reading input files. |
| name: compression |
| nullable: false |
| type: str |
| - description: This option specifies whether and where to output unwritable rows. |
| name: error_handling |
| nullable: true |
| type: Row(output=<class 'str'>) |
| - description: Filename or file pattern used to find input files. |
| name: file_pattern |
| nullable: false |
| type: str |
| - description: Validate file pattern. |
| name: validate |
| nullable: false |
| type: boolean |
| identifier: beam:schematransform:org.apache.beam:tfrecord_read:v1 |
| name: TfrecordRead |
| - default_service: sdks:java:io:expansion-service:shadowJar |
| description: '' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: Option to indicate the output sink's compression type. Default is |
| NONE. |
| name: compression |
| nullable: false |
| type: str |
| - description: This option specifies whether and where to output unwritable rows. |
| name: error_handling |
| nullable: true |
| type: Row(output=<class 'str'>) |
| - description: The suffix of each file written, combined with prefix and shardTemplate. |
| name: filename_suffix |
| nullable: true |
| type: str |
| - description: Maximum number of writers created in a bundle before spilling to |
| shuffle. |
| name: max_num_writers_per_bundle |
| nullable: true |
| type: int32 |
| - description: Whether to skip the spilling of data caused by having maxNumWritersPerBundle. |
| name: no_spilling |
| nullable: true |
| type: boolean |
| - description: The number of shards to use, or 0 for automatic. |
| name: num_shards |
| nullable: false |
| type: int32 |
| - description: The directory to which files will be written. |
| name: output_prefix |
| nullable: false |
| type: str |
| - description: The shard template of each file written, combined with prefix and |
| suffix. |
| name: shard_template |
| nullable: true |
| type: str |
| identifier: beam:schematransform:org.apache.beam:tfrecord_write:v1 |
| name: TfrecordWrite |
| - default_service: sdks:java:io:messaging-expansion-service:shadowJar |
| description: 'Reads messages from a JMS broker and outputs each message as a single |
| `payload` (string) field. |
| |
| |
| By default the read is unbounded (streaming): it keeps consuming messages from |
| the specified queue or topic until the pipeline is stopped. Setting `maxNumRecords` |
| and/or `maxReadTimeSeconds` bounds the read, producing a bounded (batch) PCollection.' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: 'The JMS acknowledge mode: CLIENT_ACKNOWLEDGE, CLIENT_ACKNOWLEDGE_UNSAFE, |
| or INDIVIDUAL_ACKNOWLEDGE.' |
| name: acknowledge_mode |
| nullable: true |
| type: str |
| - description: Close timeout for the JMS connection in seconds. |
| name: close_timeout_seconds |
| nullable: true |
| type: int64 |
| - description: 'Configuration options to set up the JMS connection. |
| |
| Note: if connection factory class name is set, spin up a persistent expansion |
| service with the provider client JAR on the classpath: |
| |
| java -cp <expansion-service-jar>:<provider-client-jar> |
| |
| org.apache.beam.sdk.expansion.service.ExpansionService <port> |
| |
| and pass expansion_service=''localhost:<port>'' to the transform. Currently, |
| only ActiveMQ (org.apache.activemq.ActiveMQConnectionFactory) is embedded into |
| the expansion service' |
| name: connection_configuration |
| nullable: false |
| type: Row(connection_factory_class_name=typing.Optional[str], password=typing.Optional[str], |
| server_uri=<class 'str'>, username=typing.Optional[str]) |
| - description: The proprietary integer code for individual message acknowledgment |
| when using INDIVIDUAL_ACKNOWLEDGE. |
| name: individual_acknowledge_mode_code |
| nullable: true |
| type: int32 |
| - description: The max number of records to receive. Setting this will result in |
| a bounded PCollection. |
| name: max_num_records |
| nullable: true |
| type: int64 |
| - description: The maximum time for this source to read messages. Setting this will |
| result in a bounded PCollection. |
| name: max_read_time_seconds |
| nullable: true |
| type: int64 |
| - description: The JMS queue to read from. Exclusively one of queue or topic must |
| be specified. |
| name: queue |
| nullable: true |
| type: str |
| - description: The JMS topic to read from. Exclusively one of queue or topic must |
| be specified. |
| name: topic |
| nullable: true |
| type: str |
| identifier: beam:schematransform:org.apache.beam:jms_read:v1 |
| name: ReadFromJms |
| - default_service: sdks:java:io:messaging-expansion-service:shadowJar |
| description: 'Publishes messages to a JMS broker. Expects an input PCollection of |
| rows with a `payload` (string) or `bytes` field, each of which is published as |
| one JMS TextMessage. |
| |
| |
| Works with both bounded (batch) and unbounded (streaming) input PCollections.' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: 'Configuration options to set up the JMS connection. |
| |
| Note: if connection factory class name is set, spin up a persistent expansion |
| service with the provider client JAR on the classpath: |
| |
| java -cp <expansion-service-jar>:<provider-client-jar> |
| |
| org.apache.beam.sdk.expansion.service.ExpansionService <port> |
| |
| and pass expansion_service=''localhost:<port>'' to the transform. Currently, |
| only ActiveMQ (org.apache.activemq.ActiveMQConnectionFactory) is embedded into |
| the expansion service' |
| name: connection_configuration |
| nullable: false |
| type: Row(connection_factory_class_name=typing.Optional[str], password=typing.Optional[str], |
| server_uri=<class 'str'>, username=typing.Optional[str]) |
| - description: The JMS queue to write to. Exclusively one of queue or topic must |
| be specified. |
| name: queue |
| nullable: true |
| type: str |
| - description: The JMS topic to write to. Exclusively one of queue or topic must |
| be specified. |
| name: topic |
| nullable: true |
| type: str |
| identifier: beam:schematransform:org.apache.beam:jms_write:v1 |
| name: WriteToJms |
| - default_service: sdks:java:io:messaging-expansion-service:shadowJar |
| description: 'Reads messages from an MQTT broker and outputs each payload as a single |
| `bytes` field. |
| |
| |
| By default the read is unbounded (streaming): it keeps consuming messages from |
| the subscribed topic until the pipeline is stopped. Setting `maxNumRecords` and/or |
| `maxReadTimeSeconds` bounds the read, producing a bounded (batch) PCollection. |
| |
| |
| Note: streaming reads require a runner that supports portable streaming (e.g. |
| Prism, Flink, or Dataflow). The legacy local Python DirectRunner does not execute |
| portable streaming cross-language reads.' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: Configuration options to set up the MQTT connection. |
| name: connection_configuration |
| nullable: false |
| type: Row(client_id=typing.Optional[str], password=typing.Optional[str], server_uri=<class |
| 'str'>, topic=typing.Optional[str], username=typing.Optional[str]) |
| - description: The max number of records to receive. Setting this will result in |
| a bounded PCollection. |
| name: max_num_records |
| nullable: true |
| type: int64 |
| - description: The maximum time for this source to read messages. Setting this will |
| result in a bounded PCollection. |
| name: max_read_time_seconds |
| nullable: true |
| type: int64 |
| identifier: beam:schematransform:org.apache.beam:mqtt_read:v1 |
| name: ReadFromMqtt |
| - default_service: sdks:java:io:messaging-expansion-service:shadowJar |
| description: 'Publishes messages to an MQTT broker. Expects an input PCollection |
| of rows with a single `bytes` field, each of which is published as one MQTT message. |
| |
| |
| Works with both bounded (batch) and unbounded (streaming) input PCollections.' |
| destinations: |
| python: apache_beam/io |
| fields: |
| - description: Configuration options to set up the MQTT connection. |
| name: connection_configuration |
| nullable: false |
| type: Row(client_id=typing.Optional[str], password=typing.Optional[str], server_uri=<class |
| 'str'>, topic=typing.Optional[str], username=typing.Optional[str]) |
| - description: Whether or not the publish message should be retained by the messaging |
| engine. When a subscriber connects, it gets the latest retained message. Defaults |
| to `False`, which will clear the retained message from the server. |
| name: retained |
| nullable: true |
| type: boolean |
| identifier: beam:schematransform:org.apache.beam:mqtt_write:v1 |
| name: WriteToMqtt |
