| # | |
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| # This file enumerates the various IOs that are available by default as | |
| # top-level transforms in Beam's YAML. | |
| # | |
| # Note that there may be redundant implementations. In these cases the specs | |
| # should be kept in sync. | |
| # TODO(yaml): See if this can be enforced programmatically. | |
| # BigQuery | |
| - type: renaming | |
| transforms: | |
| 'ReadFromBigQuery': 'ReadFromBigQuery' | |
| 'WriteToBigQuery': 'WriteToBigQuery' | |
| config: | |
| mappings: | |
| 'ReadFromBigQuery': | |
| query: 'query' | |
| table: 'table_spec' | |
| fields: 'selected_fields' | |
| row_restriction: 'row_restriction' | |
| 'WriteToBigQuery': | |
| table: 'table' | |
| create_disposition: 'create_disposition' | |
| write_disposition: 'write_disposition' | |
| error_handling: 'error_handling' | |
| # TODO(https://github.com/apache/beam/issues/30058): Required until autosharding support is fixed | |
| num_streams: 'num_streams' | |
| # Storage Write API CDC (row_mutation_info + record input schema) | |
| use_cdc_writes: 'use_cdc_writes' | |
| primary_key: 'primary_key' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromBigQuery': 'beam:schematransform:org.apache.beam:bigquery_storage_read:v1' | |
| 'WriteToBigQuery': 'beam:schematransform:org.apache.beam:bigquery_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar' | |
| managed_replacement: | |
| # Following transforms may be replaced with equivalent managed transforms, | |
| # if the pipelines 'updateCompatibilityBeamVersion' match the provided | |
| # version. | |
| 'ReadFromBigQuery': '2.69.0' | |
| 'WriteToBigQuery': '2.69.0' | |
| # BigQuery CDC (Storage Write API row mutations) | |
| # Input rows must wrap the table record: | |
| # {record: {...}, row_mutation_info: {mutation_type, change_sequence_number}} | |
| # mutation_type is UPSERT or DELETE (see RowMutationInformation.MutationType). | |
| - type: renaming | |
| transforms: | |
| 'WriteToBigQueryCDC': 'WriteToBigQueryCDC' | |
| config: | |
| mappings: | |
| 'WriteToBigQueryCDC': | |
| table: 'table' | |
| primary_key: 'primary_key' | |
| create_disposition: 'create_disposition' | |
| write_disposition: 'write_disposition' | |
| error_handling: 'error_handling' | |
| # TODO(https://github.com/apache/beam/issues/30058): Required until autosharding support is fixed | |
| num_streams: 'num_streams' | |
| use_at_least_once_semantics: 'use_at_least_once_semantics' | |
| triggering_frequency_seconds: 'triggering_frequency_seconds' | |
| auto_sharding: 'auto_sharding' | |
| defaults: | |
| 'WriteToBigQueryCDC': | |
| use_cdc_writes: true | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| # Route directly to Storage Write API. Managed bigquery_write:v1 may choose | |
| # file loads for bounded pipelines without error_handling, which cannot do CDC. | |
| 'WriteToBigQueryCDC': 'beam:schematransform:org.apache.beam:bigquery_storage_write:v2' | |
| config: | |
| gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar' | |
| # Kafka | |
| - type: renaming | |
| transforms: | |
| 'ReadFromKafka': 'ReadFromKafka' | |
| 'WriteToKafka': 'WriteToKafka' | |
| config: | |
| mappings: | |
| 'ReadFromKafka': | |
| 'schema': 'schema' | |
| 'consumer_config': 'consumer_config_updates' | |
| 'format': 'format' | |
| 'topic': 'topic' | |
| 'bootstrap_servers': 'bootstrap_servers' | |
| 'confluent_schema_registry_url': 'confluent_schema_registry_url' | |
| 'confluent_schema_registry_subject': 'confluent_schema_registry_subject' | |
| 'auto_offset_reset_config': 'auto_offset_reset_config' | |
| 'error_handling': 'error_handling' | |
| 'file_descriptor_path': 'file_descriptor_path' | |
| 'message_name': 'message_name' | |
| 'max_read_time_seconds': 'max_read_time_seconds' | |
| 'with_gcp_adc': 'with_gcp_adc' | |
| 'num_partitions': 'num_partitions' | |
| 'WriteToKafka': | |
| 'format': 'format' | |
| 'topic': 'topic' | |
| 'bootstrap_servers': 'bootstrap_servers' | |
| 'producer_config_updates': 'producer_config_updates' | |
| 'error_handling': 'error_handling' | |
| 'file_descriptor_path': 'file_descriptor_path' | |
| 'message_name': 'message_name' | |
| 'schema': 'schema' | |
| 'with_gcp_adc': 'with_gcp_adc' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromKafka': 'beam:schematransform:org.apache.beam:kafka_read:v1' | |
| 'WriteToKafka': 'beam:schematransform:org.apache.beam:kafka_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:expansion-service:shadowJar' | |
| managed_replacement: | |
| # Following transforms may be replaced with equivalent managed transforms, | |
| # if the pipelines 'updateCompatibilityBeamVersion' match the provided | |
| # version. | |
| 'ReadFromKafka': '2.65.0' | |
| 'WriteToKafka': '2.65.0' | |
| # TODO(yaml): Tests are assuming python providers are before java ones, hence | |
| # the order below. This should be fixed in the future. | |
| # Python Providers | |
| - type: python | |
| transforms: | |
| 'ReadFromBigQuery': 'apache_beam.yaml.yaml_io.read_from_bigquery' | |
| # Disable until https://github.com/apache/beam/issues/28162 is resolved. | |
| # 'WriteToBigQuery': 'apache_beam.yaml.yaml_io.write_to_bigquery' | |
| 'ReadFromText': 'apache_beam.yaml.yaml_io.read_from_text' | |
| 'WriteToText': 'apache_beam.yaml.yaml_io.write_to_text' | |
| 'ReadFromPubSub': 'apache_beam.yaml.yaml_io.read_from_pubsub' | |
| 'WriteToPubSub': 'apache_beam.yaml.yaml_io.write_to_pubsub' | |
| 'ReadFromIceberg': 'apache_beam.yaml.yaml_io.read_from_iceberg' | |
| 'WriteToIceberg': 'apache_beam.yaml.yaml_io.write_to_iceberg' | |
| 'ReadFromTFRecord': 'apache_beam.yaml.yaml_io.read_from_tfrecord' | |
| 'WriteToTFRecord': 'apache_beam.yaml.yaml_io.write_to_tfrecord' | |
| 'ReadFromMongoDB': 'apache_beam.yaml.yaml_io.read_from_mongodb' | |
| 'WriteToMongoDB': 'apache_beam.yaml.yaml_io.write_to_mongodb' | |
| 'ReadFromDelta': 'apache_beam.yaml.yaml_io.read_from_delta' | |
| 'ReadFromDeltaCDC': 'apache_beam.yaml.yaml_io.read_from_delta_cdc' | |
| 'DicomSearch': 'apache_beam.yaml.yaml_io.dicom_search' | |
| # General File Formats | |
| # Declared as a renaming transform to avoid exposing all | |
| # (implementation-specific) pandas arguments and aligning with possible Java | |
| # implementation. | |
| # Invoking these directly as a PyTransform is still an option for anyone wanting | |
| # to use these power-features in a language-dependent manner. | |
| - type: renaming | |
| transforms: | |
| 'ReadFromCsv': 'ReadFromCsv' | |
| 'WriteToCsv': 'WriteToCsv' | |
| 'ReadFromJson': 'ReadFromJson' | |
| 'WriteToJson': 'WriteToJson' | |
| 'ReadFromParquet': 'ReadFromParquet' | |
| 'WriteToParquet': 'WriteToParquet' | |
| 'ReadFromAvro': 'ReadFromAvro' | |
| 'WriteToAvro': 'WriteToAvro' | |
| config: | |
| mappings: | |
| 'ReadFromCsv': | |
| path: 'path' | |
| delimiter: 'sep' | |
| comment: 'comment' | |
| filename_column: 'filename_column' | |
| 'WriteToCsv': | |
| path: 'path' | |
| delimiter: 'sep' | |
| 'ReadFromJson': | |
| path: 'path' | |
| 'WriteToJson': | |
| path: 'path' | |
| 'ReadFromParquet': | |
| path: 'file_pattern' | |
| 'WriteToParquet': | |
| path: 'file_path_prefix' | |
| 'ReadFromAvro': | |
| path: 'file_pattern' | |
| 'WriteToAvro': | |
| path: 'file_path_prefix' | |
| defaults: | |
| 'ReadFromParquet': | |
| as_rows: True | |
| 'ReadFromAvro': | |
| as_rows: True | |
| underlying_provider: | |
| type: python | |
| transforms: | |
| 'ReadFromCsv': 'apache_beam.io.ReadFromCsv' | |
| 'WriteToCsv': 'apache_beam.io.WriteToCsv' | |
| 'ReadFromJson': 'apache_beam.io.ReadFromJson' | |
| 'WriteToJson': 'apache_beam.io.WriteToJson' | |
| 'ReadFromParquet': 'apache_beam.io.ReadFromParquet' | |
| 'WriteToParquet': 'apache_beam.io.WriteToParquet' | |
| 'ReadFromAvro': 'apache_beam.io.ReadFromAvro' | |
| 'WriteToAvro': 'apache_beam.io.WriteToAvro' | |
| # BeamJar Providers | |
| - type: beamJar | |
| transforms: | |
| 'WriteToCsv': 'beam:schematransform:org.apache.beam:csv_write:v1' | |
| 'WriteToJson': 'beam:schematransform:org.apache.beam:json_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:extensions:schemaio-expansion-service:shadowJar' | |
| # JMS | |
| - type: renaming | |
| transforms: | |
| 'ReadFromJms': 'ReadFromJms' | |
| 'WriteToJms': 'WriteToJms' | |
| config: | |
| mappings: | |
| 'ReadFromJms': | |
| connection_configuration: 'connection_configuration' | |
| queue: 'queue' | |
| topic: 'topic' | |
| max_num_records: 'max_num_records' | |
| max_read_time_seconds: 'max_read_time_seconds' | |
| close_timeout_seconds: 'close_timeout_seconds' | |
| acknowledge_mode: 'acknowledge_mode' | |
| individual_acknowledge_mode_code: 'individual_acknowledge_mode_code' | |
| 'WriteToJms': | |
| connection_configuration: 'connection_configuration' | |
| queue: 'queue' | |
| topic: 'topic' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromJms': 'beam:schematransform:org.apache.beam:jms_read:v1' | |
| 'WriteToJms': 'beam:schematransform:org.apache.beam:jms_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:messaging-expansion-service:shadowJar' | |
| - type: renaming | |
| transforms: | |
| 'ReadFromIbmMQ': 'ReadFromIbmMQ' | |
| 'WriteToIbmMQ': 'WriteToIbmMQ' | |
| config: | |
| mappings: | |
| 'ReadFromIbmMQ': | |
| connection_configuration: 'connection_configuration' | |
| queue: 'queue' | |
| topic: 'topic' | |
| max_num_records: 'max_num_records' | |
| max_read_time_seconds: 'max_read_time_seconds' | |
| close_timeout_seconds: 'close_timeout_seconds' | |
| acknowledge_mode: 'acknowledge_mode' | |
| individual_acknowledge_mode_code: 'individual_acknowledge_mode_code' | |
| 'WriteToIbmMQ': | |
| connection_configuration: 'connection_configuration' | |
| queue: 'queue' | |
| topic: 'topic' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromIbmMQ': 'beam:schematransform:org.apache.beam:jms_read:v1' | |
| 'WriteToIbmMQ': 'beam:schematransform:org.apache.beam:jms_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:messaging-expansion-service:shadowJar' | |
| classpath: | |
| - 'com.ibm.mq:com.ibm.mq.allclient:9.3.0.25' | |
| - 'org.json:json:20251224' | |
| # Debezium | |
| - type: renaming | |
| transforms: | |
| 'ReadFromDebezium': 'ReadFromDebezium' | |
| config: | |
| mappings: | |
| 'ReadFromDebezium': | |
| connector: 'database' | |
| username: 'username' | |
| password: 'password' | |
| host: 'host' | |
| port: 'port' | |
| table: 'table' | |
| connection_properties: 'debezium_connection_properties' | |
| max_number_of_records: 'max_number_of_records' | |
| max_time_to_run: 'max_time_to_run' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromDebezium': 'beam:schematransform:org.apache.beam:debezium_read:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:debezium:expansion-service:shadowJar' | |
| # Kinesis | |
| - type: renaming | |
| transforms: | |
| 'ReadFromKinesis': 'ReadFromKinesis' | |
| 'WriteToKinesis': 'WriteToKinesis' | |
| config: | |
| mappings: | |
| 'ReadFromKinesis': | |
| stream_name: 'stream_name' | |
| aws_access_key: 'aws_access_key' | |
| aws_secret_key: 'aws_secret_key' | |
| region: 'region' | |
| service_endpoint: 'service_endpoint' | |
| verify_certificate: 'verify_certificate' | |
| max_num_records: 'max_num_records' | |
| max_read_time: 'max_read_time' | |
| initial_position_in_stream: 'initial_position_in_stream' | |
| initial_timestamp_in_stream: 'initial_timestamp_in_stream' | |
| request_records_limit: 'request_records_limit' | |
| up_to_date_threshold: 'up_to_date_threshold' | |
| max_capacity_per_shard: 'max_capacity_per_shard' | |
| watermark_policy: 'watermark_policy' | |
| watermark_idle_duration_threshold: 'watermark_idle_duration_threshold' | |
| rate_limit: 'rate_limit' | |
| 'WriteToKinesis': | |
| stream_name: 'stream_name' | |
| aws_access_key: 'aws_access_key' | |
| aws_secret_key: 'aws_secret_key' | |
| region: 'region' | |
| partition_key: 'partition_key' | |
| service_endpoint: 'service_endpoint' | |
| verify_certificate: 'verify_certificate' | |
| aggregation_enabled: 'aggregation_enabled' | |
| aggregation_max_bytes: 'aggregation_max_bytes' | |
| aggregation_max_buffered_time: 'aggregation_max_buffered_time' | |
| aggregation_shard_refresh_interval: 'aggregation_shard_refresh_interval' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromKinesis': 'beam:schematransform:org.apache.beam:kinesis_read:v1' | |
| 'WriteToKinesis': 'beam:schematransform:org.apache.beam:kinesis_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:amazon-web-services2:expansion-service:shadowJar' | |
| # Snowflake | |
| - type: renaming | |
| transforms: | |
| 'ReadFromSnowflake': 'ReadFromSnowflake' | |
| 'WriteToSnowflake': 'WriteToSnowflake' | |
| config: | |
| mappings: | |
| 'ReadFromSnowflake': | |
| server_name: 'server_name' | |
| username: 'username' | |
| password: 'password' | |
| oauth_token: 'oauth_token' | |
| private_key: 'private_key' | |
| private_key_passphrase: 'private_key_passphrase' | |
| database: 'database' | |
| snowflake_schema: 'snowflake_schema' | |
| warehouse: 'warehouse' | |
| role: 'role' | |
| table: 'table' | |
| query: 'query' | |
| staging_bucket_name: 'staging_bucket_name' | |
| storage_integration_name: 'storage_integration_name' | |
| schema: 'schema' | |
| quotation_mark: 'quotation_mark' | |
| 'WriteToSnowflake': | |
| server_name: 'server_name' | |
| username: 'username' | |
| password: 'password' | |
| oauth_token: 'oauth_token' | |
| private_key: 'private_key' | |
| private_key_passphrase: 'private_key_passphrase' | |
| database: 'database' | |
| schema: 'schema' | |
| warehouse: 'warehouse' | |
| role: 'role' | |
| table: 'table' | |
| snow_pipe: 'snow_pipe' | |
| staging_bucket_name: 'staging_bucket_name' | |
| storage_integration_name: 'storage_integration_name' | |
| create_disposition: 'create_disposition' | |
| write_disposition: 'write_disposition' | |
| quotation_mark: 'quotation_mark' | |
| flush_row_limit: 'flush_row_limit' | |
| flush_time_limit_millis: 'flush_time_limit_millis' | |
| shards_number: 'shards_number' | |
| debug_mode: 'debug_mode' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromSnowflake': | |
| 'beam:schematransform:org.apache.beam:snowflake_read:v1' | |
| 'WriteToSnowflake': | |
| 'beam:schematransform:org.apache.beam:snowflake_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:snowflake:expansion-service:shadowJar' | |
| # Databases | |
| - type: renaming | |
| transforms: | |
| 'ReadFromJdbc': 'ReadFromJdbc' | |
| 'WriteToJdbc': 'WriteToJdbc' | |
| 'ReadFromMySql': 'ReadFromMySql' | |
| 'WriteToMySql': 'WriteToMySql' | |
| 'ReadFromPostgres': 'ReadFromPostgres' | |
| 'WriteToPostgres': 'WriteToPostgres' | |
| 'ReadFromOracle': 'ReadFromOracle' | |
| 'WriteToOracle': 'WriteToOracle' | |
| 'ReadFromSqlServer': 'ReadFromSqlServer' | |
| 'WriteToSqlServer': 'WriteToSqlServer' | |
| config: | |
| mappings: | |
| 'ReadFromJdbc': | |
| url: 'jdbc_url' | |
| connection_init_sql: 'connection_init_sql' | |
| connection_properties: 'connection_properties' | |
| disable_auto_commit: 'disable_auto_commit' | |
| driver_class_name: 'driver_class_name' | |
| driver_jars: 'driver_jars' | |
| fetch_size: 'fetch_size' | |
| output_parallelization: 'output_parallelization' | |
| password: 'password' | |
| query: 'read_query' | |
| table: 'location' | |
| partition_column : 'partition_column' | |
| num_partitions: 'num_partitions' | |
| secret_manager: 'secret_manager' | |
| type: 'jdbc_type' | |
| username: 'username' | |
| 'WriteToJdbc': | |
| url: 'jdbc_url' | |
| auto_sharding: 'autosharding' | |
| connection_init_sql: 'connection_init_sql' | |
| connection_properties: 'connection_properties' | |
| driver_class_name: 'driver_class_name' | |
| driver_jars: 'driver_jars' | |
| password: 'password' | |
| table: 'location' | |
| batch_size: 'batch_size' | |
| secret_manager: 'secret_manager' | |
| type: 'jdbc_type' | |
| username: 'username' | |
| query: 'write_statement' | |
| 'ReadFromMySql': 'ReadFromJdbc' | |
| 'WriteToMySql': 'WriteToJdbc' | |
| 'ReadFromPostgres': 'ReadFromJdbc' | |
| 'WriteToPostgres': 'WriteToJdbc' | |
| 'ReadFromOracle': 'ReadFromJdbc' | |
| 'WriteToOracle': 'WriteToJdbc' | |
| 'ReadFromSqlServer': 'ReadFromJdbc' | |
| 'WriteToSqlServer': 'WriteToJdbc' | |
| defaults: | |
| 'ReadFromMySql': | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'WriteToMySql': | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'ReadFromPostgres': | |
| connection_init_sql: [] | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'WriteToPostgres': | |
| connection_init_sql: [] | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'ReadFromOracle': | |
| connection_init_sql: [] | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'WriteToOracle': | |
| connection_init_sql: [] | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'ReadFromSqlServer': | |
| connection_init_sql: [] | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| 'WriteToSqlServer': | |
| connection_init_sql: [] | |
| driver_class_name: '' | |
| driver_jars: '' | |
| jdbc_type: '' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromJdbc': 'beam:schematransform:org.apache.beam:jdbc_read:v1' | |
| 'ReadFromMySql': 'beam:schematransform:org.apache.beam:mysql_read:v1' | |
| 'ReadFromPostgres': 'beam:schematransform:org.apache.beam:postgres_read:v1' | |
| 'ReadFromOracle': 'beam:schematransform:org.apache.beam:oracle_read:v1' | |
| 'ReadFromSqlServer': 'beam:schematransform:org.apache.beam:sql_server_read:v1' | |
| 'WriteToJdbc': 'beam:schematransform:org.apache.beam:jdbc_write:v1' | |
| 'WriteToMySql': 'beam:schematransform:org.apache.beam:mysql_write:v1' | |
| 'WriteToPostgres': 'beam:schematransform:org.apache.beam:postgres_write:v1' | |
| 'WriteToOracle': 'beam:schematransform:org.apache.beam:oracle_write:v1' | |
| 'WriteToSqlServer': 'beam:schematransform:org.apache.beam:sql_server_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:extensions:schemaio-expansion-service:shadowJar' | |
| managed_replacement: | |
| # Following transforms may be replaced with equivalent managed transforms, | |
| # if the pipelines 'updateCompatibilityBeamVersion' match the provided | |
| # version. | |
| 'ReadFromPostgres': '2.73.0' | |
| 'WriteToPostgres': '2.73.0' | |
| 'ReadFromMySql': '2.73.0' | |
| 'WriteToMySql': '2.73.0' | |
| 'ReadFromSqlServer': '2.73.0' | |
| 'WriteToSqlServer': '2.73.0' | |
| # Spanner | |
| - type: renaming | |
| transforms: | |
| 'ReadFromSpanner': 'ReadFromSpanner' | |
| 'WriteToSpanner': 'WriteToSpanner' | |
| config: | |
| mappings: | |
| 'ReadFromSpanner': | |
| project: 'project_id' | |
| instance: 'instance_id' | |
| database: 'database_id' | |
| table: 'table_id' | |
| query: 'query' | |
| columns: 'columns' | |
| index: 'index' | |
| batching: 'batching' | |
| error_handling: 'error_handling' | |
| 'WriteToSpanner': | |
| project: 'project_id' | |
| instance: 'instance_id' | |
| database: 'database_id' | |
| table: 'table_id' | |
| error_handling: 'error_handling' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromSpanner': 'beam:schematransform:org.apache.beam:spanner_read:v1' | |
| 'WriteToSpanner': 'beam:schematransform:org.apache.beam:spanner_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar' | |
| # Spanner CDC | |
| - type: renaming | |
| transforms: | |
| 'ReadFromSpannerCDC': 'ReadFromSpannerCDC' | |
| config: | |
| mappings: | |
| 'ReadFromSpannerCDC': | |
| project: 'project_id' | |
| instance: 'instance_id' | |
| database: 'database_id' | |
| table: 'table' | |
| change_stream: 'change_stream_name' | |
| start_at: 'start_at_timestamp' | |
| end_at: 'end_at_timestamp' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromSpannerCDC': 'beam:schematransform:org.apache.beam:spanner_cdc_read:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar' | |
| # Firestore | |
| - type: renaming | |
| transforms: | |
| 'ReadFromFirestore': 'ReadFromFirestore' | |
| 'WriteToFirestore': 'WriteToFirestore' | |
| config: | |
| mappings: | |
| 'ReadFromFirestore': | |
| project: 'project_id' | |
| database: 'database_id' | |
| collection: 'collection_id' | |
| schema: 'schema' | |
| error_handling: 'error_handling' | |
| 'WriteToFirestore': | |
| project: 'project_id' | |
| database: 'database_id' | |
| collection: 'collection_id' | |
| document_id_field: 'document_id_field' | |
| error_handling: 'error_handling' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromFirestore': 'beam:schematransform:org.apache.beam:firestore_read:v1' | |
| 'WriteToFirestore': 'beam:schematransform:org.apache.beam:firestore_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar' | |
| # TFRecord | |
| - type: renaming | |
| transforms: | |
| 'ReadFromTFRecord': 'ReadFromTFRecord' | |
| 'WriteToTFRecord': 'WriteToTFRecord' | |
| config: | |
| mappings: | |
| 'ReadFromTFRecord': | |
| file_pattern: 'file_pattern' | |
| compression_type: 'compression' | |
| validate: 'validate' | |
| 'WriteToTFRecord': | |
| file_path_prefix: 'output_prefix' | |
| shard_name_template: 'shard_template' | |
| file_name_suffix: 'filename_suffix' | |
| num_shards: 'num_shards' | |
| compression_type: 'compression' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromTFRecord': 'beam:schematransform:org.apache.beam:tfrecord_read:v1' | |
| 'WriteToTFRecord': 'beam:schematransform:org.apache.beam:tfrecord_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:expansion-service:shadowJar' | |
| #BigTable | |
| - type: renaming | |
| transforms: | |
| 'ReadFromBigTable': 'ReadFromBigTable' | |
| 'WriteToBigTable': 'WriteToBigTable' | |
| config: | |
| mappings: | |
| #Temp removing read from bigTable IO | |
| 'ReadFromBigTable': | |
| project: 'project_id' | |
| instance: 'instance_id' | |
| table: 'table_id' | |
| flatten: "flatten" | |
| 'WriteToBigTable': | |
| project: 'project_id' | |
| instance: 'instance_id' | |
| table: 'table_id' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromBigTable': 'beam:schematransform:org.apache.beam:bigtable_read:v1' | |
| 'WriteToBigTable': 'beam:schematransform:org.apache.beam:bigtable_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar' | |
| #IcebergCDC | |
| - type: renaming | |
| transforms: | |
| 'ReadFromIcebergCDC': 'ReadFromIcebergCDC' | |
| config: | |
| mappings: | |
| 'ReadFromIcebergCDC': | |
| table: 'table' | |
| catalog_name: 'catalog_name' | |
| catalog_properties: 'catalog_properties' | |
| config_properties: 'config_properties' | |
| drop: 'drop' | |
| filter: 'filter' | |
| from_snapshot: 'from_snapshot' | |
| from_timestamp: 'from_timestamp' | |
| keep: 'keep' | |
| poll_interval_seconds: 'poll_interval_seconds' | |
| starting_strategy: 'starting_strategy' | |
| streaming: 'streaming' | |
| to_snapshot: 'to_snapshot' | |
| to_timestamp: 'to_timestamp' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromIcebergCDC': 'beam:schematransform:org.apache.beam:iceberg_cdc_read:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:expansion-service:shadowJar' | |
| #IcebergAddFiles | |
| - type: renaming | |
| transforms: | |
| 'IcebergAddFiles': 'IcebergAddFiles' | |
| config: | |
| mappings: | |
| 'IcebergAddFiles': | |
| table: 'table' | |
| catalog_properties: 'catalog_properties' | |
| config_properties: 'config_properties' | |
| triggering_frequency_seconds: 'triggering_frequency_seconds' | |
| manifest_file_size: 'manifest_file_size' | |
| location_prefix: 'location_prefix' | |
| partition_fields: 'partition_fields' | |
| table_properties: 'table_properties' | |
| sort_fields: 'sort_fields' | |
| schema_evolution_options: 'schema_evolution_options' | |
| required_columns: 'required_columns' | |
| incompatible_schema_handling: 'incompatible_schema_handling' | |
| unverifiable_file_handling: 'unverifiable_file_handling' | |
| dry_run: 'dry_run' | |
| error_handling: 'error_handling' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'IcebergAddFiles': 'beam:schematransform:iceberg_add_files:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:expansion-service:shadowJar' | |
| #Datadog | |
| - type: renaming | |
| transforms: | |
| 'WriteToDatadog': 'WriteToDatadog' | |
| config: | |
| mappings: | |
| 'WriteToDatadog': | |
| 'url': 'url' | |
| 'api_key': 'api_key' | |
| 'min_batch_count': 'min_batch_count' | |
| 'batch_count': 'batch_count' | |
| 'max_buffer_size': 'max_buffer_size' | |
| 'parallelism': 'parallelism' | |
| 'error_handling': 'error_handling' | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'WriteToDatadog': 'beam:schematransform:org.apache.beam:datadog_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:expansion-service:shadowJar' | |
| #MongoDB | |
| - type: renaming | |
| transforms: | |
| 'ReadFromMongoDB': 'ReadFromMongoDB' | |
| 'WriteToMongoDB': 'WriteToMongoDB' | |
| config: | |
| mappings: | |
| 'ReadFromMongoDB': | |
| uri: "uri" | |
| database: "database" | |
| collection: "collection" | |
| schema: "schema" | |
| filter: "filter" | |
| error_handling: "error_handling" | |
| 'WriteToMongoDB': | |
| uri: "uri" | |
| database: "database" | |
| collection: "collection" | |
| batch_size: "batch_size" | |
| error_handling: "error_handling" | |
| underlying_provider: | |
| type: beamJar | |
| transforms: | |
| 'ReadFromMongoDB': 'beam:schematransform:org.apache.beam:mongodb_read:v1' | |
| 'WriteToMongoDB': 'beam:schematransform:org.apache.beam:mongodb_write:v1' | |
| config: | |
| gradle_target: 'sdks:java:io:expansion-service:shadowJar' | |