blob: 1f8634f263ea1b3500b8b54e047e81841a823b98 [file]
import logging
import apache_beam as beam
from framework import prism_options, dataflow_options
from framework import periodic_source
from framework import dump_to_log
logging.basicConfig(level=logging.INFO)
with beam.Pipeline(options=prism_options) as p:
_ = (
p
| periodic_source
| beam.BatchElements(min_batch_size=5, max_batch_size=100, max_batch_duration_secs=4)
| dump_to_log
)