blob: ef265be8015df526ffc6fbdd57195231b27118c4 [file]
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
import json
import os
import time
import unittest
from datetime import datetime
from unittest.mock import patch
from zoneinfo import ZoneInfo
from py4j.protocol import Py4JJavaError
import pyflink.dataframe as pf
from pyflink.common import JobStatus
from pyflink.dataframe import DataType
from pyflink.dataframe.io import _build_filesystem_options
from pyflink.datastream import RuntimeExecutionMode
from pyflink.java_gateway import get_gateway
from pyflink.table import EnvironmentSettings, StreamTableEnvironment, TableDescriptor, TableResult
from pyflink.testing.test_case_utils import PyFlinkDataFrameUTTestCase
class FilesystemOptionsTests(unittest.TestCase):
def test_format_options_are_namespaced_without_mutating_input(self):
format_options = {
"ignore-parse-errors": "true",
"json.timestamp-format.standard": "ISO-8601",
}
original = dict(format_options)
options = _build_filesystem_options(
"/input", "json", extra_format_options=format_options
)
self.assertEqual(options, {
"path": "/input",
"format": "json",
"json.ignore-parse-errors": "true",
"json.timestamp-format.standard": "ISO-8601",
})
self.assertEqual(format_options, original)
def test_format_options_cannot_override_filesystem_options(self):
options = _build_filesystem_options(
"/input", "json", extra_format_options={"path": "/other", "format": "parquet"}
)
self.assertEqual(options["path"], "/input")
self.assertEqual(options["format"], "json")
def test_rejects_invalid_format_options(self):
cases = [
([], TypeError, "options must be a dict"),
({1: "true"}, TypeError, "option keys must be strings"),
({"": "true"}, ValueError, "option keys must not be empty"),
({"ignore-parse-errors": True}, TypeError, "must have a string value"),
({"connector": "blackhole"}, ValueError, "connector argument"),
({"ignore-parse-errors": "true", "json.ignore-parse-errors": "false"},
ValueError, "duplicate format option"),
]
for options, error, message in cases:
with self.subTest(options=options):
with self.assertRaisesRegex(error, message):
_build_filesystem_options(
"/input", "json", extra_format_options=options)
def test_connector_options_are_validated_without_mutating_input(self):
options = {"source.monitor-interval": "1s", "source.report-statistics": "NONE"}
original = dict(options)
actual = _build_filesystem_options(
"/input", "json", connector_parameters={"source.monitor-interval": "1s"},
extra_connector_options=options,
)
self.assertEqual(actual["source.report-statistics"], "NONE")
self.assertEqual(options, original)
for invalid in ({"path": "/other"}, {"format": "parquet"},
{"connector": "blackhole"}, {"json.ignore-parse-errors": "true"}):
with self.subTest(options=invalid):
with self.assertRaises(ValueError):
_build_filesystem_options(
"/input", "json", extra_connector_options=invalid)
for invalid in ([], {1: "value"}, {"source.monitor-interval": 1}):
with self.subTest(options=invalid):
with self.assertRaises(TypeError):
_build_filesystem_options(
"/input", "json", extra_connector_options=invalid)
def test_conflicting_explicit_options_are_rejected(self):
with self.assertRaisesRegex(ValueError, "conflicting values.*source.monitor-interval"):
_build_filesystem_options(
"/input", "json", connector_parameters={"source.monitor-interval": "1s"},
extra_connector_options={"source.monitor-interval": "2s"},
)
for prefix in ("", "parquet."):
with self.subTest(prefix=prefix):
options = {prefix + "compression": "GZIP"}
with self.assertRaisesRegex(ValueError, "conflicting values.*parquet.compression"):
_build_filesystem_options(
"/input", "parquet", extra_format_options=options,
format_parameters={"compression": "SNAPPY"},
)
actual = _build_filesystem_options(
"/input", "parquet", extra_format_options=options,
format_parameters={"compression": "GZIP"},
)
self.assertEqual(actual["parquet.compression"], "GZIP")
def test_readers_reject_invalid_paths_and_option_types(self):
cases = [
({"path": None}, TypeError, "path must be a string"),
({"path": ""}, ValueError, "path must not be empty"),
({"monitor_interval": 60}, TypeError, "must have a string value"),
({"path_regex_pattern": 1}, TypeError, "must have a string value"),
]
for reader in (pf.read_json, pf.read_parquet):
for overrides, error, message in cases:
with self.subTest(reader=reader.__name__, overrides=overrides):
arguments = {"path": "/input", "schema": {"id": DataType.int64()}}
arguments.update(overrides)
with self.assertRaisesRegex(error, message):
reader(**arguments)
def test_writers_reject_invalid_arguments_before_execution(self):
dataframe = pf.DataFrame(None)
cases = [
({"path": None}, TypeError, "path must be a string"),
({"path": ""}, ValueError, "path must not be empty"),
({"mode": None}, TypeError, "mode must be a string"),
({"mode": "ignore"}, ValueError, "mode must be 'append' or 'overwrite'"),
({"rolling_policy_file_size": 128}, TypeError, "must have a string value"),
({"rolling_policy_inactivity_interval": 1}, TypeError, "must have a string value"),
({"rolling_policy_check_interval": 5}, TypeError, "must have a string value"),
({"partition_commit_delay": 1}, TypeError, "must have a string value"),
({"sink_parallelism": True}, TypeError, "sink_parallelism must be an int"),
({"sink_parallelism": "1"}, TypeError, "sink_parallelism must be an int"),
({"sink_shuffle_by_partition": "true"}, TypeError,
"sink_shuffle_by_partition must be a bool"),
({"auto_compaction": 1}, TypeError, "auto_compaction must be a bool"),
({"compaction_file_size": 128}, TypeError, "must have a string value"),
({"partition_by": [1]}, TypeError, "partition_by must be a string"),
({"partition_by": []}, ValueError, "partition_by must not be empty"),
({"partition_by": ""}, ValueError, "column names must not be empty"),
({"partition_by": ["id", "id"]}, ValueError, "must not be duplicated"),
]
for writer in (dataframe.write_json, dataframe.write_parquet):
for overrides, error, message in cases:
with self.subTest(writer=writer.__name__, overrides=overrides):
arguments = {"path": "/output"}
arguments.update(overrides)
with self.assertRaisesRegex(error, message):
writer(**arguments)
with self.assertRaisesRegex(TypeError, "must have a string value"):
dataframe.write_parquet("/output", compression=1)
with self.assertRaisesRegex(TypeError, "utc_timezone must be a bool"):
dataframe.write_parquet("/output", utc_timezone="true")
with self.assertRaisesRegex(TypeError, "ignore_parse_errors must be a bool"):
pf.read_json("/input", schema={"id": DataType.int64()}, ignore_parse_errors=1)
for parameter in ("ignore_null_fields", "decimal_as_plain_number"):
with self.subTest(parameter=parameter):
with self.assertRaisesRegex(TypeError, parameter + " must be a bool"):
dataframe.write_json("/output", **{parameter: "true"})
class FilesystemIOTests(PyFlinkDataFrameUTTestCase):
_SCHEMA = {"id": DataType.int64(), "name": DataType.string()}
def test_readers_declare_computed_columns_and_watermarks(self):
for reader in (pf.read_json, pf.read_parquet):
with self.subTest(reader=reader.__name__):
dataframe = reader(
"/input", schema={"ts_millis": DataType.int64()},
computed_columns={"ts": "TO_TIMESTAMP_LTZ(ts_millis, 3)"},
watermark=("ts", "ts - INTERVAL '5' SECOND"),
)
resolved = dataframe.to_table().get_resolved_schema()
self.assertEqual(list(resolved.get_column_names()), ["ts_millis", "ts"])
self.assertEqual(len(resolved.get_watermark_specs()), 1)
physical = reader(
"/input", schema={"ts": DataType.timestamp(3)},
watermark=("ts", "ts - INTERVAL '5' SECOND"),
)
self.assertEqual(len(physical.to_table().get_resolved_schema()
.get_watermark_specs()), 1)
def test_readers_forward_connector_and_format_parameters(self):
cases = [
(pf.read_json, {"ignore_parse_errors": True, "fail_on_missing_field": False,
"timestamp_format": "ISO-8601"},
{"json.ignore-parse-errors": "true", "json.fail-on-missing-field": "false",
"json.timestamp-format.standard": "ISO-8601"}),
(pf.read_parquet, {"utc_timezone": True}, {"parquet.utc-timezone": "true"}),
]
for reader, parameters, expected in cases:
with self.subTest(reader=reader.__name__):
with patch.object(self.t_env, "from_descriptor",
wraps=self.t_env.from_descriptor) as from_descriptor:
reader("/input", schema=self._SCHEMA, **parameters,
connector_options={"source.report-statistics": "NONE"})
options = dict(from_descriptor.call_args.args[0].get_options())
self.assertEqual(options["source.report-statistics"], "NONE")
for key, value in expected.items():
self.assertEqual(options[key], value)
def test_invalid_partition_columns_are_rejected_by_the_table_api(self):
for reader in (pf.read_json, pf.read_parquet):
with self.subTest(reader=reader.__name__):
with self.assertRaisesRegex(ValueError, "missing"):
reader("/input", schema=self._SCHEMA, partition_by="missing")
dataframe = pf.from_records([(1,)], schema=["id"])
for writer in (dataframe.write_json, dataframe.write_parquet):
with self.subTest(writer=writer.__name__):
with self.assertRaisesRegex(ValueError, "missing"):
writer("/output", partition_by="missing")
def test_writers_accept_raw_options_without_overriding_them_with_defaults(self):
dataframe = pf.from_records([(1,)], schema=["id"])
with patch.object(dataframe._table, "execute_insert") as execute_insert:
dataframe.write_parquet(
"/output", format_options={"compression": "GZIP"},
connector_options={"sink.rolling-policy.file-size": "64mb"},
)
options = dict(execute_insert.call_args.args[0].get_options())
self.assertEqual(options["parquet.compression"], "GZIP")
self.assertEqual(options["sink.rolling-policy.file-size"], "64mb")
def test_readers_build_filesystem_descriptors(self):
for file_format, reader in [("json", pf.read_json), ("parquet", pf.read_parquet)]:
for continuous in (False, True):
with self.subTest(file_format=file_format, continuous=continuous):
arguments = {}
expected = {"connector": "filesystem", "path": "/input",
"format": file_format}
if continuous:
arguments = {"monitor_interval": "60s", "path_regex_pattern": ".*data.*"}
expected.update({"source.monitor-interval": "60s",
"source.path.regex-pattern": ".*data.*"})
with patch.object(self.t_env, "from_descriptor",
wraps=self.t_env.from_descriptor) as from_descriptor:
dataframe = reader("/input", schema=self._SCHEMA, **arguments)
descriptor = from_descriptor.call_args.args[0]
self.assertEqual(dict(descriptor.get_options()), expected)
self.assert_dataframe_schema(dataframe, ["id", "name"])
def test_json_reader_forwards_format_options(self):
with patch.object(self.t_env, "from_descriptor",
wraps=self.t_env.from_descriptor) as from_descriptor:
pf.read_json("/input", schema=self._SCHEMA,
format_options={"ignore-parse-errors": "true"})
options = from_descriptor.call_args.args[0].get_options()
self.assertEqual(options.get("json.ignore-parse-errors"), "true")
def test_writers_build_descriptors_and_apply_write_mode(self):
dataframe = pf.from_records([(1, "a")], schema=["id", "name"])
for file_format in ("json", "parquet"):
writer = getattr(dataframe, "write_" + file_format)
for mode in (None, "append", "overwrite"):
with self.subTest(file_format=file_format, mode=mode):
arguments = {} if mode is None else {"mode": mode}
with patch.object(dataframe._table, "execute_insert") as execute_insert:
self.assertIsNone(writer("/output", **arguments))
descriptor = execute_insert.call_args.args[0]
expected = {
"connector": "filesystem", "path": "/output", "format": file_format,
}
self.assertEqual(dict(descriptor.get_options()), expected)
self.assertIsNone(descriptor.get_schema())
self.assertEqual(execute_insert.call_args.kwargs,
{"overwrite": mode == "overwrite"})
def test_writers_forward_compaction_and_shuffle_options(self):
dataframe = pf.from_records([(1, "a")], schema=["id", "name"])
for writer in (dataframe.write_json, dataframe.write_parquet):
for enabled in (False, True):
with self.subTest(writer=writer.__name__, enabled=enabled):
raw_options = {
"sink.shuffle-by-partition.enable": str(enabled).lower(),
"auto-compaction": str(enabled).lower(),
"compaction.file-size": "64mb",
}
with patch.object(dataframe._table, "execute_insert") as execute_insert:
writer("/output", partition_by="name",
sink_shuffle_by_partition=enabled, auto_compaction=enabled,
compaction_file_size="64mb", connector_options=raw_options)
options = dict(execute_insert.call_args.args[0].get_options())
for key, value in raw_options.items():
self.assertEqual(options[key], value)
self.assertNotIn("sink.rolling-policy.file-size", options)
def test_json_writer_forwards_encoding_options(self):
dataframe = pf.from_records([(1,)], schema=["id"])
for enabled in (False, True):
with self.subTest(enabled=enabled):
raw_options = {
"encode.ignore-null-fields": str(enabled).lower(),
"json.encode.decimal-as-plain-number": str(enabled).lower(),
}
with patch.object(dataframe._table, "execute_insert") as execute_insert:
dataframe.write_json(
"/output", ignore_null_fields=enabled, decimal_as_plain_number=enabled,
format_options=raw_options,
)
options = dict(execute_insert.call_args.args[0].get_options())
self.assertEqual(options["json.encode.ignore-null-fields"], str(enabled).lower())
self.assertEqual(options["json.encode.decimal-as-plain-number"],
str(enabled).lower())
def test_writers_reject_conflicting_convenience_options_before_execution(self):
dataframe = pf.from_records([(1,)], schema=["id"])
cases = [
{"sink_shuffle_by_partition": True,
"connector_options": {"sink.shuffle-by-partition.enable": "false"}},
{"auto_compaction": True, "connector_options": {"auto-compaction": "false"}},
{"compaction_file_size": "64mb",
"connector_options": {"compaction.file-size": "128mb"}},
]
for writer in (dataframe.write_json, dataframe.write_parquet):
for arguments in cases:
with self.subTest(writer=writer.__name__, arguments=arguments):
with patch.object(dataframe._table, "execute_insert") as execute_insert:
with self.assertRaisesRegex(ValueError, "conflicting values"):
writer("/output", **arguments)
execute_insert.assert_not_called()
for parameter, option in [("ignore_null_fields", "encode.ignore-null-fields"),
("decimal_as_plain_number", "encode.decimal-as-plain-number")]:
with self.subTest(parameter=parameter):
with patch.object(dataframe._table, "execute_insert") as execute_insert:
with self.assertRaisesRegex(ValueError, "conflicting values"):
dataframe.write_json(
"/output", **{parameter: True}, format_options={option: "false"})
execute_insert.assert_not_called()
def test_writers_forward_custom_sink_options_and_wait_locally(self):
dataframe = pf.from_records([(1,)], schema=["id"])
for file_format in ("json", "parquet"):
for target in ("local", "minicluster", "remote"):
with self.subTest(file_format=file_format, target=target):
self.t_env.get_config().set("execution.target", target)
arguments = ({"compression": "GZIP"} if file_format == "parquet" else
{"timestamp_format": "ISO-8601"})
with patch.object(dataframe._table, "execute_insert") as execute_insert:
getattr(dataframe, "write_" + file_format)(
"/output", mode="append", rolling_policy_file_size="64mb",
rolling_policy_rollover_interval="10min",
rolling_policy_inactivity_interval="2min", sink_parallelism=2,
rolling_policy_check_interval="1min",
partition_commit_trigger="partition-time", partition_commit_delay="1h",
partition_commit_policy_kind="success-file", **arguments)
options = dict(execute_insert.call_args.args[0].get_options())
self.assertEqual(options["sink.rolling-policy.file-size"], "64mb")
self.assertEqual(options["sink.rolling-policy.rollover-interval"], "10min")
self.assertEqual(options["sink.rolling-policy.inactivity-interval"], "2min")
self.assertEqual(options["sink.parallelism"], "2")
self.assertEqual(options["sink.rolling-policy.check-interval"], "1min")
self.assertEqual(options["sink.partition-commit.trigger"], "partition-time")
self.assertEqual(options["sink.partition-commit.delay"], "1h")
self.assertEqual(options["sink.partition-commit.policy.kind"], "success-file")
if file_format == "parquet":
self.assertEqual(options["parquet.compression"], "GZIP")
else:
self.assertEqual(options["json.timestamp-format.standard"], "ISO-8601")
if target in ("local", "minicluster"):
execute_insert.return_value.wait.assert_called_once_with()
else:
execute_insert.return_value.wait.assert_not_called()
class FilesystemIOITTests(PyFlinkDataFrameUTTestCase):
_SCHEMA = {"id": DataType.int64(), "name": DataType.string()}
def setUp(self):
super().setUp()
self.env.set_runtime_mode(RuntimeExecutionMode.BATCH)
self.env.set_parallelism(1)
self.t_env = StreamTableEnvironment.create(
self.env, environment_settings=EnvironmentSettings.in_batch_mode()
)
pf.set_table_environment(self.t_env)
def test_json_round_trip_overwrite_and_append(self):
self._check_round_trip_overwrite_and_append("json")
def test_json_partitioned_round_trip_and_overwrite(self):
self._check_partitioned_round_trip_and_overwrite("json")
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_partitioned_round_trip_and_overwrite(self):
self._check_partitioned_round_trip_and_overwrite("parquet")
def _check_partitioned_round_trip_and_overwrite(self, file_format):
output_path = os.path.join(self.tempdir, "partitioned-" + file_format)
reader = getattr(pf, "read_" + file_format)
original = pf.from_records([(1, "a"), (2, "b")], schema=["id", "name"])
getattr(original, "write_" + file_format)(output_path, partition_by="name")
self.assertTrue(os.path.isdir(os.path.join(output_path, "name=a")))
self.assertTrue(os.path.isdir(os.path.join(output_path, "name=b")))
rows = reader(output_path, schema=self._SCHEMA, partition_by="name").collect()
self.assertEqual(sorted(tuple(row) for row in rows), [(1, "a"), (2, "b")])
replacement = pf.from_records([(3, "a")], schema=["id", "name"])
getattr(replacement, "write_" + file_format)(
output_path, partition_by=["name"], mode="overwrite")
rows = reader(output_path, schema=self._SCHEMA, partition_by=["name"]).collect()
self.assertEqual(sorted(tuple(row) for row in rows), [(2, "b"), (3, "a")])
additional = pf.from_records([(4, "a/b=c")], schema=["id", "name"])
getattr(additional, "write_" + file_format)(
output_path, partition_by="name")
rows = reader(output_path, schema=self._SCHEMA, partition_by="name").collect()
self.assertEqual(sorted(tuple(row) for row in rows),
[(2, "b"), (3, "a"), (4, "a/b=c")])
default_path = os.path.join(self.tempdir, "default-partition-" + file_format)
default_values = pf.from_records([(5, None), (6, "")], schema=["id", "name"])
getattr(default_values, "write_" + file_format)(
default_path, partition_by="name",
connector_options={"partition.default-name": "unspecified"},
)
self.assertEqual([name for name in os.listdir(default_path) if not name.startswith(".")],
["name=unspecified"])
def test_partition_column_order_and_payload(self):
output_path = os.path.join(self.tempdir, "partition-order")
dataframe = pf.from_records([(1, "a", "b")], schema=["id", "first", "second"])
dataframe.write_json(output_path, partition_by=["second", "first"])
partition_path = os.path.join(output_path, "second=b", "first=a")
self.assertTrue(os.path.isdir(partition_path))
records = []
for filename in os.listdir(partition_path):
if filename.startswith("part-"):
with open(os.path.join(partition_path, filename), encoding="utf-8") as output:
records.extend(json.loads(line) for line in output)
self.assertEqual(records, [{"id": 1}])
def test_json_continuous_discovery_and_partition_commit(self):
self._check_continuous_discovery_and_partition_commit("json")
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_reader_discovers_new_files(self):
input_path = os.path.join(self.tempdir, "parquet-source")
output_path = os.path.join(self.tempdir, "json-sink")
os.makedirs(input_path)
staged_files = []
expected_rows = [(1, "first"), (2, "second")]
for row in expected_rows:
staging_path = os.path.join(self.tempdir, "parquet-staging-" + str(row[0]))
pf.from_records([row], schema=["id", "name"]).write_parquet(staging_path)
files = self._finished_files(staging_path)
self.assertEqual(len(files), 1)
staged_files.append(os.path.join(staging_path, files[0]))
# Publish only finished files so discovery cannot race with Parquet footer writes.
os.replace(staged_files[0], os.path.join(input_path, "first.parquet"))
self.env.set_runtime_mode(RuntimeExecutionMode.STREAMING)
self.env.enable_checkpointing(100)
self.t_env = StreamTableEnvironment.create(self.env)
pf.set_table_environment(self.t_env)
dataframe = pf.read_parquet(input_path, schema=self._SCHEMA, monitor_interval="50ms")
sink = (TableDescriptor.for_connector("filesystem")
.option("path", output_path)
.option("format", "json")
.option("sink.rolling-policy.rollover-interval", "100ms")
.option("sink.rolling-policy.check-interval", "50ms")
.build())
def read_finished_rows():
rows = []
for filename in self._finished_files(output_path):
with open(os.path.join(output_path, filename), encoding="utf-8") as output:
for line in output:
record = json.loads(line)
rows.append((record["id"], record["name"]))
return rows
result = dataframe.to_table().execute_insert(sink)
job_client = result.get_job_client()
execution_result = job_client.get_job_execution_result()
try:
self._wait_for(lambda: expected_rows[0] in read_finished_rows(),
"initial Parquet file was not read")
self.assertFalse(execution_result.done(), "Parquet source stopped after initial files")
os.replace(staged_files[1], os.path.join(input_path, "second.parquet"))
self._wait_for(lambda: expected_rows[1] in read_finished_rows(),
"new Parquet file was not discovered")
self.assertFalse(execution_result.done(), "continuous Parquet source terminated")
finally:
self._cancel_job_and_wait(job_client)
self.assertCountEqual(read_finished_rows(), expected_rows)
def test_json_partition_time_commit(self):
self._check_continuous_discovery_and_partition_commit("json", "partition-time")
def test_json_partition_time_commit_with_local_timezone(self):
self._check_continuous_discovery_and_partition_commit(
"json", "partition-time", "Asia/Shanghai")
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_continuous_discovery_and_partition_commit(self):
self._check_continuous_discovery_and_partition_commit("parquet")
def test_json_streaming_compaction(self):
self._check_continuous_discovery_and_partition_commit("json", auto_compaction=True)
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_streaming_compaction(self):
self._check_continuous_discovery_and_partition_commit("parquet", auto_compaction=True)
def _check_continuous_discovery_and_partition_commit(
self, file_format, partition_commit_trigger="process-time", local_timezone="UTC",
auto_compaction=None,
):
self.env.set_runtime_mode(RuntimeExecutionMode.STREAMING)
self.env.enable_checkpointing(100)
self.t_env = StreamTableEnvironment.create(self.env)
self.t_env.get_config().set_local_timezone(local_timezone)
pf.set_table_environment(self.t_env)
suffix = self._testMethodName
input_path = os.path.join(self.tempdir, "continuous-input-" + suffix)
output_path = os.path.join(self.tempdir, "continuous-output-" + suffix)
os.makedirs(input_path)
partition_value = "2026-01-01"
# Keep the watermark near the partition boundary to expose time zone mismatches.
timestamp = datetime(2026, 1, 1, 0, 1, tzinfo=ZoneInfo(local_timezone))
record = {"id": 1, "name": partition_value,
"ts_millis": int(timestamp.timestamp() * 1000)}
self._publish_json_file(input_path, "first.json", record)
dataframe = pf.read_json(
input_path, schema={**self._SCHEMA, "ts_millis": DataType.int64()},
monitor_interval="50ms",
computed_columns={"ts": "TO_TIMESTAMP_LTZ(ts_millis, 3)"},
watermark=("ts", "ts - INTERVAL '1' SECOND"),
)
results = []
execute_insert = dataframe._table.execute_insert
def capture_result(*args, **kwargs):
result = execute_insert(*args, **kwargs)
results.append(result)
return result
# Keep the public writer's actual execution, but manage this unbounded job in the test.
with patch.object(dataframe._table, "execute_insert", side_effect=capture_result), \
patch.object(TableResult, "wait"):
getattr(dataframe, "write_" + file_format)(
output_path, partition_by="name",
auto_compaction=auto_compaction,
compaction_file_size="1mb" if auto_compaction else None,
sink_shuffle_by_partition=True if auto_compaction else None,
rolling_policy_rollover_interval="100ms", rolling_policy_check_interval="50ms",
partition_commit_policy_kind="success-file",
partition_commit_trigger=partition_commit_trigger,
connector_options={
"sink.partition-commit.success-file.name": "_READY",
"partition.time-extractor.timestamp-pattern": "$name 00:00:00",
"sink.partition-commit.watermark-time-zone": local_timezone,
},
)
job_client = results[0].get_job_client()
partition_path = os.path.join(output_path, "name=" + partition_value)
try:
self._wait_for(
lambda: os.path.exists(os.path.join(partition_path, "_READY"))
and len(self._finished_files(partition_path)) >= 1,
"first checkpoint did not finish and commit the partition",
)
self._publish_json_file(input_path, "second.json", {**record, "id": 2})
self._wait_for(
lambda: len(self._finished_files(partition_path)) >= 2,
"newly discovered file was not committed by a later checkpoint",
)
if auto_compaction:
self.assertTrue(all(name.startswith("compacted-")
for name in self._finished_files(partition_path)))
self.assertEqual(job_client.get_job_status().result(), JobStatus.RUNNING)
finally:
self._cancel_job_and_wait(job_client)
self.env.set_runtime_mode(RuntimeExecutionMode.BATCH)
self.t_env = StreamTableEnvironment.create(
self.env, environment_settings=EnvironmentSettings.in_batch_mode())
pf.set_table_environment(self.t_env)
rows = getattr(pf, "read_" + file_format)(
output_path, schema=self._SCHEMA, partition_by="name").collect()
self.assertEqual(sorted(tuple(row) for row in rows),
[(1, partition_value), (2, partition_value)])
@staticmethod
def _publish_json_file(directory, filename, record):
temporary_path = os.path.join(directory, ".pending")
with open(temporary_path, "w", encoding="utf-8") as source:
source.write(json.dumps(record) + "\n")
os.replace(temporary_path, os.path.join(directory, filename))
@staticmethod
def _finished_files(directory):
if not os.path.isdir(directory):
return []
return [name for name in os.listdir(directory)
if name.startswith(("part-", "compacted-part-"))]
def _cancel_job_and_wait(self, job_client):
execution_result = job_client.get_job_execution_result()
# A finished job may have already shut down its MiniCluster.
if not execution_result.done():
cancellation = job_client.cancel()
self._wait_for(cancellation.done, "streaming job cancellation was not acknowledged")
cancellation.result()
# Cancellation acknowledges the request before the job has fully terminated.
self._wait_for(execution_result.done, "streaming job did not terminate")
def _wait_for(self, predicate, message):
deadline = time.monotonic() + 60
while not predicate():
if time.monotonic() >= deadline:
self.fail(message)
time.sleep(0.05)
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_round_trip_overwrite_and_append(self):
self._check_round_trip_overwrite_and_append("parquet")
def _check_round_trip_overwrite_and_append(self, file_format):
output_path = os.path.join(self.tempdir, file_format)
reader = getattr(pf, "read_" + file_format)
original = pf.from_records([(1, "a"), (2, None)], schema=["id", "name"])
getattr(original, "write_" + file_format)(output_path)
if file_format == "parquet":
self._assert_parquet_compression(output_path, "SNAPPY")
rows = reader(output_path, schema=self._SCHEMA).collect()
self.assertEqual(sorted(tuple(row) for row in rows), [(1, "a"), (2, None)])
replacement = pf.from_records([(3, "c")], schema=["id", "name"])
options = {"compression": "GZIP"} if file_format == "parquet" else {}
getattr(replacement, "write_" + file_format)(output_path, mode="overwrite", **options)
if file_format == "parquet":
self._assert_parquet_compression(output_path, "GZIP")
rows = reader(output_path, schema=self._SCHEMA).collect()
self.assertEqual([tuple(row) for row in rows], [(3, "c")])
getattr(original, "write_" + file_format)(output_path)
rows = reader(output_path, schema=self._SCHEMA).collect()
self.assertEqual(sorted(tuple(row) for row in rows),
[(1, "a"), (2, None), (3, "c")])
def _assert_parquet_compression(self, output_path, expected_codec):
jvm = get_gateway().jvm
hadoop_conf = jvm.org.apache.hadoop.conf.Configuration()
codecs = set()
for filename in os.listdir(output_path):
if filename.startswith("part-"):
path = jvm.org.apache.hadoop.fs.Path(os.path.join(output_path, filename))
footer = jvm.org.apache.parquet.hadoop.ParquetFileReader.readFooter(
hadoop_conf, path
)
for block in footer.getBlocks():
codecs.update(column.getCodec().name() for column in block.getColumns())
self.assertEqual(codecs, {expected_codec})
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_timestamp_encoding_options(self):
timestamp = datetime(1970, 1, 1, 0, 0, 1, 123456)
dataframe = pf.sql("SELECT TIMESTAMP '1970-01-01 00:00:01.123456' AS ts")
jvm = get_gateway().jvm
hadoop_conf = jvm.org.apache.hadoop.conf.Configuration()
cases = [(None, None), ("millis", 1123), ("micros", 1123456), ("nanos", 1123456000)]
for unit, encoded_timestamp in cases:
with self.subTest(unit=unit):
output_path = os.path.join(self.tempdir, "parquet-timestamp-" + str(unit))
options = {} if unit is None else {
"write.int64.timestamp": "true", "timestamp.time.unit": unit,
}
dataframe.write_parquet(output_path, utc_timezone=True, format_options=options)
files = self._finished_files(output_path)
self.assertEqual(len(files), 1)
path = jvm.org.apache.hadoop.fs.Path(os.path.join(output_path, files[0]))
footer = jvm.org.apache.parquet.hadoop.ParquetFileReader.readFooter(
hadoop_conf, path)
column_type = footer.getFileMetaData().getSchema().getType("ts").asPrimitiveType()
if unit is None:
self.assertEqual(column_type.getPrimitiveTypeName().name(), "INT96")
else:
self.assertEqual(column_type.getPrimitiveTypeName().name(), "INT64")
self.assertEqual(column_type.getLogicalTypeAnnotation().getUnit().name(),
unit.upper())
reader = (jvm.org.apache.parquet.hadoop.ParquetReader.builder(
jvm.org.apache.parquet.hadoop.example.GroupReadSupport(), path)
.withConf(hadoop_conf).build())
try:
self.assertEqual(reader.read().getLong("ts", 0), encoded_timestamp)
self.assertIsNone(reader.read())
finally:
reader.close()
restored = pf.read_parquet(
output_path, schema={"ts": DataType.timestamp(6)}, utc_timezone=True)
expected = timestamp.replace(microsecond=123000) if unit == "millis" else timestamp
view_name = "parquet_timestamps_" + str(unit)
self.t_env.create_temporary_view(view_name, restored.to_table())
# Compare in SQL so precision checks do not depend on Python timestamp conversion.
rows = pf.sql(
"SELECT ts = TIMESTAMP '" + expected.isoformat(sep=" ")
+ "' AS is_equal FROM " + view_name).collect()
self.assertEqual([tuple(row) for row in rows], [(True,)])
def test_json_streaming_append_and_overwrite_rejection(self):
self._check_streaming_append_and_overwrite_rejection("json")
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_streaming_append_and_overwrite_rejection(self):
self._check_streaming_append_and_overwrite_rejection("parquet")
def _check_streaming_append_and_overwrite_rejection(self, file_format):
self.env.set_runtime_mode(RuntimeExecutionMode.STREAMING)
self.t_env = StreamTableEnvironment.create(self.env)
pf.set_table_environment(self.t_env)
dataframe = pf.from_records([(1, "a")], schema=["id", "name"])
output_path = os.path.join(self.tempdir, "stream-" + file_format)
writer = getattr(dataframe, "write_" + file_format)
with self.assertRaisesRegex(Py4JJavaError, "Streaming mode not support overwrite"):
writer(output_path, mode="overwrite")
self.assertFalse(os.path.exists(output_path))
writer(output_path)
writer(output_path, mode="append")
rows = getattr(pf, "read_" + file_format)(
output_path, schema=self._SCHEMA
).collect()
self.assertEqual([tuple(row) for row in rows], [(1, "a"), (1, "a")])
def test_json_encoding_options(self):
dataframe = pf.sql(
"SELECT CAST(NULL AS STRING) AS name, CAST(0.0000001 AS DECIMAL(10, 7)) AS amount")
for enabled in (None, False, True):
with self.subTest(enabled=enabled):
output_path = os.path.join(self.tempdir, "json-encoding-" + str(enabled))
dataframe.write_json(
output_path, ignore_null_fields=enabled, decimal_as_plain_number=enabled)
records = []
for filename in os.listdir(output_path):
if filename.startswith("part-"):
with open(os.path.join(output_path, filename), encoding="utf-8") as output:
for line in output:
self.assertIsInstance(json.loads(line)["amount"], float)
records.append(json.loads(line, parse_float=str))
expected = {"amount": "0.0000001"} if enabled else {"name": None, "amount": "1E-7"}
self.assertEqual(records, [expected])
def test_json_map_null_keys_are_independent_of_null_fields(self):
dataframe = pf.sql(
"SELECT CAST(NULL AS STRING) AS name, "
"MAP[CAST(NULL AS STRING), 1, 'known', 2] AS attributes")
cases = [
({"map-null-key.mode": "DROP"}, {"attributes": {"known": 2}}),
({"map-null-key.mode": "LITERAL", "map-null-key.literal": "missing"},
{"attributes": {"missing": 1, "known": 2}}),
]
for options, expected in cases:
with self.subTest(options=options):
output_path = os.path.join(self.tempdir, "json-map-" + options["map-null-key.mode"])
dataframe.write_json(output_path, ignore_null_fields=True, format_options=options)
records = []
for filename in self._finished_files(output_path):
with open(os.path.join(output_path, filename), encoding="utf-8") as output:
records.extend(json.loads(line) for line in output)
self.assertEqual(records, [expected])
def test_json_format_options_and_path_filter(self):
input_path = os.path.join(self.tempdir, "json-input")
os.makedirs(input_path)
with open(os.path.join(input_path, "data.json"), "w", encoding="utf-8") as source:
source.write('{"id":1,"name":"a"}\nnot-json\n{"id":2,"name":null}\n')
with open(os.path.join(input_path, "excluded.json"), "w", encoding="utf-8") as source:
source.write('{"id":99,"name":"excluded"}\n')
rows = pf.read_json(
input_path, schema=self._SCHEMA, path_regex_pattern=".*data[.]json",
format_options={"ignore-parse-errors": "true"},
).collect()
self.assertEqual(sorted(tuple(row) for row in rows), [(1, "a"), (2, None)])
rows = pf.read_json(
input_path, schema=self._SCHEMA, path_regex_pattern="data[.]json",
).collect()
self.assertEqual(rows, [])
def test_json_continuous_source_is_rejected_in_batch_mode(self):
self._check_continuous_source_is_rejected_in_batch_mode(pf.read_json)
@unittest.skipIf(os.environ.get('HADOOP_CLASSPATH') is None,
'Hadoop libraries are required for Parquet format tests')
def test_parquet_continuous_source_is_rejected_in_batch_mode(self):
self._check_continuous_source_is_rejected_in_batch_mode(pf.read_parquet)
def _check_continuous_source_is_rejected_in_batch_mode(self, reader):
input_path = os.path.join(self.tempdir, self._testMethodName)
os.makedirs(input_path)
dataframe = reader(input_path, schema=self._SCHEMA, monitor_interval="1s")
with self.assertRaisesRegex(Py4JJavaError, "[Uu]nbounded"):
dataframe.to_table().explain()
def test_computed_watermark_supports_event_time_windows(self):
input_path = os.path.join(self.tempdir, "event-time.json")
with open(input_path, "w", encoding="utf-8") as source:
source.write('{"ts_millis":1000}\n{"ts_millis":2000}\n')
self.env.set_runtime_mode(RuntimeExecutionMode.STREAMING)
self.t_env = StreamTableEnvironment.create(self.env)
pf.set_table_environment(self.t_env)
dataframe = pf.read_json(
input_path, schema={"ts_millis": DataType.int64()},
computed_columns={"ts": "TO_TIMESTAMP_LTZ(ts_millis, 3)"},
watermark=("ts", "ts - INTERVAL '1' SECOND"),
)
self.t_env.create_temporary_view("events", dataframe.to_table())
result = pf.sql(
"SELECT COUNT(*) AS cnt FROM TABLE("
"TUMBLE(TABLE events, DESCRIPTOR(ts), INTERVAL '5' SECOND)) "
"GROUP BY window_start, window_end"
).collect()
self.assertEqual([tuple(row) for row in result], [(2,)])
def test_json_timestamp_format_options_round_trip(self):
output_path = os.path.join(self.tempdir, "json-timestamp")
timestamp = datetime(2026, 1, 2, 3, 4, 5)
dataframe = pf.sql("SELECT TIMESTAMP '2026-01-02 03:04:05' AS ts")
dataframe.write_json(
output_path, timestamp_format="ISO-8601",
format_options={"timestamp-format.standard": "ISO-8601"},
)
records = []
for filename in os.listdir(output_path):
if filename.startswith("part-"):
with open(os.path.join(output_path, filename), encoding="utf-8") as output:
records.extend(json.loads(line) for line in output)
self.assertEqual(records, [{"ts": timestamp.isoformat()}])
rows = pf.read_json(
output_path, schema={"ts": DataType.timestamp()},
timestamp_format="ISO-8601",
format_options={"json.timestamp-format.standard": "ISO-8601"},
).collect()
self.assertEqual([tuple(row) for row in rows], [(timestamp,)])
if __name__ == "__main__":
unittest.main()