blob: f4fdc60ba6eb9bb8326e606b69aea3df398a75c8 [file]
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run with Java 22+ and local spark-submit from a build with native data sources (4.4.0+)."""
import argparse
import json
import tempfile
from pathlib import Path
from pyspark.errors import AnalysisException
from pyspark.sql import SparkSession
def committed_rows(root):
manifest = json.loads((root / "manifest.json").read_text())
assert manifest["version"] == 1, manifest
result = []
for entry in manifest["files"]:
rows = [json.loads(line) for line in (root / entry["path"]).read_text().splitlines()]
assert len(rows) == entry["rows"], entry
result.extend((row["id"], row["label"]) for row in rows)
return sorted(result)
def assert_clean(root):
assert not (root / ".write-lock").exists(), "The operation lock was not released"
staging = root / ".staging"
assert not staging.exists() or not list(staging.iterdir()), "Staged files were not cleaned"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--package", type=Path, required=True)
args = parser.parse_args()
package = args.package.resolve(strict=True)
spark = (
SparkSession.builder.appName("Rust native batch sink smoke")
.config("spark.sql.adaptive.enabled", "false")
.config("spark.task.maxFailures", "1")
.getOrCreate()
)
try:
if not spark.sparkContext.master.startswith("local"):
raise ValueError(
"This smoke script uses driver-local temporary files; use --master local"
)
spark.addArtifact(str(package))
def data(start, end):
return spark.range(start, end, numPartitions=2).selectExpr(
"id", "concat('row-', id) AS label"
)
with tempfile.TemporaryDirectory(prefix="rust-native-sink-") as directory:
root = Path(directory)
def write(frame, mode):
frame.write.format("rust_sdk_sink").mode(mode).save(str(root))
assert_clean(root)
write(data(0, 5), "append")
assert committed_rows(root) == [(i, f"row-{i}") for i in range(5)]
write(data(5, 9), "append")
assert committed_rows(root) == [(i, f"row-{i}") for i in range(9)]
old_manifest = json.loads((root / "manifest.json").read_text())
write(data(20, 23), "overwrite")
assert committed_rows(root) == [(i, f"row-{i}") for i in range(20, 23)]
assert all((root / entry["path"]).exists() for entry in old_manifest["files"])
# No input shuffle: the failure happens inside the write job, after
# the driver has created the coordinator and its staging directory.
before = (root / "manifest.json").read_bytes()
previous_operations = sorted(path.name for path in (root / "objects").iterdir())
failing = spark.range(0, 8, numPartitions=2).selectExpr(
"id",
"if(id >= 4, cast(raise_error('intentional native sink input failure') AS STRING), "
"concat('row-', id)) AS label",
)
try:
failing.write.format("rust_sdk_sink").mode("overwrite").save(str(root))
except Exception as error:
assert "intentional native sink input failure" in str(error), str(error)
else:
raise AssertionError("The input error should abort the overwrite")
assert (root / "manifest.json").read_bytes() == before
assert sorted(path.name for path in (root / "objects").iterdir()) == previous_operations
assert committed_rows(root) == [(i, f"row-{i}") for i in range(20, 23)]
assert_clean(root)
for mode in ("append", "overwrite"):
try:
(
data(0, 1)
.write.format("rust_sdk_sink")
.mode(mode)
.option("readonly", "true")
.save(str(root))
)
except AnalysisException as error:
assert error.getCondition() == "UNSUPPORTED_FEATURE.TABLE_OPERATION", str(error)
else:
raise AssertionError(f"readonly=true should reject {mode}")
assert (root / "manifest.json").read_bytes() == before
assert_clean(root)
# A streaming write records the epoch of each micro-batch that it commits: a restart
# without new input commits nothing, and new input commits the next epoch.
with tempfile.TemporaryDirectory(prefix="rust-native-sink-stream-") as directory:
base = Path(directory)
source, output = base / "input", base / "output"
source.mkdir()
def add_input(name, ids):
lines = "".join(json.dumps({"id": i, "label": f"row-{i}"}) + "\n" for i in ids)
(source / f"{name}.json").write_text(lines)
def run_stream():
query = (
spark.readStream.schema("id BIGINT, label STRING")
.json(str(source))
.writeStream.format("rust_sdk_sink")
.option("checkpointLocation", str(base / "checkpoint"))
.trigger(availableNow=True)
.start(str(output))
)
query.awaitTermination()
assert_clean(output)
return json.loads((output / "manifest.json").read_text())
add_input("first", range(3))
first = run_stream()
assert list(first["epochs"].values()) == [0], first
assert committed_rows(output) == [(i, f"row-{i}") for i in range(3)]
assert run_stream() == first
add_input("second", range(3, 5))
second = run_stream()
assert second["epochs"] == {query: 1 for query in first["epochs"]}, second
assert committed_rows(output) == [(i, f"row-{i}") for i in range(5)]
print("Rust native sink Spark smoke checks passed")
finally:
spark.stop()
if __name__ == "__main__":
main()