blob: ff92da9dcc772468bfcd556901499fa5a402bd2e [file]
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run with Java 22+ and spark-submit from a distribution with native data sources (4.4.0+)."""
import argparse
import tempfile
from pathlib import Path
from pyspark.errors import AnalysisException
from pyspark.sql import SparkSession
from pyspark.sql import functions as F
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--package", type=Path, required=True)
args = parser.parse_args()
package = args.package.resolve(strict=True)
spark = (
SparkSession.builder.appName("Rust native data source smoke")
.config("spark.sql.adaptive.enabled", "true")
.getOrCreate()
)
try:
spark.addArtifact(str(package))
df = (
spark.read.format("rust_sdk_range")
.option("start", 0)
.option("end", 31)
.option("numpartitions", 4)
.option("batchsize", 3)
.load()
)
assert [tuple(row) for row in df.orderBy("id").collect()] == [
(i, f"row-{i}") for i in range(31)
]
# Spark pushes a limit only to a scan whose rows it does not filter.
unfiltered_limit = df.limit(4).collect()
assert len(unfiltered_limit) == 4
assert all(0 <= row.id < 31 for row in unfiltered_limit)
projected = df.where((F.col("id") >= 7) & (F.col("id") < 17)).select("label", "id")
assert [tuple(row) for row in projected.orderBy("id").collect()] == [
(f"row-{i}", i) for i in range(7, 17)
]
limited = df.where((F.col("id") >= 10) & (F.col("id") < 20)).limit(4).collect()
assert len(limited) == 4
assert all(10 <= row.id < 20 for row in limited)
# Count needs row counts even when column pruning produces zero-column Arrow batches.
assert df.select().count() == 31
assert df.where(F.col("id") < 0).count() == 0
# Spark evaluates the filters after reading, as the source pushes none down.
arithmetic = df.where((F.col("id") % 3) == 1).select("id").orderBy("id").collect()
assert [row.id for row in arithmetic] == list(range(1, 31, 3))
labels = df.where(F.col("label").endswith("3")).select("id").orderBy("id").collect()
assert [row.id for row in labels] == [3, 13, 23]
residual_limit = df.where(F.col("label") == "row-21").limit(1).select("id").collect()
assert [row.id for row in residual_limit] == [21]
# A streaming read of the fixed range reads all its rows in its first micro-batch, with
# the reader of batch scans, or with maxRowsPerTrigger, in micro-batches of at most that
# many rows, also with Trigger.AvailableNow.
for name, options, batches in [
("rust_sdk_range_stream", {}, [31]),
("rust_sdk_range_limited", {"maxRowsPerTrigger": 10}, [10, 10, 10, 1]),
]:
with tempfile.TemporaryDirectory(prefix="rust-native-range-stream-") as checkpoint:
query = (
spark.readStream.format("rust_sdk_range")
.option("start", 0)
.option("end", 31)
.option("numpartitions", 4)
.option("batchsize", 3)
.options(**options)
.load()
.writeStream.format("memory")
.queryName(name)
.option("checkpointLocation", checkpoint)
.trigger(availableNow=True)
.start()
)
query.awaitTermination()
rows = [p["numInputRows"] for p in query.recentProgress if p["numInputRows"]]
assert rows == batches, rows
streamed = spark.table(name).orderBy("id").collect()
assert [tuple(row) for row in streamed] == [(i, f"row-{i}") for i in range(31)]
# Analysis must reject both modes before AQE can run the input shuffle.
failing_input = (
spark.range(10)
.selectExpr(
"if(id >= 0, raise_error('input was evaluated'), id) AS id", "'row' AS label"
)
.repartition(2)
)
for mode in ("append", "overwrite"):
try:
failing_input.write.format("rust_sdk_range").mode(mode).save()
except AnalysisException as error:
assert error.getCondition() == "UNSUPPORTED_FEATURE.TABLE_OPERATION", str(error)
assert error.getMessageParameters() == {
"tableName": "`rust_sdk_range`",
"operation": "batch write",
}, str(error)
else:
raise AssertionError(f"{mode} should reject this batch-read-only source")
print("Rust native data source Spark smoke checks passed")
finally:
spark.stop()
if __name__ == "__main__":
main()