blob: 5c0652e9a015d6a2a2fe8c430281801af5ef2821 [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use crate::{create_comet_physical_fun, IfExpr};
use crate::{remainder_by_zero_error, Cast, EvalMode, SparkCastOptions};
use arrow::array::{ArrayRef, ArrowNativeTypeOp, AsArray, PrimitiveArray};
use arrow::compute::kernels::arity::try_binary;
use arrow::compute::kernels::numeric::rem;
use arrow::datatypes::*;
use arrow::error::ArrowError;
use datafusion::common::{
exec_err, internal_err, not_impl_err, DataFusionError, Result, ScalarValue,
};
use datafusion::config::ConfigOptions;
use datafusion::execution::FunctionRegistry;
use datafusion::physical_expr::expressions::{lit, BinaryExpr};
use datafusion::physical_expr::ScalarFunctionExpr;
use datafusion::physical_expr_common::datum::{apply, apply_cmp_for_nested};
use datafusion::{
logical_expr::{ColumnarValue, Operator},
physical_expr::PhysicalExpr,
};
use std::cmp::max;
use std::sync::Arc;
/// Spark-compliant modulo function. If `fail_on_error` is true, then this function computes modulo
/// in ANSI mode and returns an error on division by zero, otherwise it returns `NULL` for such
/// cases.
pub fn spark_modulo(args: &[ColumnarValue], fail_on_error: bool) -> Result<ColumnarValue> {
if args.len() != 2 {
return exec_err!("modulo expects exactly two arguments");
}
let lhs = &args[0];
let rhs = &args[1];
let left_data_type = lhs.data_type();
let right_data_type = rhs.data_type();
if left_data_type.is_nested() {
if right_data_type != left_data_type {
return internal_err!("Type mismatch for spark modulo operation");
}
return apply_cmp_for_nested(Operator::Modulo, lhs, rhs);
}
// Arrow's `rem` kernel only signals `DivideByZero` for integer and decimal types. For
// floating point it uses `mod_wrapping`, so `x % 0.0` silently yields NaN. Under ANSI
// mode we compute the float remainder with `mod_checked` instead, which reports
// `DivideByZero` for a zero divisor.
if fail_on_error
&& left_data_type == right_data_type
&& matches!(right_data_type, DataType::Float32 | DataType::Float64)
{
return checked_float_modulo(lhs, rhs, &right_data_type);
}
match apply(lhs, rhs, rem) {
Ok(result) => Ok(result),
Err(e) if e.to_string().contains("Divide by zero") && fail_on_error => {
// Return Spark-compliant remainder by zero error.
Err(remainder_by_zero_error().into())
}
Err(e) => Err(e),
}
}
/// Computes floating point remainder in ANSI mode using `mod_checked`, which raises
/// `DivideByZero` when the divisor is zero. `-0.0` triggers the error too, since
/// `mod_checked` compares the divisor against zero with IEEE 754 equality — matching
/// Spark, which also treats `-0.0` as a zero divisor.
fn checked_float_modulo(
lhs: &ColumnarValue,
rhs: &ColumnarValue,
data_type: &DataType,
) -> Result<ColumnarValue> {
let (left, right): (ArrayRef, ArrayRef) = match (lhs, rhs) {
(ColumnarValue::Array(l), ColumnarValue::Array(r)) => (Arc::clone(l), Arc::clone(r)),
(ColumnarValue::Scalar(l), ColumnarValue::Array(r)) => {
(l.to_array_of_size(r.len())?, Arc::clone(r))
}
(ColumnarValue::Array(l), ColumnarValue::Scalar(r)) => {
(Arc::clone(l), r.to_array_of_size(l.len())?)
}
(ColumnarValue::Scalar(l), ColumnarValue::Scalar(r)) => (l.to_array()?, r.to_array()?),
};
let result: ArrayRef = match data_type {
DataType::Float32 => Arc::new(checked_modulo_kernel::<Float32Type>(&left, &right)?),
DataType::Float64 => Arc::new(checked_modulo_kernel::<Float64Type>(&left, &right)?),
_ => return not_impl_err!("checked_float_modulo doesn't support {data_type}"),
};
if matches!(
(lhs, rhs),
(ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
) {
// Preserve `apply`'s behavior of returning a scalar when both operands are scalars.
Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
&result, 0,
)?))
} else {
Ok(ColumnarValue::Array(result))
}
}
/// `try_binary` only invokes the closure for rows that are valid in both arrays, so rows
/// where either operand is null produce a null result without raising — matching Spark's
/// row-wise semantics.
fn checked_modulo_kernel<T: ArrowPrimitiveType>(
lhs: &ArrayRef,
rhs: &ArrayRef,
) -> Result<PrimitiveArray<T>> {
try_binary::<_, _, _, T>(lhs.as_primitive::<T>(), rhs.as_primitive::<T>(), |l, r| {
l.mod_checked(r)
})
.map_err(|e| match e {
ArrowError::DivideByZero => remainder_by_zero_error().into(),
other => DataFusionError::from(other),
})
}
pub fn create_modulo_expr(
left: Arc<dyn PhysicalExpr>,
right: Arc<dyn PhysicalExpr>,
data_type: DataType,
input_schema: SchemaRef,
fail_on_error: bool,
registry: &dyn FunctionRegistry,
) -> Result<Arc<dyn PhysicalExpr>, DataFusionError> {
// For non-ANSI mode, wrap the right expression such that any zero value is replaced with `NULL`
// to prevent divide by zero error.
let right_non_ansi_safe = if !fail_on_error {
null_if_zero_primitive(right, &input_schema)?
} else {
right
};
// If the data type is `Decimal128` and the (scale + integral part) exceeds the maximum allowed
// for `Decimal128`, then cast both operands to `Decimal256` before creating the modulo scalar
// expression, otherwise, create the modulo scalar expression directly.
match (
left.data_type(&input_schema),
right_non_ansi_safe.data_type(&input_schema),
) {
(Ok(DataType::Decimal128(p1, s1)), Ok(DataType::Decimal128(p2, s2)))
if max(s1, s2) as u8 + max(p1 - s1 as u8, p2 - s2 as u8) > DECIMAL128_MAX_PRECISION =>
{
let left_256 = Arc::new(Cast::new(
left,
DataType::Decimal256(p1, s1),
SparkCastOptions::new_without_timezone(EvalMode::Legacy, false),
None,
None,
));
let right_256 = Arc::new(Cast::new(
right_non_ansi_safe,
DataType::Decimal256(p2, s2),
SparkCastOptions::new_without_timezone(EvalMode::Legacy, false),
None,
None,
));
// The UDF's return type must match what Arrow's rem function will actually return.
// Since we're operating on Decimal256 inputs, rem will return Decimal256.
let decimal256_return_type = match &data_type {
DataType::Decimal128(p, s) => DataType::Decimal256(*p, *s),
other => other.clone(),
};
let modulo_scalar_func = create_modulo_scalar_function(
left_256,
right_256,
&decimal256_return_type,
registry,
fail_on_error,
)?;
Ok(Arc::new(Cast::new(
modulo_scalar_func,
data_type,
SparkCastOptions::new_without_timezone(EvalMode::Legacy, false),
None,
None,
)))
}
_ => create_modulo_scalar_function(
left,
right_non_ansi_safe,
&data_type,
registry,
fail_on_error,
),
}
}
fn null_if_zero_primitive(
expression: Arc<dyn PhysicalExpr>,
input_schema: &Schema,
) -> Result<Arc<dyn PhysicalExpr>, DataFusionError> {
let expr_data_type = expression.data_type(input_schema)?;
if is_primitive_datatype(&expr_data_type) {
let zero = match expr_data_type {
DataType::Int8 => ScalarValue::Int8(Some(0)),
DataType::Int16 => ScalarValue::Int16(Some(0)),
DataType::Int32 => ScalarValue::Int32(Some(0)),
DataType::Int64 => ScalarValue::Int64(Some(0)),
DataType::UInt8 => ScalarValue::UInt8(Some(0)),
DataType::UInt16 => ScalarValue::UInt16(Some(0)),
DataType::UInt32 => ScalarValue::UInt32(Some(0)),
DataType::UInt64 => ScalarValue::UInt64(Some(0)),
DataType::Float32 => ScalarValue::Float32(Some(0.0)),
DataType::Float64 => ScalarValue::Float64(Some(0.0)),
DataType::Decimal128(s, p) => ScalarValue::Decimal128(Some(0), s, p),
DataType::Decimal256(s, p) => ScalarValue::Decimal256(Some(i256::from(0)), s, p),
_ => return Ok(expression),
};
// Create an expression like - `if (eval(expr) == Literal(0)) then NULL else eval(expr)`.
// This expression evaluates to null for rows with zero values to prevent divide by zero
// error.
let eq_expr = Arc::new(BinaryExpr::new(
Arc::<dyn PhysicalExpr>::clone(&expression),
Operator::Eq,
lit(zero),
));
let null_literal = lit(ScalarValue::try_new_null(&expr_data_type)?);
let if_expr = Arc::new(IfExpr::new(eq_expr, null_literal, expression));
Ok(if_expr)
} else {
Ok(expression)
}
}
fn is_primitive_datatype(dt: &DataType) -> bool {
matches!(
dt,
DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
| DataType::Float32
| DataType::Float64
| DataType::Decimal128(_, _)
| DataType::Decimal256(_, _)
)
}
fn create_modulo_scalar_function(
left: Arc<dyn PhysicalExpr>,
right: Arc<dyn PhysicalExpr>,
data_type: &DataType,
registry: &dyn FunctionRegistry,
fail_on_error: bool,
) -> Result<Arc<dyn PhysicalExpr>, DataFusionError> {
let func_name = "spark_modulo";
let modulo_expr =
create_comet_physical_fun(func_name, data_type.clone(), registry, Some(fail_on_error))?;
Ok(Arc::new(ScalarFunctionExpr::new(
func_name,
modulo_expr,
vec![left, right],
Arc::new(Field::new(func_name, data_type.clone(), true)),
Arc::new(ConfigOptions::default()),
)))
}
#[cfg(test)]
mod tests {
use super::*;
use arrow::array::{
Array, ArrayRef, Decimal128Array, Decimal128Builder, Float32Array, Float64Array,
Int32Array, PrimitiveArray, RecordBatch,
};
use datafusion::logical_expr::ColumnarValue;
use datafusion::physical_expr::expressions::{Column, Literal};
use datafusion::prelude::SessionContext;
fn with_fail_on_error<F: Fn(bool)>(test_fn: F) {
for fail_on_error in [true, false] {
test_fn(fail_on_error);
}
}
pub fn verify_result<T>(
expr: Arc<dyn PhysicalExpr>,
batch: RecordBatch,
should_fail: bool,
expected_result: Option<Arc<PrimitiveArray<T>>>,
) where
T: ArrowPrimitiveType,
{
let actual_result = expr.evaluate(&batch);
if should_fail {
match actual_result {
Err(error) => {
assert!(
error
.to_string()
.contains("[REMAINDER_BY_ZERO] Remainder by zero"),
"Error message did not match. Actual message: {error}"
);
}
Ok(value) => {
panic!("Expected error, but got: {value:?}");
}
}
} else {
match (actual_result, expected_result) {
(Ok(ColumnarValue::Array(ref actual)), Some(expected)) => {
assert_eq!(actual.len(), expected.len(), "Array length mismatch");
let actual_arr = actual.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
let expected_arr = expected
.as_any()
.downcast_ref::<PrimitiveArray<T>>()
.unwrap();
for i in 0..actual_arr.len() {
assert_eq!(
actual_arr.is_null(i),
expected_arr.is_null(i),
"Nullity mismatch at index {i}"
);
if !actual_arr.is_null(i) {
let actual_value = actual_arr.value(i);
let expected_value = expected_arr.value(i);
assert_eq!(
actual_value, expected_value,
"Mismatch at index {i}, actual {actual_value:?}, expected {expected_value:?}"
);
}
}
}
(actual, expected) => {
panic!("Actual: {actual:?}, expected: {expected:?}");
}
}
}
}
#[test]
fn test_modulo_basic_int() {
with_fail_on_error(|fail_on_error| {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
]));
let a_array = Arc::new(Int32Array::from(vec![3, 2, i32::MIN]));
let b_array = Arc::new(Int32Array::from(vec![1, 5, -1]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![a_array, b_array]).unwrap();
let left_expr = Arc::new(Column::new("a", 0));
let right_expr = Arc::new(Column::new("b", 1));
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
left_expr,
right_expr,
DataType::Int32,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
// This test case should not fail as there is no division by zero.
let should_fail = false;
let expected_result = Arc::new(Int32Array::from(vec![0, 2, 0]));
verify_result(modulo_expr, batch, should_fail, Some(expected_result));
})
}
#[test]
fn test_modulo_basic_decimal() {
with_fail_on_error(|fail_on_error| {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Decimal128(18, 4), false),
Field::new("b", DataType::Decimal128(18, 4), false),
]));
let mut a_builder =
Decimal128Builder::with_capacity(2).with_data_type(DataType::Decimal128(18, 4));
a_builder.append_value(3000000000000000000);
a_builder.append_value(2000000000000000000);
let a_array: ArrayRef = Arc::new(a_builder.finish());
let mut b_builder =
Decimal128Builder::with_capacity(2).with_data_type(DataType::Decimal128(18, 4));
b_builder.append_value(1000000000000000000);
b_builder.append_value(5000000000000000000);
let b_array: ArrayRef = Arc::new(b_builder.finish());
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![a_array, b_array]).unwrap();
let left_expr = Arc::new(Column::new("a", 0));
let right_expr = Arc::new(Column::new("b", 1));
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
left_expr,
right_expr,
DataType::Decimal128(18, 4),
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
// This test case should not fail as there is no division by zero.
let should_fail = false;
let expected_result = Arc::new(Decimal128Array::from(vec![
Some(0),
Some(2000000000000000000),
]));
verify_result(modulo_expr, batch, should_fail, Some(expected_result));
})
}
#[test]
fn test_modulo_divide_by_zero_int() {
with_fail_on_error(|fail_on_error| {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
]));
let a_array = Arc::new(Int32Array::from(vec![3]));
let b_array = Arc::new(Int32Array::from(vec![0]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![a_array, b_array]).unwrap();
let left_expr = Arc::new(Column::new("a", 0));
let right_expr = Arc::new(Column::new("b", 1));
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
left_expr,
right_expr,
DataType::Int32,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
// Expected result in non-ANSI mode.
let expected_result = Arc::new(Int32Array::from(vec![None]));
verify_result(modulo_expr, batch, fail_on_error, Some(expected_result));
})
}
#[test]
fn test_division_by_zero_with_complex_int_expr() {
with_fail_on_error(|fail_on_error| {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("c", DataType::Int32, false),
]));
let a_array = Arc::new(Int32Array::from(vec![3, 0]));
let b_array = Arc::new(Int32Array::from(vec![2, 4]));
let c_array = Arc::new(Int32Array::from(vec![4, 5]));
let batch =
RecordBatch::try_new(Arc::clone(&schema), vec![a_array, b_array, c_array]).unwrap();
let left_expr = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
Operator::Divide,
Arc::new(Column::new("b", 1)),
));
let right_expr = Arc::new(BinaryExpr::new(
Arc::new(Literal::new(ScalarValue::Int32(Some(0)))),
Operator::Divide,
Arc::new(Column::new("c", 2)),
));
// Computes modulo of (a / b) % (0 / c).
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
left_expr,
right_expr,
DataType::Int32,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
// Expected result in non-ANSI mode.
let expected_result = Arc::new(Int32Array::from(vec![None, None]));
verify_result(modulo_expr, batch, fail_on_error, Some(expected_result));
})
}
fn run_float_modulo<T: ArrowPrimitiveType>(
data_type: DataType,
lhs: ArrayRef,
rhs: ArrayRef,
fail_on_error: bool,
should_fail: bool,
expected: Option<Arc<PrimitiveArray<T>>>,
) {
let schema = Arc::new(Schema::new(vec![
Field::new("a", data_type.clone(), true),
Field::new("b", data_type.clone(), true),
]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![lhs, rhs]).unwrap();
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
Arc::new(Column::new("a", 0)),
Arc::new(Column::new("b", 1)),
data_type,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
verify_result(modulo_expr, batch, should_fail, expected);
}
#[test]
fn test_modulo_divide_by_zero_float64_ansi() {
// ANSI mode with a zero divisor must raise REMAINDER_BY_ZERO for Float64.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![Some(1.0)])),
Arc::new(Float64Array::from(vec![Some(0.0)])),
true,
true,
None,
);
}
#[test]
fn test_modulo_divide_by_negative_zero_float64_ansi() {
// `-0.0` is equal to `0.0` under IEEE 754, and Spark treats it as a zero divisor.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![Some(1.0)])),
Arc::new(Float64Array::from(vec![Some(-0.0)])),
true,
true,
None,
);
}
#[test]
fn test_modulo_divide_by_zero_float32_ansi() {
run_float_modulo::<Float32Type>(
DataType::Float32,
Arc::new(Float32Array::from(vec![Some(1.0_f32)])),
Arc::new(Float32Array::from(vec![Some(0.0_f32)])),
true,
true,
None,
);
}
#[test]
fn test_modulo_divide_by_zero_float64_non_ansi() {
// Non-ANSI mode preserves the existing behavior: the divisor is nulled out and the
// result is null.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![Some(1.0)])),
Arc::new(Float64Array::from(vec![Some(0.0)])),
false,
false,
Some(Arc::new(Float64Array::from(vec![None]))),
);
}
#[test]
fn test_modulo_null_dividend_zero_divisor_float64_ansi() {
// A null dividend must not raise even when the divisor is zero — Spark returns null
// whenever either operand is null.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![None])),
Arc::new(Float64Array::from(vec![Some(0.0)])),
true,
false,
Some(Arc::new(Float64Array::from(vec![None]))),
);
}
#[test]
fn test_modulo_mixed_null_and_zero_divisor_float64_ansi() {
// Row 0: null lhs paired with zero divisor -> null result, no error.
// Row 1: non-null lhs paired with zero divisor -> must raise.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![None, Some(1.0)])),
Arc::new(Float64Array::from(vec![Some(0.0), Some(0.0)])),
true,
true,
None,
);
}
#[test]
fn test_modulo_all_null_lhs_zero_scalar_divisor_float64_ansi() {
// Every lhs row is null, so the zero scalar divisor cannot pair with a non-null
// dividend and no error should be raised.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![None, None])),
Arc::new(Float64Array::from(vec![Some(0.0), Some(0.0)])),
true,
false,
Some(Arc::new(Float64Array::from(vec![None, None]))),
);
}
/// Evaluates `a % <literal>` where `a` is a Float64 column, so `spark_modulo` receives
/// `(Array, Scalar)` rather than the `(Array, Array)` pair that `run_float_modulo`
/// produces.
fn run_float_modulo_with_literal_divisor(
lhs: ArrayRef,
divisor: ScalarValue,
fail_on_error: bool,
should_fail: bool,
expected: Option<Arc<Float64Array>>,
) {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![lhs]).unwrap();
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
Arc::new(Column::new("a", 0)),
Arc::new(Literal::new(divisor)),
DataType::Float64,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
verify_result(modulo_expr, batch, should_fail, expected);
}
#[test]
fn test_modulo_literal_zero_divisor_float64_ansi() {
// `(Array, Scalar)` operands with a zero literal divisor must raise.
run_float_modulo_with_literal_divisor(
Arc::new(Float64Array::from(vec![Some(1.0), Some(2.0)])),
ScalarValue::Float64(Some(0.0)),
true,
true,
None,
);
}
#[test]
fn test_modulo_literal_negative_zero_divisor_float64_ansi() {
run_float_modulo_with_literal_divisor(
Arc::new(Float64Array::from(vec![Some(1.0)])),
ScalarValue::Float64(Some(-0.0)),
true,
true,
None,
);
}
#[test]
fn test_modulo_all_null_lhs_literal_zero_divisor_float64_ansi() {
// Every dividend is null, so no row pairs a non-null dividend with the zero
// literal divisor and no error should be raised.
run_float_modulo_with_literal_divisor(
Arc::new(Float64Array::from(vec![None, None])),
ScalarValue::Float64(Some(0.0)),
true,
false,
Some(Arc::new(Float64Array::from(vec![None, None]))),
);
}
#[test]
fn test_modulo_literal_non_zero_divisor_float64() {
with_fail_on_error(|fail_on_error| {
run_float_modulo_with_literal_divisor(
Arc::new(Float64Array::from(vec![Some(5.0), None])),
ScalarValue::Float64(Some(2.0)),
fail_on_error,
false,
Some(Arc::new(Float64Array::from(vec![Some(1.0), None]))),
);
})
}
/// Evaluates `<literal> % b` where `b` is a Float64 column, covering the
/// `(Scalar, Array)` operand pair.
fn run_float_modulo_with_literal_dividend(
dividend: ScalarValue,
rhs: ArrayRef,
fail_on_error: bool,
should_fail: bool,
expected: Option<Arc<Float64Array>>,
) {
let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Float64, true)]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![rhs]).unwrap();
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
Arc::new(Literal::new(dividend)),
Arc::new(Column::new("b", 0)),
DataType::Float64,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
verify_result(modulo_expr, batch, should_fail, expected);
}
#[test]
fn test_modulo_literal_dividend_zero_divisor_float64_ansi() {
// `(Scalar, Array)` operands: row 1 pairs the non-null literal dividend with a
// zero divisor and must raise.
run_float_modulo_with_literal_dividend(
ScalarValue::Float64(Some(1.0)),
Arc::new(Float64Array::from(vec![Some(2.0), Some(0.0)])),
true,
true,
None,
);
}
#[test]
fn test_modulo_null_literal_dividend_zero_divisor_float64_ansi() {
// A null literal dividend must not raise even though the divisor is zero.
run_float_modulo_with_literal_dividend(
ScalarValue::Float64(None),
Arc::new(Float64Array::from(vec![Some(0.0)])),
true,
false,
Some(Arc::new(Float64Array::from(vec![None]))),
);
}
#[test]
fn test_modulo_both_literals_zero_divisor_float64_ansi() {
// `(Scalar, Scalar)` operands. Spark folds fully-literal expressions before
// execution, so this pair only reaches `spark_modulo` via a plan built directly,
// but the branch must still raise rather than return NaN.
let session_ctx = SessionContext::new();
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(Float64Array::from(vec![Some(1.0)])) as ArrayRef],
)
.unwrap();
let modulo_expr = create_modulo_expr(
Arc::new(Literal::new(ScalarValue::Float64(Some(1.0)))),
Arc::new(Literal::new(ScalarValue::Float64(Some(0.0)))),
DataType::Float64,
schema,
true,
&session_ctx.state(),
)
.unwrap();
verify_result::<Float64Type>(modulo_expr, batch, true, None);
}
#[test]
fn test_modulo_both_literals_non_zero_divisor_float64_ansi() {
// `(Scalar, Scalar)` operands must still evaluate to a scalar, not a one-row array.
let session_ctx = SessionContext::new();
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(Float64Array::from(vec![Some(1.0), Some(2.0)])) as ArrayRef],
)
.unwrap();
let modulo_expr = create_modulo_expr(
Arc::new(Literal::new(ScalarValue::Float64(Some(5.0)))),
Arc::new(Literal::new(ScalarValue::Float64(Some(2.0)))),
DataType::Float64,
schema,
true,
&session_ctx.state(),
)
.unwrap();
match modulo_expr.evaluate(&batch).unwrap() {
ColumnarValue::Scalar(ScalarValue::Float64(Some(v))) => assert_eq!(v, 1.0),
other => panic!("Expected a Float64 scalar, got {other:?}"),
}
}
#[test]
fn test_modulo_special_dividends_zero_divisor_float64_ansi() {
// Spark's `DivModLike.eval` only inspects the divisor when deciding whether to
// raise, so NaN, +/-Infinity and 0.0 dividends must all throw rather than
// producing NaN.
for dividend in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, 0.0, -0.0] {
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![Some(dividend)])),
Arc::new(Float64Array::from(vec![Some(0.0)])),
true,
true,
None,
);
// Same dividends against a literal zero divisor, i.e. the `Array/Scalar` pair.
run_float_modulo_with_literal_divisor(
Arc::new(Float64Array::from(vec![Some(dividend)])),
ScalarValue::Float64(Some(0.0)),
true,
true,
None,
);
}
}
/// Evaluates `a % b` over two Float64 columns and returns the result array.
///
/// Used by tests whose expected result is NaN. `verify_result` compares with `==`, under
/// which `NaN != NaN`, and arrow's `is_eq`/`total_cmp` is no better here because the NaN
/// produced by an invalid `fmod` has a platform-dependent sign — x86 and aarch64 disagree
/// on whether it is `+NaN` or `-NaN`. These tests assert NaN-ness instead.
fn eval_float64_modulo(lhs: ArrayRef, rhs: ArrayRef, fail_on_error: bool) -> Float64Array {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Float64, true),
Field::new("b", DataType::Float64, true),
]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![lhs, rhs]).unwrap();
let session_ctx = SessionContext::new();
let modulo_expr = create_modulo_expr(
Arc::new(Column::new("a", 0)),
Arc::new(Column::new("b", 1)),
DataType::Float64,
schema,
fail_on_error,
&session_ctx.state(),
)
.unwrap();
let ColumnarValue::Array(result) = modulo_expr.evaluate(&batch).unwrap() else {
panic!("Expected an array result");
};
result.as_primitive::<Float64Type>().clone()
}
#[test]
fn test_modulo_special_dividends_non_zero_divisor_float64_ansi() {
// The same dividends must not raise when the divisor is non-zero: `x % 2.0` is NaN
// for NaN and for both infinities, 0.0 for a zero dividend, and 1.0 for 5.0.
let result = eval_float64_modulo(
Arc::new(Float64Array::from(vec![
Some(f64::NAN),
Some(f64::INFINITY),
Some(f64::NEG_INFINITY),
Some(0.0),
Some(5.0),
])),
Arc::new(Float64Array::from(vec![Some(2.0); 5])),
true,
);
assert_eq!(result.null_count(), 0);
for i in 0..3 {
assert!(
result.value(i).is_nan(),
"Expected NaN at index {i}, got {}",
result.value(i)
);
}
assert_eq!(result.value(3), 0.0);
assert_eq!(result.value(4), 1.0);
}
#[test]
fn test_modulo_mixed_batch_float64_ansi() {
// A batch mixing non-zero divisors, a zero divisor with a non-null dividend, and a
// zero divisor with a null dividend must raise, because of the middle row.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![
Some(1.0),
Some(3.0),
None,
Some(5.0),
])),
Arc::new(Float64Array::from(vec![
Some(2.0),
Some(0.0),
Some(0.0),
Some(1.5),
])),
true,
true,
None,
);
}
#[test]
fn test_modulo_mixed_batch_zero_divisors_only_null_dividends_float64_ansi() {
// Every zero divisor in this batch pairs with a null dividend, so no row can raise
// and the non-zero rows must still compute.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![
Some(5.0),
None,
Some(7.0),
None,
Some(9.0),
])),
Arc::new(Float64Array::from(vec![
Some(2.0),
Some(0.0),
Some(4.0),
Some(-0.0),
Some(2.0),
])),
true,
false,
Some(Arc::new(Float64Array::from(vec![
Some(1.0),
None,
Some(3.0),
None,
Some(1.0),
]))),
);
}
#[test]
fn test_modulo_null_divisor_float64_ansi() {
// A null divisor is not a zero divisor: Spark returns null without raising.
run_float_modulo::<Float64Type>(
DataType::Float64,
Arc::new(Float64Array::from(vec![Some(1.0), Some(2.0)])),
Arc::new(Float64Array::from(vec![None, Some(2.0)])),
true,
false,
Some(Arc::new(Float64Array::from(vec![None, Some(0.0)]))),
);
}
#[test]
fn test_modulo_nan_divisor_float64_ansi() {
// A NaN divisor is not zero either, so `x % NaN` is NaN rather than an error.
let result = eval_float64_modulo(
Arc::new(Float64Array::from(vec![Some(1.0)])),
Arc::new(Float64Array::from(vec![Some(f64::NAN)])),
true,
);
assert_eq!(result.null_count(), 0);
assert!(
result.value(0).is_nan(),
"Expected NaN, got {}",
result.value(0)
);
}
}