Google Git Google Git
Sign in
apache/datafusion-comet/refs/heads/codex/comet-df-bitmap/./native/spark-expr/src/conversion_funcs/numeric.rs
blob: f7fbae54f9822997a2d9821a0f9dcaf7e942953b [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use crate::conversion_funcs::string::{digits_to_i128, div_round_half_up_i128, pow10_i128};
use crate::conversion_funcs::utils::cast_overflow;
use crate::conversion_funcs::utils::MICROS_PER_SECOND;
use crate::{EvalMode, SparkError, SparkResult};
use arrow::array::{
Array, ArrayRef, AsArray, BooleanArray, Decimal128Array, Float32Array, Float64Array,
GenericStringBuilder, Int16Array, Int32Array, Int64Array, Int8Array, OffsetSizeTrait,
PrimitiveArray, Scalar, StringBuilder, TimestampMicrosecondBuilder,
};
use arrow::buffer::BooleanBuffer;
use arrow::compute::kernels::cmp::neq;
use arrow::datatypes::{
format_decimal_str, i256, is_validate_decimal_precision, ArrowPrimitiveType, DataType,
Decimal128Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type,
};
use num::{cast::AsPrimitive, Float, ToPrimitive};
use std::fmt::{self, Write};
use std::sync::Arc;
/// Check if DataFusion cast from integer types is Spark compatible
pub(crate) fn is_df_cast_from_int_spark_compatible(to_type: &DataType) -> bool {
matches!(
to_type,
DataType::Boolean
| DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::Float32
| DataType::Float64
| DataType::Utf8
)
}
/// Check if DataFusion cast from float types is Spark compatible
pub(crate) fn is_df_cast_from_float_spark_compatible(to_type: &DataType) -> bool {
matches!(
to_type,
DataType::Boolean
| DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::Float32
| DataType::Float64
)
}
/// Check if DataFusion cast from decimal types is Spark compatible
pub(crate) fn is_df_cast_from_decimal_spark_compatible(to_type: &DataType) -> bool {
matches!(
to_type,
DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
// Float32 / Float64 are intentionally absent: DataFusion computes
// `(unscaled as f64) / 10^scale`, which rounds twice (three times for f32, via the
// f64 intermediate) and can differ from Spark's correctly rounded
// BigDecimal.doubleValue() / floatValue() by one ulp once |unscaled| > 2^53.
// cast.rs routes Decimal128 sources to cast_decimal128_to_float64 / _float32.
| DataType::Decimal128(_, _)
| DataType::Decimal256(_, _)
// DataFusion's Decimal128→Utf8 cast uses plain notation (toPlainString semantics),
// matching Spark's TRY and ANSI modes. LEGACY mode is handled by a separate match
// arm in cast_array that applies Java BigDecimal.toString() (scientific notation
// for values where adjusted_exponent < -6, e.g. "0E-18" for zero with scale=18).
| DataType::Utf8
)
// Note: Boolean is intentionally absent. Decimal-to-boolean uses a dedicated
// spark_cast_decimal_to_boolean function that compares against a zero decimal of
// the same precision/scale, bypassing the DataFusion cast kernel entirely.
}
macro_rules! cast_float_to_timestamp_impl {
($array:expr, $builder:expr, $primitive_type:ty, $eval_mode:expr) => {{
let arr = $array.as_primitive::<$primitive_type>();
for i in 0..arr.len() {
if arr.is_null(i) {
$builder.append_null();
} else {
let val = arr.value(i) as f64;
// Path 1: NaN/Infinity check - error says TIMESTAMP
if val.is_nan() || val.is_infinite() {
if $eval_mode == EvalMode::Ansi {
return Err(SparkError::CastInvalidValue {
value: val.to_string(),
from_type: "DOUBLE".to_string(),
to_type: "TIMESTAMP".to_string(),
});
}
$builder.append_null();
} else {
// Path 2: Multiply then check overflow - error says BIGINT
let micros = val * MICROS_PER_SECOND as f64;
if micros.floor() <= i64::MAX as f64 && micros.ceil() >= i64::MIN as f64 {
$builder.append_value(micros as i64);
} else {
if $eval_mode == EvalMode::Ansi {
let value_str = if micros.is_infinite() {
if micros.is_sign_positive() {
"Infinity".to_string()
} else {
"-Infinity".to_string()
}
} else if micros.is_nan() {
"NaN".to_string()
} else {
format!("{:e}", micros).to_uppercase() + "D"
};
return Err(SparkError::CastOverFlow {
value: value_str,
from_type: "DOUBLE".to_string(),
to_type: "BIGINT".to_string(),
});
}
$builder.append_null();
}
}
}
}
}};
}
/// A float width that Java renders through `Float.toString` / `Double.toString`.
///
/// `num::Float` supplies the arithmetic predicates; the two widths differ only in the plain-notation
/// window's endpoints and in the literal text of the smallest subnormal, which Java's algorithm
/// spells with more digits than a shortest-round-trip formatter produces.
///
/// Sealed: the crate root re-exports this module, and `f32` and `f64` are the only widths Java has.
pub trait JavaFloatString: sealed::Sealed + Float + fmt::Display + fmt::UpperExp {
/// `Float.MIN_VALUE` / `Double.MIN_VALUE`: the value one ULP above zero, the one Java does not
/// render shortest. `Float::min_positive_value` is the smallest *normal*, so this has no `num`
/// equivalent.
const MIN_SUBNORMAL: Self;
/// `MIN_SUBNORMAL` as Java spells it.
const MIN_SUBNORMAL_TEXT: &'static str;
/// Plain notation covers `[0.001, 10^7)`; anything outside it is scientific.
const PLAIN_LOWER: Self;
const PLAIN_UPPER: Self;
}
mod sealed {
pub trait Sealed {}
impl Sealed for f32 {}
impl Sealed for f64 {}
}
impl JavaFloatString for f32 {
const MIN_SUBNORMAL: Self = f32::from_bits(1);
const MIN_SUBNORMAL_TEXT: &'static str = "1.4E-45";
const PLAIN_LOWER: Self = 0.001;
const PLAIN_UPPER: Self = 10000000.0;
}
impl JavaFloatString for f64 {
const MIN_SUBNORMAL: Self = f64::from_bits(1);
const MIN_SUBNORMAL_TEXT: &'static str = "4.9E-324";
const PLAIN_LOWER: Self = 0.001;
const PLAIN_UPPER: Self = 10000000.0;
}
/// Writes `value` as Java's `Float.toString` / `Double.toString` renders it.
///
/// If the absolute value is less than 10,000,000 and greater or equal than 0.001, the result is
/// expressed without scientific notation with at least one digit on either side of the decimal
/// point. Otherwise the value is a mantissa followed by `E` and an exponent, the mantissa having
/// an optional leading minus sign followed by one digit to the left of the decimal point and the
/// minimal number of digits greater than zero to the right.
/// Source: <https://docs.databricks.com/en/sql/language-manual/functions/cast.html>
///
/// Rust's own `Display` and `UpperExp` give the same digits but drop a whole coefficient's
/// fractional zero (`1` for `1.0`) and never switch to an exponent, so `Double.MAX_VALUE` would
/// render as 309 digits. Both matter beyond cosmetics: Spark spells a `cast(double as string)`
/// this way, and iceberg-java spells a float or double partition directory this way, where the
/// unabbreviated form overruns the filesystem's limit on one path component.
///
/// Rust's digits are shortest-round-trip, which is what JDK 19+ `Double.toString` produces. Earlier
/// JDKs sometimes emit a longer string (JDK-4511638: `2.0E23` came out as `1.9999999999999998E23`),
/// and those values are not corrected for here. The smallest subnormal, which every JDK spells as
/// `4.9E-324` rather than the shortest `5E-324`, is.
///
/// Errors only if `out` does. Writing into a `String` or an arrow string builder cannot fail, which
/// is what lets the callers of this function discard the result.
pub fn write_java_float_string<T: JavaFloatString, W: fmt::Write>(
value: T,
out: &mut W,
) -> fmt::Result {
let abs = value.abs();
if (T::PLAIN_LOWER..T::PLAIN_UPPER).contains(&abs) || abs.is_zero() {
write!(out, "{value}")?;
if value.fract().is_zero() {
// Java always renders a fractional digit; Rust omits it.
out.write_str(".0")?;
}
Ok(())
} else if !value.is_finite() {
// NaN and the infinities are excluded by the range check above.
out.write_str(if value.is_nan() {
"NaN"
} else if value.is_sign_negative() {
"-Infinity"
} else {
"Infinity"
})
} else if abs == T::MIN_SUBNORMAL {
if value.is_sign_negative() {
out.write_str("-")?;
}
out.write_str(T::MIN_SUBNORMAL_TEXT)
} else {
// The coefficient has to be inspected before any of it is emitted, so it is formatted
// into a stack buffer rather than into `out`, which may not be rewindable.
let mut scratch = ExponentBuf::default();
write!(scratch, "{value:E}")?;
let text = scratch.as_str();
match text.split_once('E') {
Some((coefficient, exponent)) if !coefficient.contains('.') => {
// Java keeps the fractional digit Rust drops from a whole coefficient.
out.write_str(coefficient)?;
out.write_str(".0E")?;
out.write_str(exponent)
}
_ => out.write_str(text),
}
}
}
/// `Float.toString` / `Double.toString` of `value` as an owned `String`, for callers that need one
/// rendering rather than a column of them.
pub fn java_float_string<T: JavaFloatString>(value: T) -> String {
let mut out = String::new();
// Cannot fail; see `write_java_float_string`.
let _ = write_java_float_string(value, &mut out);
out
}
/// Scratch space for one `{:E}` rendering. `{:E}` emits at most 17 significant digits for an
/// `f64`, so the longest output is sign + digit + point + 16 digits + `E` + sign + 3 exponent
/// digits, 24 bytes.
#[derive(Default)]
struct ExponentBuf {
bytes: [u8; 32],
len: usize,
}
impl ExponentBuf {
fn as_str(&self) -> &str {
// Only `{:E}` output, which is ASCII, is ever written.
std::str::from_utf8(&self.bytes[..self.len]).expect("ascii float text")
}
}
impl fmt::Write for ExponentBuf {
fn write_str(&mut self, s: &str) -> fmt::Result {
let end = self.len + s.len();
// Unreachable for `{:E}` of an f32 or f64 (see the size bound above); returning an error
// rather than panicking keeps a future caller's mistake out of the JNI boundary.
let target = self.bytes.get_mut(self.len..end).ok_or(fmt::Error)?;
target.copy_from_slice(s.as_bytes());
self.len = end;
Ok(())
}
}
/// Casts a float array to strings the way Spark's `cast(float as string)` does, which is Java's
/// `Float.toString` / `Double.toString`.
fn spark_cast_float_to_utf8<T, OffsetSize>(from: &dyn Array) -> SparkResult<ArrayRef>
where
T: ArrowPrimitiveType,
T::Native: JavaFloatString,
OffsetSize: OffsetSizeTrait,
{
let array = from.as_primitive::<T>();
// Values are formatted straight into the builder, so no intermediate String is allocated per
// row. Capacity hint matches arrow-rs's own AVERAGE_STRING_LENGTH (16 bytes / value) so
// typical fractional and scientific outputs like "1234.5678" or "-1.4E-45" do not force a
// mid-loop grow.
let mut builder =
GenericStringBuilder::<OffsetSize>::with_capacity(array.len(), array.len() * 16);
for value in array.iter() {
match value {
None => builder.append_null(),
Some(value) => {
// Cannot fail; see `write_java_float_string`.
let _ = write_java_float_string(value, &mut builder);
builder.append_value("");
}
}
}
Ok(Arc::new(builder.finish()))
}
// eval mode is not needed since all ints can be implemented in binary format
#[macro_export]
macro_rules! cast_whole_num_to_binary {
($array:expr, $primitive_type:ty, $byte_size:expr) => {{
let input_arr = $array
.as_any()
.downcast_ref::<$primitive_type>()
.ok_or_else(|| SparkError::Internal("Expected numeric array".to_string()))?;
let len = input_arr.len();
let mut builder = BinaryBuilder::with_capacity(len, len * $byte_size);
for i in 0..input_arr.len() {
if input_arr.is_null(i) {
builder.append_null();
} else {
builder.append_value(input_arr.value(i).to_be_bytes());
}
}
Ok(Arc::new(builder.finish()) as ArrayRef)
}};
}
macro_rules! cast_int_to_timestamp_impl {
($array:expr, $builder:expr, $primitive_type:ty) => {{
let arr = $array.as_primitive::<$primitive_type>();
for i in 0..arr.len() {
if arr.is_null(i) {
$builder.append_null();
} else {
// saturating_mul limits to i64::MIN/MAX on overflow instead of panicking,
// which could occur when converting extreme values (e.g., Long.MIN_VALUE)
// matching spark behavior (irrespective of EvalMode)
let micros = (arr.value(i) as i64).saturating_mul(MICROS_PER_SECOND);
$builder.append_value(micros);
}
}
}};
}
macro_rules! cast_int_to_int_macro {
(
$array: expr,
$eval_mode:expr,
$from_arrow_primitive_type: ty,
$to_arrow_primitive_type: ty,
$to_native_type: ty,
$spark_from_data_type_name: expr,
$spark_to_data_type_name: expr,
$spark_int_literal_suffix: expr
) => {{
let cast_array = $array
.as_any()
.downcast_ref::<PrimitiveArray<$from_arrow_primitive_type>>()
.unwrap();
let output_array: PrimitiveArray<$to_arrow_primitive_type> = match $eval_mode {
// Legacy narrowing keeps the low-order bits of the source value, which is what a
// wrapping `as` conversion does. Being infallible, it maps the values buffer in a
// single pass and carries the null buffer over untouched.
EvalMode::Legacy => {
cast_array.unary::<_, $to_arrow_primitive_type>(|value| value as $to_native_type)
}
// `try_unary` only applies the conversion to non-null slots, so an out-of-range
// value sitting under a null cannot raise a spurious overflow error.
_ => cast_array.try_unary::<_, $to_arrow_primitive_type, SparkError>(|value| {
<$to_native_type>::try_from(value).map_err(|_| {
cast_overflow(
&(value.to_string() + $spark_int_literal_suffix),
$spark_from_data_type_name,
$spark_to_data_type_name,
)
})
})?,
};
Ok(Arc::new(output_array) as ArrayRef)
}};
}
/// Spark's ANSI range check for casting a floating point value to a 32-bit or 64-bit integer,
/// from `FloatExactNumeric`/`DoubleExactNumeric.toInt`/`toLong` in Spark's `numerics.scala`:
/// `Math.floor(x) <= MaxValue && Math.ceil(x) >= MinValue`. The JVM widens a float source to
/// double for `Math.floor`/`Math.ceil` and converts the integer bounds to double for the
/// comparison, so the check is evaluated in `f64` here as well. A value that passes it is then
/// truncated towards zero with saturation by `as`, exactly like the JVM's `d2i`/`d2l`, so the
/// exactly representable bounds are valid inputs; note that `i64::MAX as f64 == 2^63`, so `±2^63`
/// passes the check and saturates to `i64::MAX`/`i64::MIN`, as it does in Spark. NaN fails both
/// comparisons and the infinities fail one of them.
fn spark_float_fits_integral<F: Into<f64>>(value: F, min: f64, max: f64) -> bool {
let value: f64 = value.into();
value.floor() <= max && value.ceil() >= min
}
// When Spark casts to Byte/Short Types, it does not cast directly to Byte/Short.
// It casts to Int first and then to Byte/Short. Because of potential overflows in the Int cast,
// this can cause unexpected Short/Byte cast results. Replicate this behavior.
macro_rules! cast_float_to_int16_down {
(
$array:expr,
$eval_mode:expr,
$src_array_type:ty,
$dest_array_type:ty,
$rust_src_type:ty,
$rust_dest_type:ty,
$src_type_str:expr,
$dest_type_str:expr,
$format_str:expr
) => {{
let cast_array = $array
.as_any()
.downcast_ref::<$src_array_type>()
.expect(concat!("Expected a ", stringify!($src_array_type)));
let output_array = match $eval_mode {
EvalMode::Ansi => cast_array
.iter()
.map(|value| match value {
Some(value) => {
// Spark's ANSI cast converts to Int with the exact 32-bit range check
// first and then requires the truncated Int to round-trip through the
// narrower type.
if !spark_float_fits_integral(value, i32::MIN as f64, i32::MAX as f64) {
return Err(cast_overflow(
&format!($format_str, value).replace("e", "E"),
$src_type_str,
$dest_type_str,
));
}
let i32_value = value as i32;
<$rust_dest_type>::try_from(i32_value)
.map_err(|_| {
cast_overflow(
&format!($format_str, value).replace("e", "E"),
$src_type_str,
$dest_type_str,
)
})
.map(Some)
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
_ => cast_array
.iter()
.map(|value| match value {
Some(value) => {
let i32_value = value as i32;
Ok::<Option<$rust_dest_type>, SparkError>(Some(
i32_value as $rust_dest_type,
))
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
};
Ok(Arc::new(output_array) as ArrayRef)
}};
}
macro_rules! cast_float_to_int32_up {
(
$array:expr,
$eval_mode:expr,
$src_array_type:ty,
$dest_array_type:ty,
$rust_src_type:ty,
$rust_dest_type:ty,
$src_type_str:expr,
$dest_type_str:expr,
$format_str:expr
) => {{
let cast_array = $array
.as_any()
.downcast_ref::<$src_array_type>()
.expect(concat!("Expected a ", stringify!($src_array_type)));
let output_array = match $eval_mode {
EvalMode::Ansi => cast_array
.iter()
.map(|value| match value {
Some(value) => {
if !spark_float_fits_integral(
value,
<$rust_dest_type>::MIN as f64,
<$rust_dest_type>::MAX as f64,
) {
return Err(cast_overflow(
&format!($format_str, value).replace("e", "E"),
$src_type_str,
$dest_type_str,
));
}
// `as` truncates towards zero and saturates like the JVM's d2i/d2l
Ok(Some(value as $rust_dest_type))
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
_ => cast_array
.iter()
.map(|value| match value {
Some(value) => {
Ok::<Option<$rust_dest_type>, SparkError>(Some(value as $rust_dest_type))
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
};
Ok(Arc::new(output_array) as ArrayRef)
}};
}
// When Spark casts to Byte/Short Types, it does not cast directly to Byte/Short.
// It casts to Int first and then to Byte/Short. Because of potential overflows in the Int cast,
// this can cause unexpected Short/Byte cast results. Replicate this behavior.
macro_rules! cast_decimal_to_int16_down {
(
$array:expr,
$eval_mode:expr,
$dest_array_type:ty,
$rust_dest_type:ty,
$dest_type_str:expr,
$precision:expr,
$scale:expr
) => {{
let cast_array = $array
.as_any()
.downcast_ref::<Decimal128Array>()
.expect("Expected a Decimal128ArrayType");
let output_array = match $eval_mode {
EvalMode::Ansi => cast_array
.iter()
.map(|value| match value {
Some(value) => {
let divisor = 10_i128.pow($scale as u32);
let truncated = value / divisor;
let is_overflow = truncated.abs() > i32::MAX.into();
if is_overflow {
return Err(cast_overflow(
&format!(
"{}BD",
format_decimal_str(
&value.to_string(),
$precision as usize,
$scale
)
),
&format!("DECIMAL({},{})", $precision, $scale),
$dest_type_str,
));
}
let i32_value = truncated as i32;
<$rust_dest_type>::try_from(i32_value)
.map_err(|_| {
cast_overflow(
&format!(
"{}BD",
format_decimal_str(
&value.to_string(),
$precision as usize,
$scale
)
),
&format!("DECIMAL({},{})", $precision, $scale),
$dest_type_str,
)
})
.map(Some)
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
_ => cast_array
.iter()
.map(|value| match value {
Some(value) => {
let divisor = 10_i128.pow($scale as u32);
let i32_value = (value / divisor) as i32;
Ok::<Option<$rust_dest_type>, SparkError>(Some(
i32_value as $rust_dest_type,
))
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
};
Ok(Arc::new(output_array) as ArrayRef)
}};
}
macro_rules! cast_decimal_to_int32_up {
(
$array:expr,
$eval_mode:expr,
$dest_array_type:ty,
$rust_dest_type:ty,
$dest_type_str:expr,
$max_dest_val:expr,
$precision:expr,
$scale:expr
) => {{
let cast_array = $array
.as_any()
.downcast_ref::<Decimal128Array>()
.expect("Expected a Decimal128ArrayType");
let output_array = match $eval_mode {
EvalMode::Ansi => cast_array
.iter()
.map(|value| match value {
Some(value) => {
let divisor = 10_i128.pow($scale as u32);
let truncated = value / divisor;
let is_overflow = truncated.abs() > $max_dest_val.into();
if is_overflow {
return Err(cast_overflow(
&format!(
"{}BD",
format_decimal_str(
&value.to_string(),
$precision as usize,
$scale
)
),
&format!("DECIMAL({},{})", $precision, $scale),
$dest_type_str,
));
}
Ok(Some(truncated as $rust_dest_type))
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
_ => cast_array
.iter()
.map(|value| match value {
Some(value) => {
let divisor = 10_i128.pow($scale as u32);
let truncated = value / divisor;
Ok::<Option<$rust_dest_type>, SparkError>(Some(
truncated as $rust_dest_type,
))
}
None => Ok(None),
})
.collect::<Result<$dest_array_type, _>>()?,
};
Ok(Arc::new(output_array) as ArrayRef)
}};
}
/// Casts a Decimal128 array to string using Java's BigDecimal.toString() semantics,
/// which is Spark's LEGACY eval mode behavior. Plain notation when scale >= 0 and
/// adjusted_exponent >= -6, otherwise scientific notation (e.g. "0E-18" for zero
/// with scale=18, since adjusted_exponent = -18 + 0 = -18 < -6).
///
/// TRY and ANSI modes produce plain notation via DataFusion's cast instead.
pub(crate) fn cast_decimal128_to_utf8(array: &ArrayRef, scale: i8) -> SparkResult<ArrayRef> {
let decimal_array = array
.as_any()
.downcast_ref::<Decimal128Array>()
.expect("Expected a Decimal128Array");
let mut builder = StringBuilder::with_capacity(decimal_array.len(), decimal_array.len() * 16);
// Reuse the output and digit buffers across rows to keep the loop allocation-free.
let mut buf = String::with_capacity(48);
let mut digits = [0u8; MAX_COEFF_DIGITS];
for opt_val in decimal_array.iter() {
match opt_val {
None => builder.append_null(),
Some(unscaled) => {
buf.clear();
decimal128_to_java_string(unscaled, scale, &mut digits, &mut buf);
builder.append_value(&buf);
}
}
}
Ok(Arc::new(builder.finish()))
}
/// The largest power of ten that fits in a `u64`.
const POW10_19: u128 = 10_000_000_000_000_000_000;
/// Digits in the largest `u128`, which bounds the coefficient of any Decimal128.
const MAX_COEFF_DIGITS: usize = 39;
/// Renders the base-10 digits of `value` into `buf`, returning them without leading zeroes
/// (except for `value == 0`, which renders as `"0"`).
fn render_digits(mut value: u128, buf: &mut [u8; MAX_COEFF_DIGITS]) -> &str {
let mut pos = buf.len();
// Peel off 19 digits at a time so that all but the final group is produced with 64-bit
// arithmetic, which is much cheaper than repeated 128-bit division. Every group taken here
// has a non-zero quotient above it, so emitting its leading zeroes is correct.
while value >= POW10_19 {
let mut group = (value % POW10_19) as u64;
value /= POW10_19;
for _ in 0..19 {
pos -= 1;
buf[pos] = b'0' + (group % 10) as u8;
group /= 10;
}
}
let mut rest = value as u64;
loop {
pos -= 1;
buf[pos] = b'0' + (rest % 10) as u8;
rest /= 10;
if rest == 0 {
break;
}
}
// Only ASCII digits were written above.
std::str::from_utf8(&buf[pos..]).expect("ascii digits")
}
/// Formats a Decimal128 unscaled value into `out` matching Java's BigDecimal.toString():
/// - Plain notation when scale >= 0 and adjusted_exponent >= -6
/// - Scientific notation otherwise
///
/// adjusted_exponent = -scale + (numDigits - 1)
///
/// `digits` is scratch space for the coefficient; the caller reuses one buffer across rows.
fn decimal128_to_java_string(
unscaled: i128,
scale: i8,
digits: &mut [u8; MAX_COEFF_DIGITS],
out: &mut String,
) {
use std::fmt::Write;
let coeff = render_digits(unscaled.unsigned_abs(), digits);
let num_digits = coeff.len();
let adj_exp = -(scale as i64) + (num_digits as i64 - 1);
if unscaled < 0 {
out.push('-');
}
if scale >= 0 && adj_exp >= -6 {
let scale = scale as usize;
if scale == 0 {
out.push_str(coeff);
} else if num_digits > scale {
let (int_part, frac_part) = coeff.split_at(num_digits - scale);
out.push_str(int_part);
out.push('.');
out.push_str(frac_part);
} else {
out.push_str("0.");
for _ in 0..scale - num_digits {
out.push('0');
}
out.push_str(coeff);
}
} else {
if num_digits > 1 {
out.push_str(&coeff[..1]);
out.push('.');
out.push_str(&coeff[1..]);
} else {
out.push_str(coeff);
}
out.push('E');
if adj_exp > 0 {
out.push('+');
}
write!(out, "{adj_exp}").unwrap();
}
}
pub(crate) fn spark_cast_float64_to_utf8<OffsetSize>(
from: &dyn Array,
_eval_mode: EvalMode,
) -> SparkResult<ArrayRef>
where
OffsetSize: OffsetSizeTrait,
{
spark_cast_float_to_utf8::<Float64Type, OffsetSize>(from)
}
pub(crate) fn spark_cast_float32_to_utf8<OffsetSize>(
from: &dyn Array,
_eval_mode: EvalMode,
) -> SparkResult<ArrayRef>
where
OffsetSize: OffsetSizeTrait,
{
spark_cast_float_to_utf8::<Float32Type, OffsetSize>(from)
}
fn cast_int_to_decimal128_internal<T>(
array: &PrimitiveArray<T>,
precision: u8,
scale: i8,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef>
where
T: ArrowPrimitiveType,
T::Native: Into<i128>,
{
let multiplier = 10_i128.pow(scale as u32);
// Single spelling of the "does this value fit at the target precision after scaling" check,
// shared between the vectorized pass and the ANSI rescan below.
let fits = |v: i128| -> Option<i128> {
v.checked_mul(multiplier)
.filter(|scaled| is_validate_decimal_precision(*scaled, precision))
};
// Single vectorized pass: a value that overflows the multiply or does not fit the output
// precision maps to null. `unary_opt` only applies the closure to non-null slots and carries
// the input null buffer over, replacing the per-element builder loop without a second pass.
let result: Decimal128Array = array.unary_opt::<_, Decimal128Type>(|v| fits(v.into()));
// ANSI must raise on out-of-range values instead of nulling them. `unary_opt` only nulls
// non-null inputs that overflow, so a null count beyond the input's signals an overflow to
// report. This check is O(1); the element-wise rescan runs only on the rare error path and
// reports the first offending value with Spark's exact error.
if eval_mode == EvalMode::Ansi && result.null_count() > array.null_count() {
for i in 0..array.len() {
if !array.is_null(i) {
let v: i128 = array.value(i).into();
if fits(v).is_none() {
return Err(SparkError::NumericValueOutOfRange {
value: v.to_string(),
precision,
scale,
});
}
}
}
}
Ok(Arc::new(result.with_precision_and_scale(precision, scale)?))
}
pub(crate) fn cast_int_to_decimal128(
array: &dyn Array,
eval_mode: EvalMode,
from_type: &DataType,
to_type: &DataType,
precision: u8,
scale: i8,
) -> SparkResult<ArrayRef> {
match (from_type, to_type) {
(DataType::Int8, DataType::Decimal128(_p, _s)) => {
cast_int_to_decimal128_internal::<Int8Type>(
array.as_primitive::<Int8Type>(),
precision,
scale,
eval_mode,
)
}
(DataType::Int16, DataType::Decimal128(_p, _s)) => {
cast_int_to_decimal128_internal::<Int16Type>(
array.as_primitive::<Int16Type>(),
precision,
scale,
eval_mode,
)
}
(DataType::Int32, DataType::Decimal128(_p, _s)) => {
cast_int_to_decimal128_internal::<Int32Type>(
array.as_primitive::<Int32Type>(),
precision,
scale,
eval_mode,
)
}
(DataType::Int64, DataType::Decimal128(_p, _s)) => {
cast_int_to_decimal128_internal::<Int64Type>(
array.as_primitive::<Int64Type>(),
precision,
scale,
eval_mode,
)
}
_ => Err(SparkError::Internal(format!(
"Unsupported cast from datatype : {}",
from_type
))),
}
}
pub(crate) fn spark_cast_int_to_int(
array: &dyn Array,
eval_mode: EvalMode,
from_type: &DataType,
to_type: &DataType,
) -> SparkResult<ArrayRef> {
match (from_type, to_type) {
(DataType::Int64, DataType::Int32) => cast_int_to_int_macro!(
array, eval_mode, Int64Type, Int32Type, i32, "BIGINT", "INT", "L"
),
(DataType::Int64, DataType::Int16) => cast_int_to_int_macro!(
array, eval_mode, Int64Type, Int16Type, i16, "BIGINT", "SMALLINT", "L"
),
(DataType::Int64, DataType::Int8) => cast_int_to_int_macro!(
array, eval_mode, Int64Type, Int8Type, i8, "BIGINT", "TINYINT", "L"
),
(DataType::Int32, DataType::Int16) => cast_int_to_int_macro!(
array, eval_mode, Int32Type, Int16Type, i16, "INT", "SMALLINT", ""
),
(DataType::Int32, DataType::Int8) => {
cast_int_to_int_macro!(array, eval_mode, Int32Type, Int8Type, i8, "INT", "TINYINT", "")
}
(DataType::Int16, DataType::Int8) => cast_int_to_int_macro!(
array, eval_mode, Int16Type, Int8Type, i8, "SMALLINT", "TINYINT", "S"
),
_ => unreachable!(
"{}",
format!("invalid integer type {to_type} in cast from {from_type}")
),
}
}
pub(crate) fn spark_cast_decimal_to_boolean(array: &dyn Array) -> SparkResult<ArrayRef> {
let decimal_array = array.as_primitive::<Decimal128Type>();
// All-null fast path: skips the zero-scalar construction for a batch whose output
// is trivially all-null. Also covers all-null `Decimal128(0, 0)`, which the default
// path cannot handle (see the precision-zero fast path below).
if decimal_array.null_count() == decimal_array.len() {
return Ok(Arc::new(BooleanArray::new_null(decimal_array.len())));
}
// Precision-zero fast path. Arrow rejects `precision == 0` in `with_precision_and_scale`,
// so the default `neq`-against-zero path cannot round-trip a `Decimal128(0, 0)` batch
// even when it has valid slots. Spark reaches this shape via JVM-side writers that do
// not validate precision (e.g. a UDF returning `BigInteger.ZERO` written through
// `DecimalVector.setSafe(long)`). The type contract says only 0 is representable, but
// we still cast the raw i128 payload (`v != 0`) rather than hard-coding `false`, so an
// out-of-contract non-zero value in a valid slot still round-trips correctly.
if decimal_array.precision() == 0 {
let values: BooleanBuffer = decimal_array.values().iter().map(|&v| v != 0).collect();
return Ok(Arc::new(BooleanArray::new(
values,
decimal_array.nulls().cloned(),
)));
}
// Arrow has no Decimal-to-Boolean cast. `neq` against a zero of the same
// precision/scale is exactly `!value.is_zero()`, including null handling.
let zero = Scalar::new(
Decimal128Array::from(vec![0i128])
.with_precision_and_scale(decimal_array.precision(), decimal_array.scale())?,
);
Ok(Arc::new(neq(decimal_array, &zero)?))
}
/// Powers of ten that are exactly representable as `f64` (`10^n = 2^n * 5^n` and `5^22 < 2^53`);
/// the same table as Java's `BigDecimal.DOUBLE_10_POW`.
const F64_EXACT_POW10: [f64; 23] = [
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16,
1e17, 1e18, 1e19, 1e20, 1e21, 1e22,
];
/// Powers of ten that are exactly representable as `f32` (`5^10 < 2^24`); the same table as
/// Java's `BigDecimal.FLOAT_10_POW`.
const F32_EXACT_POW10: [f32; 11] = [1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10];
/// Largest integer magnitude that converts to `f64` without rounding.
const F64_EXACT_INT_LIMIT: u128 = 1 << 53;
/// Largest integer magnitude that converts to `f32` without rounding.
const F32_EXACT_INT_LIMIT: u128 = 1 << 24;
/// Converts a Decimal128 value to the `f64` nearest to its exact decimal value (ties to even),
/// matching Spark's `Decimal.toDouble`, i.e. `java.math.BigDecimal.doubleValue()`.
///
/// DataFusion's kernel computes `(unscaled as f64) / 10^scale`, which rounds the unscaled value
/// first and the quotient second; once `|unscaled| > 2^53` (any `DECIMAL(38,18)` value of
/// magnitude `>= 0.01`) the two roundings can land one ulp away from the correctly rounded
/// result, e.g. `12345.6789` becomes `12345.678899999999`.
pub(crate) fn decimal128_to_f64(unscaled: i128, scale: i8) -> f64 {
if scale == 0 {
return unscaled as f64;
}
// Fast path (also Java's): when the unscaled value and the power of ten are both exact
// doubles, a single IEEE division or multiplication is correctly rounded.
if unscaled.unsigned_abs() <= F64_EXACT_INT_LIMIT {
let m = unscaled as f64;
match scale {
1..=22 => return m / F64_EXACT_POW10[scale as usize],
-22..=-1 => return m * F64_EXACT_POW10[scale.unsigned_abs() as usize],
_ => {}
}
}
parse_exact_decimal(unscaled, scale)
}
/// Converts a Decimal128 value to the `f32` nearest to its exact decimal value (ties to even),
/// matching Spark's `Decimal.toFloat`, i.e. `java.math.BigDecimal.floatValue()`.
///
/// The exact value is rounded straight to `f32`, never through an `f64` intermediate, because
/// rounding twice is observable: `16777217.0000000001` is `16777218` as a float, but narrowing
/// the nearest `f64` (`16777217.0`, a tie) gives `16777216`.
pub(crate) fn decimal128_to_f32(unscaled: i128, scale: i8) -> f32 {
if scale == 0 {
return unscaled as f32;
}
if unscaled.unsigned_abs() <= F32_EXACT_INT_LIMIT {
let m = unscaled as f32;
match scale {
1..=10 => return m / F32_EXACT_POW10[scale as usize],
-10..=-1 => return m * F32_EXACT_POW10[scale.unsigned_abs() as usize],
_ => {}
}
}
parse_exact_decimal(unscaled, scale)
}
/// Rounds the exact value `unscaled * 10^-scale` once, by formatting it as `<unscaled>e<-scale>`
/// into a stack buffer and parsing it with Rust's correctly rounded float parser. This mirrors
/// Java's fallback of `Double.parseDouble(BigDecimal.toString())` /
/// `Float.parseFloat(BigDecimal.toString())`, which likewise round the exact decimal directly to
/// the target width; a value beyond the target's range parses to an infinity, as in Java.
fn parse_exact_decimal<F>(unscaled: i128, scale: i8) -> F
where
F: std::str::FromStr,
F::Err: std::fmt::Debug,
{
use std::io::Write;
// Sign and up to 39 digits, 'e', and an exponent of up to 4 characters (`-127`).
let mut buf = [0u8; 48];
let mut cursor = &mut buf[..];
write!(cursor, "{unscaled}e{}", -i32::from(scale)).expect("buffer holds any i128 and i8");
let remaining = cursor.len();
let text = std::str::from_utf8(&buf[..buf.len() - remaining]).expect("ascii");
text.parse::<F>().expect("well-formed float literal")
}
/// Casts a Decimal128 array to Float64 with `BigDecimal.doubleValue()` semantics; see
/// [`decimal128_to_f64`]. The conversion is total and cannot overflow, so every eval mode
/// behaves alike and the input null buffer carries over unchanged.
pub(crate) fn cast_decimal128_to_float64(array: &dyn Array, scale: i8) -> SparkResult<ArrayRef> {
let input = array.as_primitive::<Decimal128Type>();
let result: Float64Array = input.unary(|v| decimal128_to_f64(v, scale));
Ok(Arc::new(result))
}
/// Casts a Decimal128 array to Float32 with `BigDecimal.floatValue()` semantics; see
/// [`decimal128_to_f32`]. Every Decimal128 magnitude (`< 2^127`) is below `f32::MAX`, so for
/// the non-negative scales Spark produces the conversion cannot overflow and every eval mode
/// behaves alike.
pub(crate) fn cast_decimal128_to_float32(array: &dyn Array, scale: i8) -> SparkResult<ArrayRef> {
let input = array.as_primitive::<Decimal128Type>();
let result: Float32Array = input.unary(|v| decimal128_to_f32(v, scale));
Ok(Arc::new(result))
}
pub(crate) fn cast_float64_to_decimal128(
array: &dyn Array,
precision: u8,
scale: i8,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
cast_floating_point_to_decimal128::<Float64Type>(array, precision, scale, eval_mode)
}
pub(crate) fn cast_float32_to_decimal128(
array: &dyn Array,
precision: u8,
scale: i8,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
cast_floating_point_to_decimal128::<Float32Type>(array, precision, scale, eval_mode)
}
fn cast_floating_point_to_decimal128<T: ArrowPrimitiveType>(
array: &dyn Array,
precision: u8,
scale: i8,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef>
where
<T as ArrowPrimitiveType>::Native: AsPrimitive<f64>,
{
let input = array.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();
let mul = 10_f64.powi(scale as i32);
// Perf-only early out: both paths below already map overflow to null, but this skips
// the exact-path string formatting for values (including ±infinity) whose scaled
// magnitude clearly exceeds the precision limit no matter what their exact decimal
// digits are. The slack generously covers the same scaling error that TIE_GUARD below
// bounds, so borderline values still take the exact path.
let overflow_bound = 10_f64.powi(precision as i32) * (1.0 + 1e-12);
// A value with no decimal form (NaN / infinity) or that does not fit the output
// precision maps to null. `unary_opt` only applies the closure to non-null slots and
// carries the input null buffer over. Floats are widened to double first because Spark
// casts through `x.toDouble` before converting to a string.
let result: Decimal128Array = input.unary_opt::<_, Decimal128Type>(|v| {
let f: f64 = v.as_();
let x = f * mul;
if x.abs() > overflow_bound {
// also catches ±infinity inputs, which Spark nulls
return None;
}
// Rounding the binary product agrees with Spark's string-based result whenever
// every value within the combined error of the shortest-string representation,
// the multiplication, and `powi` (a few ulp, at most |x| * 2^-50) rounds to the
// same integer: |x - round(x)| + |x| * TIE_GUARD < 0.5, with TIE_GUARD at 4x the
// error bound. For |x| > 2^47 the guard exceeds 0.5, so large values, whose
// integer neighborhoods are wider than 1, always take the exact path; for
// high-scale targets (e.g. Decimal(38,18)) that makes the exact path the common
// path, not a rare fallback. NaN fails the comparison and takes the exact path,
// which nulls it.
const TIE_GUARD: f64 = 3.552713678800501e-15; // 2^-48
let r = x.round();
if (x - r).abs() + x.abs() * TIE_GUARD < 0.5 {
return r
.to_i128()
.filter(|u| is_validate_decimal_precision(*u, precision));
}
float_to_decimal128(f, precision, scale)
});
// ANSI must raise on out-of-range values instead of nulling them, but NaN / infinity
// stay null even in ANSI mode: Spark catches the NumberFormatException thrown by
// `Decimal(double)` and returns null before the ANSI overflow check runs. `unary_opt`
// only nulls non-null inputs, so a null count beyond the input's signals a potential
// overflow; the element-wise rescan runs only on that rare path and reports the first
// finite offending value with Spark's exact error.
if eval_mode == EvalMode::Ansi && result.null_count() > input.null_count() {
for i in 0..input.len() {
if !input.is_null(i) && result.is_null(i) {
let input_value: f64 = input.value(i).as_();
if input_value.is_finite() {
return Err(SparkError::NumericValueOutOfRange {
value: input_value.to_string(),
precision,
scale,
});
}
}
}
}
Ok(Arc::new(result.with_precision_and_scale(precision, scale)?))
}
/// Convert a double to a decimal unscaled value with Spark semantics.
///
/// Spark converts through `BigDecimal(Double.toString(d)).setScale(scale, HALF_UP)`: it
/// rounds the shortest decimal string form of the value, not its exact binary expansion.
/// The two disagree for values like 0.5153125 whose binary value (0.51531249999...) sits
/// just below the rounding tie that the string form lands on, so a plain
/// `(f * 10^scale).round()` produces results that differ from Spark.
///
/// Returns `None` for NaN / infinity and for results that do not fit `precision`.
fn float_to_decimal128(f: f64, precision: u8, scale: i8) -> Option<i128> {
if !f.is_finite() {
return None;
}
// Shortest round-trip decimal form, same digits as Java's Double.toString
let mut buf = ryu::Buffer::new();
let (mantissa, exp10) = parse_decimal_notation(buf.format_finite(f))?;
// value = mantissa * 10^exp10, so unscaled = round(mantissa * 10^(exp10 + scale))
let shift = exp10 + scale as i32;
let unscaled = if shift >= 0 {
// Overflowing i128 here means the result cannot fit any decimal precision
mantissa.checked_mul(pow10_i128(shift as u32)?)?
} else {
match pow10_i128(-shift as u32) {
// The mantissa has at most 17 significant digits, so dividing by a power of
// ten too large for i128 always rounds to zero
None => 0,
// Divide with HALF_UP rounding (away from zero on a tie, matching BigDecimal)
Some(div) => div_round_half_up_i128(mantissa, div),
}
};
is_validate_decimal_precision(unscaled, precision).then_some(unscaled)
}
/// Parse ryu's `[-]digits[.digits][e[-]digits]` output into an integer mantissa and a
/// base-10 exponent such that the value equals `mantissa * 10^exp10`.
///
/// Returns `None` if the mantissa does not fit an `i128`, which cannot happen for ryu
/// output: a shortest-form double carries at most 17 significant digits.
fn parse_decimal_notation(s: &str) -> Option<(i128, i32)> {
let (digits, exp10) = match s.split_once('e') {
Some((digits, exp)) => (digits, exp.parse::<i32>().expect("exponent from ryu")),
None => (s, 0),
};
let (digits, negative) = match digits.strip_prefix('-') {
Some(unsigned) => (unsigned, true),
None => (digits, false),
};
let (integral, fractional) = match digits.split_once('.') {
Some((integral, fractional)) => (integral, fractional),
None => (digits, ""),
};
// value = (integral * 10^frac_digits + fractional) * 10^(exp10 - frac_digits)
let frac_digits = fractional.len() as i32;
let mantissa = pow10_i128(frac_digits as u32)
.and_then(|p| digits_to_i128(integral.as_bytes())?.checked_mul(p))
.and_then(|v| v.checked_add(digits_to_i128(fractional.as_bytes())?))?;
let mantissa = if negative { -mantissa } else { mantissa };
Some((mantissa, exp10 - frac_digits))
}
pub(crate) fn spark_cast_nonintegral_numeric_to_integral(
array: &dyn Array,
eval_mode: EvalMode,
from_type: &DataType,
to_type: &DataType,
) -> SparkResult<ArrayRef> {
match (from_type, to_type) {
(DataType::Float32, DataType::Int8) => cast_float_to_int16_down!(
array,
eval_mode,
Float32Array,
Int8Array,
f32,
i8,
"FLOAT",
"TINYINT",
"{:e}"
),
(DataType::Float32, DataType::Int16) => cast_float_to_int16_down!(
array,
eval_mode,
Float32Array,
Int16Array,
f32,
i16,
"FLOAT",
"SMALLINT",
"{:e}"
),
(DataType::Float32, DataType::Int32) => cast_float_to_int32_up!(
array,
eval_mode,
Float32Array,
Int32Array,
f32,
i32,
"FLOAT",
"INT",
"{:e}"
),
(DataType::Float32, DataType::Int64) => cast_float_to_int32_up!(
array,
eval_mode,
Float32Array,
Int64Array,
f32,
i64,
"FLOAT",
"BIGINT",
"{:e}"
),
(DataType::Float64, DataType::Int8) => cast_float_to_int16_down!(
array,
eval_mode,
Float64Array,
Int8Array,
f64,
i8,
"DOUBLE",
"TINYINT",
"{:e}D"
),
(DataType::Float64, DataType::Int16) => cast_float_to_int16_down!(
array,
eval_mode,
Float64Array,
Int16Array,
f64,
i16,
"DOUBLE",
"SMALLINT",
"{:e}D"
),
(DataType::Float64, DataType::Int32) => cast_float_to_int32_up!(
array,
eval_mode,
Float64Array,
Int32Array,
f64,
i32,
"DOUBLE",
"INT",
"{:e}D"
),
(DataType::Float64, DataType::Int64) => cast_float_to_int32_up!(
array,
eval_mode,
Float64Array,
Int64Array,
f64,
i64,
"DOUBLE",
"BIGINT",
"{:e}D"
),
(DataType::Decimal128(precision, scale), DataType::Int8) => {
cast_decimal_to_int16_down!(
array, eval_mode, Int8Array, i8, "TINYINT", *precision, *scale
)
}
(DataType::Decimal128(precision, scale), DataType::Int16) => {
cast_decimal_to_int16_down!(
array, eval_mode, Int16Array, i16, "SMALLINT", *precision, *scale
)
}
(DataType::Decimal128(precision, scale), DataType::Int32) => {
cast_decimal_to_int32_up!(
array,
eval_mode,
Int32Array,
i32,
"INT",
i32::MAX,
*precision,
*scale
)
}
(DataType::Decimal128(precision, scale), DataType::Int64) => {
cast_decimal_to_int32_up!(
array,
eval_mode,
Int64Array,
i64,
"BIGINT",
i64::MAX,
*precision,
*scale
)
}
_ => unreachable!(
"{}",
format!("invalid cast from non-integral numeric type: {from_type} to integral numeric type: {to_type}")
),
}
}
pub(crate) fn cast_int_to_timestamp(
array_ref: &ArrayRef,
target_tz: &Option<Arc<str>>,
) -> SparkResult<ArrayRef> {
// Input is seconds since epoch, multiply by MICROS_PER_SECOND to get microseconds.
let mut builder = TimestampMicrosecondBuilder::with_capacity(array_ref.len());
match array_ref.data_type() {
DataType::Int8 => cast_int_to_timestamp_impl!(array_ref, builder, Int8Type),
DataType::Int16 => cast_int_to_timestamp_impl!(array_ref, builder, Int16Type),
DataType::Int32 => cast_int_to_timestamp_impl!(array_ref, builder, Int32Type),
DataType::Int64 => cast_int_to_timestamp_impl!(array_ref, builder, Int64Type),
dt => {
return Err(SparkError::Internal(format!(
"Unsupported type for cast_int_to_timestamp: {:?}",
dt
)))
}
}
Ok(Arc::new(builder.finish().with_timezone_opt(target_tz.clone())) as ArrayRef)
}
pub(crate) fn cast_decimal_to_timestamp(
array_ref: &ArrayRef,
target_tz: &Option<Arc<str>>,
scale: i8,
) -> SparkResult<ArrayRef> {
let arr = array_ref.as_primitive::<Decimal128Type>();
let scale_factor = 10_i128.pow(scale as u32);
let mut builder = TimestampMicrosecondBuilder::with_capacity(arr.len());
for i in 0..arr.len() {
if arr.is_null(i) {
builder.append_null();
} else {
let value = arr.value(i);
// Note: spark's big decimal truncates to
// long value and does not throw error (in all leval modes)
let value_256 = i256::from_i128(value);
let micros_256 = value_256 * i256::from_i128(MICROS_PER_SECOND as i128);
let ts = micros_256 / i256::from_i128(scale_factor);
builder.append_value(ts.as_i128() as i64);
}
}
Ok(Arc::new(builder.finish().with_timezone_opt(target_tz.clone())) as ArrayRef)
}
pub(crate) fn cast_float_to_timestamp(
array_ref: &ArrayRef,
target_tz: &Option<Arc<str>>,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
let mut builder = TimestampMicrosecondBuilder::with_capacity(array_ref.len());
match array_ref.data_type() {
DataType::Float32 => {
cast_float_to_timestamp_impl!(array_ref, builder, Float32Type, eval_mode)
}
DataType::Float64 => {
cast_float_to_timestamp_impl!(array_ref, builder, Float64Type, eval_mode)
}
dt => {
return Err(SparkError::Internal(format!(
"Unsupported type for cast_float_to_timestamp: {:?}",
dt
)))
}
}
Ok(Arc::new(builder.finish().with_timezone_opt(target_tz.clone())) as ArrayRef)
}
#[cfg(test)]
mod tests {
use super::*;
use arrow::array::AsArray;
use arrow::datatypes::TimestampMicrosecondType;
#[test]
fn test_spark_cast_int_to_int_overflow() {
// Test Int64 -> Int32 overflow
let array: ArrayRef = Arc::new(Int64Array::from(vec![
Some(i64::MAX),
Some(i64::MIN),
Some(100),
]));
// Legacy mode should wrap around
let result =
spark_cast_int_to_int(&array, EvalMode::Legacy, &DataType::Int64, &DataType::Int32)
.unwrap();
let int32_array = result.as_primitive::<Int32Type>();
assert_eq!(int32_array.value(2), 100);
// Ansi mode should error on overflow
let result =
spark_cast_int_to_int(&array, EvalMode::Ansi, &DataType::Int64, &DataType::Int32);
assert!(result.is_err());
}
/// Casts `values` from the floating point type `S` to the integral type `D` with
/// `spark_cast_nonintegral_numeric_to_integral` and returns the resulting values.
fn cast_float_to_integral<S: ArrowPrimitiveType, D: ArrowPrimitiveType>(
values: Vec<S::Native>,
eval_mode: EvalMode,
) -> SparkResult<Vec<D::Native>> {
let array = PrimitiveArray::<S>::from_iter_values(values);
let result = spark_cast_nonintegral_numeric_to_integral(
&array,
eval_mode,
&S::DATA_TYPE,
&D::DATA_TYPE,
)?;
Ok(result.as_primitive::<D>().values().to_vec())
}
/// Asserts that each value overflows on its own in ANSI mode with Spark's `CAST_OVERFLOW`
/// error naming the source and target types.
fn assert_ansi_cast_overflow<S: ArrowPrimitiveType, D: ArrowPrimitiveType>(
values: &[S::Native],
from_type: &str,
to_type: &str,
) {
for value in values {
match cast_float_to_integral::<S, D>(vec![*value], EvalMode::Ansi) {
Err(SparkError::CastOverFlow {
from_type: from,
to_type: to,
..
}) => assert_eq!(
(from.as_str(), to.as_str()),
(from_type, to_type),
"unexpected types in the overflow error for {value:?}"
),
other => panic!("expected CAST_OVERFLOW for {value:?}, got {other:?}"),
}
}
}
#[test]
fn test_cast_double_to_int_ansi_boundaries() {
// Spark accepts x when floor(x) <= i32::MAX && ceil(x) >= i32::MIN and truncates it
// towards zero, so the exactly representable bounds and the fractional values inside
// them are valid inputs
assert_eq!(
cast_float_to_integral::<Float64Type, Int32Type>(
vec![
2147483647.0,
-2147483648.0,
2147483647.5,
-2147483648.5,
2147483648.0f64.next_down(),
(-2147483649.0f64).next_up(),
],
EvalMode::Ansi,
)
.unwrap(),
vec![i32::MAX, i32::MIN, i32::MAX, i32::MIN, i32::MAX, i32::MIN]
);
assert_ansi_cast_overflow::<Float64Type, Int32Type>(
&[
2147483648.0,
-2147483649.0,
2147483648.5,
-2147483649.5,
1e10,
f64::MAX,
f64::MIN,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
],
"DOUBLE",
"INT",
);
// the offending value is rendered like Spark's Double.toString plus the D suffix
match cast_float_to_integral::<Float64Type, Int32Type>(vec![2147483648.0], EvalMode::Ansi) {
Err(SparkError::CastOverFlow { value, .. }) => assert_eq!(value, "2.147483648E9D"),
other => panic!("expected CAST_OVERFLOW, got {other:?}"),
}
}
#[test]
fn test_cast_double_to_long_ansi_boundaries() {
// i64::MAX converted to f64 rounds up to 2^63, which Spark therefore accepts and
// saturates to i64::MAX, exactly like the JVM's d2l
let two_pow_63 = 9223372036854775808.0f64;
assert_eq!(i64::MAX as f64, two_pow_63);
assert_eq!(
cast_float_to_integral::<Float64Type, Int64Type>(
vec![
two_pow_63,
-two_pow_63,
two_pow_63.next_down(),
(-two_pow_63).next_up(),
],
EvalMode::Ansi,
)
.unwrap(),
vec![
i64::MAX,
i64::MIN,
9223372036854774784,
-9223372036854774784
]
);
assert_ansi_cast_overflow::<Float64Type, Int64Type>(
&[
two_pow_63.next_up(),
(-two_pow_63).next_down(),
1e19,
-1e19,
f64::MAX,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
],
"DOUBLE",
"BIGINT",
);
}
#[test]
fn test_cast_float_to_int_ansi_boundaries() {
// The bounds are compared in double precision, as in Spark: i32::MAX is not
// representable as f32 and rounds up to 2^31, which must overflow, while the largest
// f32 below 2^31 (2147483520) and -2^31 itself are accepted
let two_pow_31 = 2147483648.0f32;
assert_eq!(i32::MAX as f32, two_pow_31);
assert_eq!(
cast_float_to_integral::<Float32Type, Int32Type>(
vec![two_pow_31.next_down(), -two_pow_31, (-two_pow_31).next_up()],
EvalMode::Ansi,
)
.unwrap(),
vec![2147483520, i32::MIN, -2147483520]
);
assert_ansi_cast_overflow::<Float32Type, Int32Type>(
&[
two_pow_31,
(-two_pow_31).next_down(),
1e10,
f32::MAX,
f32::MIN,
f32::NAN,
f32::INFINITY,
f32::NEG_INFINITY,
],
"FLOAT",
"INT",
);
}
#[test]
fn test_cast_float_to_long_ansi_boundaries() {
let two_pow_63 = 9223372036854775808.0f32;
assert_eq!(i64::MAX as f32, two_pow_63);
assert_eq!(
cast_float_to_integral::<Float32Type, Int64Type>(
vec![
two_pow_63,
-two_pow_63,
two_pow_63.next_down(),
(-two_pow_63).next_up(),
],
EvalMode::Ansi,
)
.unwrap(),
vec![
i64::MAX,
i64::MIN,
9223371487098961920,
-9223371487098961920
]
);
assert_ansi_cast_overflow::<Float32Type, Int64Type>(
&[
two_pow_63.next_up(),
(-two_pow_63).next_down(),
1e19,
f32::MAX,
f32::NAN,
f32::INFINITY,
f32::NEG_INFINITY,
],
"FLOAT",
"BIGINT",
);
}
#[test]
fn test_cast_float_to_short_and_byte_ansi_boundaries() {
// Spark converts to INT first and then requires the truncated INT to round-trip through
// the narrower type, so 127.9 is a valid TINYINT while 128.0 is not, and a value that
// passes the INT range check can still overflow the narrower type
assert_eq!(
cast_float_to_integral::<Float64Type, Int8Type>(
vec![127.0, -128.0, 127.9, -128.9],
EvalMode::Ansi
)
.unwrap(),
vec![i8::MAX, i8::MIN, i8::MAX, i8::MIN]
);
assert_ansi_cast_overflow::<Float64Type, Int8Type>(
&[
128.0,
-129.0,
2147483647.5,
2147483648.0,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
],
"DOUBLE",
"TINYINT",
);
assert_eq!(
cast_float_to_integral::<Float64Type, Int16Type>(
vec![32767.0, -32768.0, 32767.9, -32768.9],
EvalMode::Ansi
)
.unwrap(),
vec![i16::MAX, i16::MIN, i16::MAX, i16::MIN]
);
assert_ansi_cast_overflow::<Float64Type, Int16Type>(
&[
32768.0,
-32769.0,
2147483647.5,
2147483648.0,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
],
"DOUBLE",
"SMALLINT",
);
assert_eq!(
cast_float_to_integral::<Float32Type, Int8Type>(
vec![127.0, -128.0, 127.9, -128.9],
EvalMode::Ansi
)
.unwrap(),
vec![i8::MAX, i8::MIN, i8::MAX, i8::MIN]
);
assert_ansi_cast_overflow::<Float32Type, Int8Type>(
&[
128.0,
-129.0,
2147483520.0,
2147483648.0,
f32::NAN,
f32::INFINITY,
f32::NEG_INFINITY,
],
"FLOAT",
"TINYINT",
);
assert_eq!(
cast_float_to_integral::<Float32Type, Int16Type>(
vec![32767.0, -32768.0, 32767.9, -32768.9],
EvalMode::Ansi
)
.unwrap(),
vec![i16::MAX, i16::MIN, i16::MAX, i16::MIN]
);
assert_ansi_cast_overflow::<Float32Type, Int16Type>(
&[
32768.0,
-32769.0,
2147483520.0,
2147483648.0,
f32::NAN,
f32::INFINITY,
f32::NEG_INFINITY,
],
"FLOAT",
"SMALLINT",
);
}
#[test]
fn test_cast_float_to_integral_legacy_saturates() {
// LEGACY mode keeps truncating towards zero with saturation and mapping NaN to 0
assert_eq!(
cast_float_to_integral::<Float64Type, Int32Type>(
vec![
2147483648.0,
-2147483649.0,
1e19,
-1e19,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
],
EvalMode::Legacy,
)
.unwrap(),
vec![
i32::MAX,
i32::MIN,
i32::MAX,
i32::MIN,
0,
i32::MAX,
i32::MIN
]
);
assert_eq!(
cast_float_to_integral::<Float32Type, Int64Type>(
vec![f32::MAX, f32::MIN, f32::NAN],
EvalMode::Legacy
)
.unwrap(),
vec![i64::MAX, i64::MIN, 0]
);
// narrowing casts go through INT first, so the INT result wraps like Spark's toShort
assert_eq!(
cast_float_to_integral::<Float64Type, Int16Type>(
vec![32768.0, 2147483648.0],
EvalMode::Legacy
)
.unwrap(),
vec![i16::MIN, -1]
);
}
#[test]
fn test_spark_cast_float_min_value_to_string() {
let float_array: ArrayRef = Arc::new(Float32Array::from(vec![
Some(f32::from_bits(1)),
Some(-f32::from_bits(1)),
]));
let result = spark_cast_float32_to_utf8::<i32>(&float_array, EvalMode::Legacy).unwrap();
let strings = result.as_string::<i32>();
assert_eq!(strings.value(0), "1.4E-45");
assert_eq!(strings.value(1), "-1.4E-45");
let double_array: ArrayRef = Arc::new(Float64Array::from(vec![
Some(f64::from_bits(1)),
Some(-f64::from_bits(1)),
]));
let result = spark_cast_float64_to_utf8::<i32>(&double_array, EvalMode::Legacy).unwrap();
let strings = result.as_string::<i32>();
assert_eq!(strings.value(0), "4.9E-324");
assert_eq!(strings.value(1), "-4.9E-324");
}
#[test]
fn test_spark_cast_decimal_to_boolean() {
let array: ArrayRef = Arc::new(
Decimal128Array::from(vec![Some(0), Some(100), Some(-100), None])
.with_precision_and_scale(10, 2)
.unwrap(),
);
let result = spark_cast_decimal_to_boolean(&array).unwrap();
let bool_array = result.as_boolean();
assert!(!bool_array.value(0)); // 0 -> false
assert!(bool_array.value(1)); // 100 -> true
assert!(bool_array.value(2)); // -100 -> true
assert!(bool_array.is_null(3)); // null -> null
// A different precision/scale must still compare against a matching zero scalar.
let array: ArrayRef = Arc::new(
Decimal128Array::from(vec![Some(0), Some(1)])
.with_precision_and_scale(38, 0)
.unwrap(),
);
let result = spark_cast_decimal_to_boolean(&array).unwrap();
let bool_array = result.as_boolean();
assert!(!bool_array.value(0));
assert!(bool_array.value(1));
// All-null Decimal128(0, 0) is reachable via Spark's RDD row-to-Arrow path;
// `with_precision_and_scale(0, 0)` rejects precision 0, so the all-null fast path
// must yield an all-null boolean array without constructing the zero scalar.
// SAFETY: builds a well-formed Decimal128 ArrayData:
// - len = 2 slots
// - values buffer = 32 bytes = 2 * 16 (Decimal128 slot width), zero-initialized
// - null buffer = 1 byte, all bits clear, covering >= len bits as required
// Precision 0 skips ArrowError validation but is otherwise a legal DataType tag;
// no non-null slot is ever read, so precision-range invariants are vacuous.
let array_data = unsafe {
arrow::array::ArrayData::builder(DataType::Decimal128(0, 0))
.len(2)
.null_bit_buffer(Some(arrow::buffer::Buffer::from(&[0u8])))
.add_buffer(arrow::buffer::Buffer::from(&[0u8; 32]))
.build_unchecked()
};
let array: ArrayRef = Arc::new(Decimal128Array::from(array_data));
assert_eq!(array.data_type(), &DataType::Decimal128(0, 0));
let result = spark_cast_decimal_to_boolean(&array).unwrap();
let bool_array = result.as_boolean();
assert_eq!(bool_array.len(), 2);
assert!(bool_array.is_null(0));
assert!(bool_array.is_null(1));
// Empty Decimal128(0, 0) input: `null_count() == len()` (0 == 0) must still take
// the fast path. Precision 0 makes this discriminating — without the fast path,
// building the zero scalar would fail regardless of the empty length.
// SAFETY: len = 0, so both the (empty) values buffer and the absent null buffer
// trivially cover every slot; the precision-range invariant is vacuous.
let array_data = unsafe {
arrow::array::ArrayData::builder(DataType::Decimal128(0, 0))
.len(0)
.add_buffer(arrow::buffer::Buffer::from(&[] as &[u8]))
.build_unchecked()
};
let array: ArrayRef = Arc::new(Decimal128Array::from(array_data));
// The load-bearing check is that `spark_cast_decimal_to_boolean` returns Ok — without
// the fast path, precision 0 would fail during zero-scalar construction. Length is
// asserted for completeness; null_count is trivially 0 on any empty array.
let result = spark_cast_decimal_to_boolean(&array).unwrap();
assert_eq!(result.as_boolean().len(), 0);
// Mixed valid + null Decimal128(0, 0). Reachable via JVM-side writers that do not
// validate precision (e.g. a Java UDF returning `BigInteger.ZERO` declared as
// `DecimalType(0, 0)`, written through `DecimalVector.setSafe(long)`). The all-null
// fast path does not cover this: `null_count() < len()` yet the batch still cannot
// round-trip through a zero scalar. We build a 3-row batch where slot 0 is a valid
// zero, slot 1 is null, and slot 2 is an out-of-contract non-zero i128; the path
// must read the raw value (not hard-code `false`) so slot 2 comes back as `true`.
// Expected output: `[false, null, true]`.
// SAFETY: len = 3; values buffer = 48 bytes = 3 * 16, with i128(0), i128(0),
// i128(7) laid out little-endian; null bit buffer 0b0000_0101 = slots 0 and 2
// valid, slot 1 null.
let mut vals = [0u8; 48];
vals[32..48].copy_from_slice(&7i128.to_le_bytes()); // slot 2 = 7
let array_data = unsafe {
arrow::array::ArrayData::builder(DataType::Decimal128(0, 0))
.len(3)
.null_bit_buffer(Some(arrow::buffer::Buffer::from(&[0b0000_0101u8])))
.add_buffer(arrow::buffer::Buffer::from(&vals))
.build_unchecked()
};
let array: ArrayRef = Arc::new(Decimal128Array::from(array_data));
assert_eq!(array.data_type(), &DataType::Decimal128(0, 0));
assert_eq!(array.null_count(), 1);
let result = spark_cast_decimal_to_boolean(&array).unwrap();
let bool_array = result.as_boolean();
assert_eq!(bool_array.len(), 3);
assert!(!bool_array.is_null(0));
assert!(!bool_array.value(0)); // valid zero -> false
assert!(bool_array.is_null(1)); // null -> null
assert!(!bool_array.is_null(2));
assert!(bool_array.value(2)); // valid non-zero i128 -> true (raw-value cast)
}
#[test]
fn test_cast_int_to_decimal128() {
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(100), Some(-100), None]));
let result = cast_int_to_decimal128(
&array,
EvalMode::Legacy,
&DataType::Int32,
&DataType::Decimal128(10, 2),
10,
2,
)
.unwrap();
let decimal_array = result.as_primitive::<Decimal128Type>();
assert_eq!(decimal_array.value(0), 10000); // 100 * 10^2
assert_eq!(decimal_array.value(1), -10000); // -100 * 10^2
assert!(decimal_array.is_null(2));
}
#[test]
fn test_cast_int_to_decimal128_overflow_legacy_nulls() {
// 1000 * 10^2 = 100000 does not fit precision 3 -> null (legacy). Valid values and the
// input null are preserved, exercising the vectorized null-on-overflow path.
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(9), Some(1000), None, Some(-9)]));
let result = cast_int_to_decimal128(
&array,
EvalMode::Legacy,
&DataType::Int32,
&DataType::Decimal128(3, 2),
3,
2,
)
.unwrap();
let d = result.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 900); // 9.00
assert!(d.is_null(1)); // overflow -> null
assert!(d.is_null(2)); // input null preserved
assert_eq!(d.value(3), -900);
assert_eq!(d.data_type(), &DataType::Decimal128(3, 2));
}
#[test]
fn test_cast_int_to_decimal128_overflow_try_nulls() {
// Try shares the Legacy null-on-overflow branch but is a distinct enum arm; assert it
// explicitly so a future refactor cannot regress it silently.
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(9), Some(1000), None, Some(-9)]));
let result = cast_int_to_decimal128(
&array,
EvalMode::Try,
&DataType::Int32,
&DataType::Decimal128(3, 2),
3,
2,
)
.unwrap();
let d = result.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 900);
assert!(d.is_null(1));
assert!(d.is_null(2));
assert_eq!(d.value(3), -900);
}
#[test]
fn test_cast_int_to_decimal128_no_overflow_ansi() {
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(9), None, Some(-9)]));
let result = cast_int_to_decimal128(
&array,
EvalMode::Ansi,
&DataType::Int32,
&DataType::Decimal128(3, 2),
3,
2,
)
.unwrap();
let d = result.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 900);
assert!(d.is_null(1));
assert_eq!(d.value(2), -900);
}
#[test]
fn test_cast_int_to_decimal128_overflow_ansi_errors() {
// Two overflowing values: the rescan is documented to report the first one, so asserting
// on 1000 rather than 2000 pins the scan order as well as the error variant and payload.
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(9), Some(1000), Some(2000)]));
let result = cast_int_to_decimal128(
&array,
EvalMode::Ansi,
&DataType::Int32,
&DataType::Decimal128(3, 2),
3,
2,
);
let err = result.unwrap_err();
assert!(
matches!(
err,
SparkError::NumericValueOutOfRange {
ref value,
precision: 3,
scale: 2
} if value == "1000"
),
"unexpected error: {err:?}"
);
}
#[test]
fn test_cast_int_to_timestamp() {
let timezones: [Option<Arc<str>>; 6] = [
Some(Arc::from("UTC")),
Some(Arc::from("America/New_York")),
Some(Arc::from("America/Los_Angeles")),
Some(Arc::from("Europe/London")),
Some(Arc::from("Asia/Tokyo")),
Some(Arc::from("Australia/Sydney")),
];
for tz in &timezones {
let int8_array: ArrayRef = Arc::new(Int8Array::from(vec![
Some(0),
Some(1),
Some(-1),
Some(127),
Some(-128),
None,
]));
let result = cast_int_to_timestamp(&int8_array, tz).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000);
assert_eq!(ts_array.value(2), -1_000_000);
assert_eq!(ts_array.value(3), 127_000_000);
assert_eq!(ts_array.value(4), -128_000_000);
assert!(ts_array.is_null(5));
assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref()));
let int16_array: ArrayRef = Arc::new(Int16Array::from(vec![
Some(0),
Some(1),
Some(-1),
Some(32767),
Some(-32768),
None,
]));
let result = cast_int_to_timestamp(&int16_array, tz).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000);
assert_eq!(ts_array.value(2), -1_000_000);
assert_eq!(ts_array.value(3), 32_767_000_000_i64);
assert_eq!(ts_array.value(4), -32_768_000_000_i64);
assert!(ts_array.is_null(5));
assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref()));
let int32_array: ArrayRef = Arc::new(Int32Array::from(vec![
Some(0),
Some(1),
Some(-1),
Some(1704067200),
None,
]));
let result = cast_int_to_timestamp(&int32_array, tz).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000);
assert_eq!(ts_array.value(2), -1_000_000);
assert_eq!(ts_array.value(3), 1_704_067_200_000_000_i64);
assert!(ts_array.is_null(4));
assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref()));
let int64_array: ArrayRef = Arc::new(Int64Array::from(vec![
Some(0),
Some(1),
Some(-1),
Some(i64::MAX),
Some(i64::MIN),
]));
let result = cast_int_to_timestamp(&int64_array, tz).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000_i64);
assert_eq!(ts_array.value(2), -1_000_000_i64);
assert_eq!(ts_array.value(3), i64::MAX);
assert_eq!(ts_array.value(4), i64::MIN);
assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref()));
}
}
#[test]
fn test_decimal128_to_f64_matches_bigdecimal_double_value() {
// Expected values were checked against Java's
// `new BigDecimal(BigInteger(unscaled), scale).doubleValue()`.
let cases: Vec<(i128, i8, f64)> = vec![
// https://github.com/apache/datafusion-comet/issues/5670
(12345678900000000000000, 18, 12345.6789),
(123456789012000000000000, 18, 123456.789012),
(76543210000000000000000, 18, 76543.21),
(577312285583388355022308, 18, 577312.2855833884),
(-12345678900000000000000, 18, -12345.6789),
(-577312285583388355022308, 18, -577312.2855833884),
// zero and the smallest magnitudes
(0, 18, 0.0),
(0, 38, 0.0),
(1, 38, 1e-38),
(-1, 38, -1e-38),
// 38-digit extremes
(99999999999999999999999999999999999999, 0, 1e38),
(-99999999999999999999999999999999999999, 0, -1e38),
(99999999999999999999999999999999999999, 38, 1.0),
(-99999999999999999999999999999999999999, 38, -1.0),
(i128::MAX, 0, 2f64.powi(127)),
(i128::MIN, 0, -(2f64.powi(127))),
// 2^53 + 1 is halfway between two doubles and rounds to even
(9007199254740993, 0, 9007199254740992.0),
// exact-operand fast path
(1, 18, 1e-18),
(-15, 1, -1.5),
(123456789, 3, 123456.789),
(9007199254740992, 22, 9007199254740992e-22),
// negative scales, which arrow permits but Spark never produces
(123456789, -3, 123456789e3),
(9007199254740992, -22, 9007199254740992e22),
(9007199254740993, -22, 9007199254740993e22),
];
for (unscaled, scale, expected) in cases {
let actual = decimal128_to_f64(unscaled, scale);
assert_eq!(
actual.to_bits(),
expected.to_bits(),
"{unscaled}e{}: got {actual:?}, expected {expected:?}",
-i32::from(scale)
);
}
// The double-rounding formula the kernel replaces gets the issue's example wrong.
assert_ne!((12345678900000000000000_i128 as f64) / 1e18, 12345.6789);
}
#[test]
fn test_decimal128_to_f32_matches_bigdecimal_float_value() {
// Expected values were checked against Java's
// `new BigDecimal(BigInteger(unscaled), scale).floatValue()`.
let cases: Vec<(i128, i8, f32)> = vec![
// https://github.com/apache/datafusion-comet/issues/5670: 16777217.0000000001 lies
// above the midpoint of the floats 16777216 and 16777218, but its nearest double
// (16777217.0) is exactly that midpoint and would narrow to 16777216.
(167772170000000001, 10, 16777218.0),
(-167772170000000001, 10, -16777218.0),
(12345678900000000000000, 18, 12345.679),
(76543210000000000000000, 18, 76543.21),
(-12345678900000000000000, 18, -12345.679),
// zero and the smallest magnitudes (1e-38 is subnormal in f32)
(0, 18, 0.0),
(0, 38, 0.0),
(1, 38, 1e-38),
(-1, 38, -1e-38),
// 38-digit extremes
(99999999999999999999999999999999999999, 0, 1e38),
(-99999999999999999999999999999999999999, 0, -1e38),
(99999999999999999999999999999999999999, 38, 1.0),
(-99999999999999999999999999999999999999, 38, -1.0),
(i128::MAX, 0, 2f32.powi(127)),
(i128::MIN, 0, -(2f32.powi(127))),
// 2^24 + 1 is halfway between two floats and rounds to even
(16777217, 0, 16777216.0),
// exact-operand fast path
(1, 10, 1e-10),
(-15, 1, -1.5),
(123456789, 3, 123456.79),
(16777216, 10, 16777216e-10),
// negative scales, which arrow permits but Spark never produces; beyond the f32
// range the result is an infinity, like Float.parseFloat
(123456789, -3, 1.2345679e11),
(16777216, -10, 16777216e10),
(1, -39, f32::INFINITY),
(-1, -39, f32::NEG_INFINITY),
];
for (unscaled, scale, expected) in cases {
let actual = decimal128_to_f32(unscaled, scale);
assert_eq!(
actual.to_bits(),
expected.to_bits(),
"{unscaled}e{}: got {actual:?}, expected {expected:?}",
-i32::from(scale)
);
}
// Narrowing the (already correctly rounded) double gets the issue's example wrong.
assert_ne!(
decimal128_to_f64(167772170000000001, 10) as f32,
16777218.0_f32
);
}
#[test]
fn test_decimal128_to_float_fast_path_matches_exact_path() {
use rand::rngs::StdRng;
use rand::{RngExt, SeedableRng};
let mut rng = StdRng::seed_from_u64(5670);
let mut cases: Vec<(i128, i8)> = Vec::new();
// Magnitudes spread over the whole i128 range, so both the exact-operand fast paths
// (|unscaled| <= 2^24 for f32, <= 2^53 for f64) and the parse fallback are exercised.
for _ in 0..10_000 {
let bits = rng.random_range(0_u32..=127);
let magnitude = if bits == 0 {
0
} else {
rng.random::<u128>() >> (128 - bits)
};
let unscaled = if rng.random::<bool>() {
magnitude as i128
} else {
-(magnitude as i128)
};
cases.push((unscaled, rng.random_range(-30_i8..=38)));
}
// Values around the fast-path limits, where a wrong bound would show up.
for _ in 0..10_000 {
let limit = if rng.random::<bool>() {
1_i128 << 53
} else {
1 << 24
};
let unscaled = limit + rng.random_range(-1000_i128..=1000);
let unscaled = if rng.random::<bool>() {
unscaled
} else {
-unscaled
};
cases.push((unscaled, rng.random_range(-23_i8..=23)));
}
// Decimal-looking values with few digits, all on the fast paths.
for _ in 0..10_000 {
let unscaled = rng.random_range(-10_000_000_i128..=10_000_000);
cases.push((unscaled, rng.random_range(0_i8..=10)));
}
for (unscaled, scale) in cases {
// Rust's float parser rounds the exact decimal value once, like BigDecimal does.
let text = format!("{unscaled}e{}", -i32::from(scale));
let expected_f64: f64 = text.parse().unwrap();
let expected_f32: f32 = text.parse().unwrap();
let actual_f64 = decimal128_to_f64(unscaled, scale);
let actual_f32 = decimal128_to_f32(unscaled, scale);
assert_eq!(
actual_f64.to_bits(),
expected_f64.to_bits(),
"{text}: got {actual_f64:?}, expected {expected_f64:?}"
);
assert_eq!(
actual_f32.to_bits(),
expected_f32.to_bits(),
"{text}: got {actual_f32:?}, expected {expected_f32:?}"
);
}
}
#[test]
fn test_cast_decimal128_to_float_arrays_keep_nulls_in_every_eval_mode() {
use crate::conversion_funcs::cast::cast_array;
use crate::SparkCastOptions;
let decimals_38_18: ArrayRef = Arc::new(
Decimal128Array::from(vec![
Some(12345678900000000000000_i128),
None,
Some(-76543210000000000000000),
Some(0),
Some(577312285583388355022308),
None,
])
.with_precision_and_scale(38, 18)
.unwrap(),
);
let decimals_38_10: ArrayRef = Arc::new(
Decimal128Array::from(vec![Some(167772170000000001_i128), None, Some(-123456789)])
.with_precision_and_scale(38, 10)
.unwrap(),
);
for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] {
let options = SparkCastOptions::new(eval_mode, "UTC", false);
let doubles =
cast_array(Arc::clone(&decimals_38_18), &DataType::Float64, &options).unwrap();
let doubles = doubles.as_primitive::<Float64Type>();
assert_eq!(doubles.len(), 6);
assert_eq!(doubles.null_count(), 2);
assert_eq!(doubles.value(0), 12345.6789);
assert!(doubles.is_null(1));
assert_eq!(doubles.value(2), -76543.21);
assert_eq!(doubles.value(3).to_bits(), 0.0_f64.to_bits());
assert_eq!(doubles.value(4), 577312.2855833884);
assert!(doubles.is_null(5));
let floats =
cast_array(Arc::clone(&decimals_38_18), &DataType::Float32, &options).unwrap();
let floats = floats.as_primitive::<Float32Type>();
assert_eq!(floats.len(), 6);
assert_eq!(floats.null_count(), 2);
assert_eq!(floats.value(0), 12345.679_f32);
assert!(floats.is_null(1));
assert_eq!(floats.value(2), -76543.21_f32);
assert_eq!(floats.value(3).to_bits(), 0.0_f32.to_bits());
assert_eq!(floats.value(4), 577312.3_f32);
assert!(floats.is_null(5));
let floats =
cast_array(Arc::clone(&decimals_38_10), &DataType::Float32, &options).unwrap();
let floats = floats.as_primitive::<Float32Type>();
assert_eq!(floats.value(0), 16777218.0_f32);
assert!(floats.is_null(1));
assert_eq!(floats.value(2), -0.012345679_f32);
let doubles =
cast_array(Arc::clone(&decimals_38_10), &DataType::Float64, &options).unwrap();
let doubles = doubles.as_primitive::<Float64Type>();
// 16777217.0000000001 rounds to the double 16777217.0
assert_eq!(doubles.value(0), 16777217.0);
assert!(doubles.is_null(1));
assert_eq!(doubles.value(2), -0.0123456789);
}
}
#[test]
fn test_cast_float_to_decimal() {
let a: ArrayRef = Arc::new(Float64Array::from(vec![
Some(42.),
Some(0.5153125),
Some(-42.4242415),
Some(42e-314),
Some(0.),
Some(-4242.424242),
Some(f64::INFINITY),
Some(f64::NEG_INFINITY),
Some(f64::NAN),
None,
]));
let b =
cast_floating_point_to_decimal128::<Float64Type>(&a, 8, 6, EvalMode::Legacy).unwrap();
assert_eq!(b.len(), a.len());
let casted = b.as_primitive::<Decimal128Type>();
assert_eq!(casted.value(0), 42000000);
assert_eq!(casted.value(1), 515313);
assert_eq!(casted.value(2), -42424242);
assert_eq!(casted.value(3), 0);
assert_eq!(casted.value(4), 0);
assert!(casted.is_null(5));
assert!(casted.is_null(6));
assert!(casted.is_null(7));
assert!(casted.is_null(8));
assert!(casted.is_null(9));
}
#[test]
fn test_cast_float_to_decimal_no_overflow_fast_path() {
// All values fit precision 10, scale 2, so the vectorized fast path is taken and the
// input null is preserved.
let a: ArrayRef = Arc::new(Float64Array::from(vec![
Some(42.0),
Some(-1.5),
None,
Some(0.0),
]));
let b =
cast_floating_point_to_decimal128::<Float64Type>(&a, 10, 2, EvalMode::Legacy).unwrap();
let d = b.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 4200); // 42.00
assert_eq!(d.value(1), -150); // -1.50
assert!(d.is_null(2));
assert_eq!(d.value(3), 0);
assert_eq!(d.data_type(), &DataType::Decimal128(10, 2));
}
#[test]
fn test_cast_float_to_decimal_ansi_no_overflow() {
let a: ArrayRef = Arc::new(Float64Array::from(vec![Some(42.0), None, Some(-1.5)]));
let b =
cast_floating_point_to_decimal128::<Float64Type>(&a, 10, 2, EvalMode::Ansi).unwrap();
let d = b.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 4200);
assert!(d.is_null(1));
assert_eq!(d.value(2), -150);
}
#[test]
fn test_cast_float_to_decimal_ansi_overflow_errors() {
// Precision overflow: 4242.42 * 10^2 = 424242 does not fit precision 4. Two overflow
// rows with distinct string forms pin that the rescan reports the *first* offender.
let a: ArrayRef = Arc::new(Float64Array::from(vec![
Some(1.0),
Some(4242.42),
Some(9999.99),
]));
match cast_floating_point_to_decimal128::<Float64Type>(&a, 4, 2, EvalMode::Ansi) {
Err(SparkError::NumericValueOutOfRange {
value,
precision,
scale,
}) => {
assert_eq!(value, "4242.42");
assert_eq!(precision, 4);
assert_eq!(scale, 2);
}
other => panic!("expected NumericValueOutOfRange, got {other:?}"),
}
}
/// Cast an f64 array to Decimal128(precision, scale), panicking on error
fn cast_f64_to_decimal128(
values: Vec<Option<f64>>,
precision: u8,
scale: i8,
eval_mode: EvalMode,
) -> Decimal128Array {
let a: ArrayRef = Arc::new(Float64Array::from(values));
cast_floating_point_to_decimal128::<Float64Type>(&a, precision, scale, eval_mode)
.unwrap()
.as_primitive::<Decimal128Type>()
.clone()
}
#[test]
fn test_cast_float_to_decimal_ansi_nan_and_infinity_are_null() {
// Spark converts through Decimal(double) which throws NumberFormatException for
// NaN/infinity; the cast catches it and produces null before the ANSI overflow
// check runs, so these are null even in ANSI mode.
let d = cast_f64_to_decimal128(
vec![
Some(1.0),
Some(f64::NAN),
Some(f64::INFINITY),
Some(f64::NEG_INFINITY),
],
10,
2,
EvalMode::Ansi,
);
assert_eq!(d.value(0), 100);
assert!(d.is_null(1));
assert!(d.is_null(2));
assert!(d.is_null(3));
}
#[test]
fn test_parse_decimal_notation_round_trips_ryu_output() {
// Every shape ryu can emit: plain integral ("1.0"), plain fractional ("12.34"),
// leading-zero fractional ("0.001234", up to 21 fractional digits), single-digit
// exponent form ("1e30") and full exponent form ("1.234e33"). The extremes pin
// down that the mantissa always fits an i128, so the `None` arm stays unreachable.
let mut buf = ryu::Buffer::new();
for v in [
0.0,
1.0,
-1.0,
12.34,
-0.001234,
0.5153125,
1e16,
1e17,
1.5e10,
1e30,
1.2345678901234567e-5,
1.2345678901234567e300,
f64::MAX,
f64::MIN,
f64::MIN_POSITIVE,
f64::from_bits(1), // smallest subnormal
] {
let formatted = buf.format_finite(v);
let (mantissa, exp10) =
parse_decimal_notation(formatted).expect("ryu mantissa fits i128");
let reconstructed: f64 = format!("{mantissa}e{exp10}").parse().unwrap();
assert_eq!(reconstructed, v, "{formatted} -> {mantissa}e{exp10}");
}
}
#[test]
fn test_cast_float_to_decimal_spark_rounding_parity() {
// Spark rounds the shortest decimal string form of the double (Double.toString ->
// BigDecimal.setScale(HALF_UP)), not the binary value. These values sit exactly on
// a rounding tie in string form while the binary value is just below it, so binary
// rounding gives a different (wrong) result.
let d = cast_f64_to_decimal128(
vec![Some(0.5153125), Some(-0.5153125), Some(0.5203125)],
8,
6,
EvalMode::Legacy,
);
assert_eq!(d.value(0), 515313);
assert_eq!(d.value(1), -515313);
assert_eq!(d.value(2), 520313);
// HALF_UP rounds away from zero for negative values too
let d = cast_f64_to_decimal128(vec![Some(1.005), Some(-1.005)], 10, 2, EvalMode::Legacy);
assert_eq!(d.value(0), 101);
assert_eq!(d.value(1), -101);
// values whose shortest form uses exponent notation
let d = cast_f64_to_decimal128(vec![Some(1.5e10)], 20, 2, EvalMode::Legacy);
assert_eq!(d.value(0), 1_500_000_000_000);
let d = cast_f64_to_decimal128(vec![Some(1e30)], 10, 2, EvalMode::Legacy);
assert!(d.is_null(0));
}
#[test]
fn test_cast_float32_to_decimal_widens_to_double_first() {
// Spark widens float to double before converting to string, so 1.005f32 becomes
// 1.0049999952316284 and rounds down; using the shortest f32 form ("1.005") would
// incorrectly round up.
let a: ArrayRef = Arc::new(Float32Array::from(vec![Some(1.005f32)]));
let b =
cast_floating_point_to_decimal128::<Float32Type>(&a, 10, 2, EvalMode::Legacy).unwrap();
let d = b.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 100);
// 0.5153125f32 widens to 0.5153124928474426 which rounds down, unlike the f64 case
let a: ArrayRef = Arc::new(Float32Array::from(vec![Some(0.5153125f32)]));
let b =
cast_floating_point_to_decimal128::<Float32Type>(&a, 8, 6, EvalMode::Legacy).unwrap();
let d = b.as_primitive::<Decimal128Type>();
assert_eq!(d.value(0), 515312);
}
#[test]
// Not a correctness concern under miri, just far too slow: 240k values through the
// interpreter. The fixed-value tests above cover both paths under miri.
#[cfg_attr(miri, ignore)]
fn test_cast_float_to_decimal_fast_path_matches_exact_path() {
use rand::rngs::StdRng;
use rand::{RngExt, SeedableRng};
let mut rng = StdRng::seed_from_u64(42);
let mut values: Vec<f64> = Vec::new();
// random bit patterns cover the full range including subnormals, NaN and infinity
for _ in 0..20_000 {
values.push(f64::from_bits(rng.random::<u64>()));
}
// decimal-looking values, which land on or near rounding ties far more often
for _ in 0..20_000 {
let unscaled = rng.random_range(-10_000_000_i64..10_000_000);
let scale = rng.random_range(0_i32..8);
values.push(unscaled as f64 / 10_f64.powi(scale));
}
for (precision, scale) in [(38_u8, 18_i8), (18, 6), (10, 2), (8, 6), (4, 2), (38, 0)] {
let d = cast_f64_to_decimal128(
values.iter().copied().map(Some).collect(),
precision,
scale,
EvalMode::Legacy,
);
for (i, &v) in values.iter().enumerate() {
let expected = float_to_decimal128(v, precision, scale);
let actual = (!d.is_null(i)).then(|| d.value(i));
assert_eq!(actual, expected, "value {v:?} at ({precision},{scale})");
}
}
}
#[test]
fn test_cast_float_to_decimal_round_up_overflow() {
// 99.995 rounds up to 100.00 in string form, which no longer fits precision 4.
// Binary rounding of 99.99499999999999886... would incorrectly keep 99.99.
let d = cast_f64_to_decimal128(vec![Some(99.995)], 4, 2, EvalMode::Legacy);
assert!(d.is_null(0));
let a: ArrayRef = Arc::new(Float64Array::from(vec![Some(99.995)]));
match cast_floating_point_to_decimal128::<Float64Type>(&a, 4, 2, EvalMode::Ansi) {
Err(SparkError::NumericValueOutOfRange {
value,
precision,
scale,
}) => {
assert_eq!(value, "99.995");
assert_eq!(precision, 4);
assert_eq!(scale, 2);
}
other => panic!("expected NumericValueOutOfRange, got {other:?}"),
}
}
#[test]
fn test_cast_decimal_to_timestamp() {
let timezones: [Option<Arc<str>>; 3] = [
Some(Arc::from("UTC")),
Some(Arc::from("America/Los_Angeles")),
None,
];
for tz in &timezones {
// Decimal128 with scale 6
let decimal_array: ArrayRef = Arc::new(
Decimal128Array::from(vec![
Some(0_i128),
Some(1_000_000_i128),
Some(-1_000_000_i128),
Some(1_500_000_i128),
Some(123_456_789_i128),
None,
])
.with_precision_and_scale(18, 6)
.unwrap(),
);
let result = cast_decimal_to_timestamp(&decimal_array, tz, 6).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000);
assert_eq!(ts_array.value(2), -1_000_000);
assert_eq!(ts_array.value(3), 1_500_000);
assert_eq!(ts_array.value(4), 123_456_789);
assert!(ts_array.is_null(5));
assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref()));
// Test with scale 2
let decimal_array: ArrayRef = Arc::new(
Decimal128Array::from(vec![Some(100_i128), Some(150_i128), Some(-250_i128)])
.with_precision_and_scale(10, 2)
.unwrap(),
);
let result = cast_decimal_to_timestamp(&decimal_array, tz, 2).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 1_000_000);
assert_eq!(ts_array.value(1), 1_500_000);
assert_eq!(ts_array.value(2), -2_500_000);
}
}
#[test]
fn test_cast_float_to_timestamp() {
let timezones: [Option<Arc<str>>; 3] = [
Some(Arc::from("UTC")),
Some(Arc::from("America/Los_Angeles")),
None,
];
let eval_modes = [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try];
for tz in &timezones {
for eval_mode in &eval_modes {
// Float64 tests
let f64_array: ArrayRef = Arc::new(Float64Array::from(vec![
Some(0.0),
Some(1.0),
Some(-1.0),
Some(1.5),
Some(0.000001),
None,
]));
let result = cast_float_to_timestamp(&f64_array, tz, *eval_mode).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000);
assert_eq!(ts_array.value(2), -1_000_000);
assert_eq!(ts_array.value(3), 1_500_000);
assert_eq!(ts_array.value(4), 1);
assert!(ts_array.is_null(5));
assert_eq!(ts_array.timezone(), tz.as_ref().map(|s| s.as_ref()));
// Float32 tests
let f32_array: ArrayRef = Arc::new(Float32Array::from(vec![
Some(0.0_f32),
Some(1.0_f32),
Some(-1.0_f32),
None,
]));
let result = cast_float_to_timestamp(&f32_array, tz, *eval_mode).unwrap();
let ts_array = result.as_primitive::<TimestampMicrosecondType>();
assert_eq!(ts_array.value(0), 0);
assert_eq!(ts_array.value(1), 1_000_000);
assert_eq!(ts_array.value(2), -1_000_000);
assert!(ts_array.is_null(3));
}
}
// ANSI mode errors on NaN/Infinity
let tz = &Some(Arc::from("UTC"));
let f64_nan: ArrayRef = Arc::new(Float64Array::from(vec![Some(f64::NAN)]));
assert!(cast_float_to_timestamp(&f64_nan, tz, EvalMode::Ansi).is_err());
let f64_inf: ArrayRef = Arc::new(Float64Array::from(vec![Some(f64::INFINITY)]));
assert!(cast_float_to_timestamp(&f64_inf, tz, EvalMode::Ansi).is_err());
}
#[test]
fn test_decimal128_to_java_string() {
fn fmt(unscaled: i128, scale: i8) -> String {
let mut buf = String::new();
decimal128_to_java_string(unscaled, scale, &mut [0u8; MAX_COEFF_DIGITS], &mut buf);
buf
}
// scale >= 0, adj_exp >= -6 → plain notation
assert_eq!(fmt(0, 0), "0");
assert_eq!(fmt(0, 2), "0.00");
assert_eq!(fmt(12345, 2), "123.45");
assert_eq!(fmt(-12345, 2), "-123.45");
assert_eq!(fmt(1, 2), "0.01");
assert_eq!(fmt(42, 0), "42");
assert_eq!(fmt(-42, 0), "-42");
assert_eq!(fmt(1, 6), "0.000001"); // adj_exp = -6 (boundary)
// scale >= 0, adj_exp < -6 → scientific notation (Spark LEGACY mode)
assert_eq!(fmt(0, 18), "0E-18"); // adj_exp = -18
assert_eq!(fmt(0, 7), "0E-7"); // adj_exp = -7
assert_eq!(fmt(1, 7), "1E-7");
assert_eq!(fmt(1, 18), "1E-18");
// scale < 0 → scientific notation
assert_eq!(fmt(0, -2), "0E+2");
assert_eq!(fmt(1, -2), "1E+2");
assert_eq!(fmt(123, -2), "1.23E+4");
assert_eq!(fmt(-123, -2), "-1.23E+4");
// values needing more than one 64-bit digit group
assert_eq!(fmt(10_000_000_000_000_000_000, 0), "10000000000000000000");
// 10^19 + 5: the lower group is 0000000000000000005 and its leading zeros are load-bearing
// (the render_digits comment "emitting its leading zeroes is correct" applies here).
assert_eq!(fmt(10_000_000_000_000_000_005, 0), "10000000000000000005");
assert_eq!(fmt(i128::MAX, 0), i128::MAX.to_string());
assert_eq!(fmt(i128::MIN, 0), i128::MIN.to_string());
assert_eq!(
fmt(99_999_999_999_999_999_999_999_999_999_999_999_999, 10),
"9999999999999999999999999999.9999999999"
);
// multi-group coefficient in scientific notation, both signs, so the sign push and
// the &coeff[..1] / &coeff[1..] split are covered on a value that spans two groups.
assert_eq!(
fmt(i128::MAX, 45),
"1.70141183460469231731687303715884105727E-7"
);
assert_eq!(
fmt(-i128::MAX, 45),
"-1.70141183460469231731687303715884105727E-7"
);
}
}