Google Git Google Git
Sign in
apache/datafusion-comet/refs/heads/codex/comet-df-bitmap/./native/spark-expr/src/conversion_funcs/string.rs
blob: adedd3ba0d2134599d0ce7d3ef8bd98151a99ca1 [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use crate::conversion_funcs::trim::{trim_all, trim_all_bytes, trim_all_range, trim_java_string};
use crate::{EvalMode, SparkError, SparkResult};
use arrow::array::timezone::Tz;
use arrow::array::{
Array, ArrayRef, ArrowPrimitiveType, BooleanArray, Decimal128Builder, GenericStringArray,
OffsetSizeTrait, PrimitiveArray, PrimitiveBuilder, StringArray,
};
use arrow::datatypes::{
i256, is_validate_decimal_precision, DataType, Date32Type, Decimal256Type, Float32Type,
Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, TimestampMicrosecondType,
};
use chrono::{LocalResult, NaiveDate, NaiveTime, Offset, TimeZone, Timelike};
use num::traits::CheckedNeg;
use num::{CheckedSub, Integer};
use std::num::Wrapping;
use std::str::FromStr;
use std::sync::Arc;
// Shared macro for casting UTF-8 string arrays to timestamp types (both TZ and NTZ).
// $builder is a PrimitiveBuilder expression; $extra_args are forwarded to $cast_method
// after (value, eval_mode).
macro_rules! cast_utf8_to_timestamp {
($array:expr, $eval_mode:expr, $builder:expr, $cast_method:ident $(, $extra_arg:expr)*) => {{
let mut cast_array = $builder;
let mut cast_err: Option<SparkError> = None;
for i in 0..$array.len() {
if $array.is_null(i) {
cast_array.append_null()
} else {
// Untrimmed: the parsers trim, and Spark 4 needs the raw start to reject
// padding before a time-only `T` (" T2" -> null, "T2" -> valid).
match $cast_method($array.value(i), $eval_mode $(, $extra_arg)*) {
Ok(Some(cast_value)) => cast_array.append_value(cast_value),
Ok(None) => cast_array.append_null(),
Err(e) => {
if $eval_mode == EvalMode::Ansi {
let raw_value = $array.value(i).to_string();
let e = match e {
SparkError::InvalidInputInCastToDatetime {
from_type,
to_type,
..
} => SparkError::InvalidInputInCastToDatetime {
value: raw_value,
from_type,
to_type,
},
other => other,
};
cast_err = Some(e);
break;
}
cast_array.append_null()
}
}
}
}
if let Some(e) = cast_err {
Err(e)
} else {
Ok(Arc::new(cast_array.finish()) as ArrayRef)
}
}};
}
macro_rules! cast_utf8_to_int {
($array:expr, $array_type:ty, $parse_fn:expr) => {{
let len = $array.len();
let mut cast_array = PrimitiveArray::<$array_type>::builder(len);
let parse_fn = $parse_fn;
if $array.null_count() == 0 {
for i in 0..len {
if let Some(cast_value) = parse_fn($array.value(i))? {
cast_array.append_value(cast_value);
} else {
cast_array.append_null()
}
}
} else {
for i in 0..len {
if $array.is_null(i) {
cast_array.append_null()
} else if let Some(cast_value) = parse_fn($array.value(i))? {
cast_array.append_value(cast_value);
} else {
cast_array.append_null()
}
}
}
let result: SparkResult<ArrayRef> = Ok(Arc::new(cast_array.finish()) as ArrayRef);
result
}};
}
#[derive(Clone)]
struct TimeStampInfo {
year: i32,
month: u32,
day: u32,
hour: u32,
minute: u32,
second: u32,
microsecond: u32,
}
impl Default for TimeStampInfo {
fn default() -> Self {
TimeStampInfo {
year: 1,
month: 1,
day: 1,
hour: 0,
minute: 0,
second: 0,
microsecond: 0,
}
}
}
impl TimeStampInfo {
fn with_year(&mut self, year: i32) -> &mut Self {
self.year = year;
self
}
fn with_month(&mut self, month: u32) -> &mut Self {
self.month = month;
self
}
fn with_day(&mut self, day: u32) -> &mut Self {
self.day = day;
self
}
fn with_hour(&mut self, hour: u32) -> &mut Self {
self.hour = hour;
self
}
fn with_minute(&mut self, minute: u32) -> &mut Self {
self.minute = minute;
self
}
fn with_second(&mut self, second: u32) -> &mut Self {
self.second = second;
self
}
fn with_microsecond(&mut self, microsecond: u32) -> &mut Self {
self.microsecond = microsecond;
self
}
}
pub(crate) fn is_df_cast_from_string_spark_compatible(to_type: &DataType) -> bool {
matches!(to_type, DataType::Binary)
}
pub(crate) fn cast_string_to_float(
array: &ArrayRef,
to_type: &DataType,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
match to_type {
DataType::Float32 => cast_string_to_float_impl::<Float32Type>(array, eval_mode, "FLOAT"),
DataType::Float64 => cast_string_to_float_impl::<Float64Type>(array, eval_mode, "DOUBLE"),
_ => Err(SparkError::Internal(format!(
"Unsupported cast to float type: {:?}",
to_type
))),
}
}
fn cast_string_to_float_impl<T: ArrowPrimitiveType>(
array: &ArrayRef,
eval_mode: EvalMode,
type_name: &str,
) -> SparkResult<ArrayRef>
where
T::Native: FromStr + num::Float,
{
let arr = array
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| SparkError::Internal("Expected string array".to_string()))?;
let mut builder = PrimitiveBuilder::<T>::with_capacity(arr.len());
for i in 0..arr.len() {
if arr.is_null(i) {
builder.append_null();
} else {
// `Double.parseDouble` calls `String.trim` before parsing, so only bytes <= 0x20
// are trimmed here -- `0x7F` is not whitespace to this cast, and no non-ASCII
// whitespace is trimmed by any Spark cast.
let str_value = trim_java_string(arr.value(i));
match parse_string_to_float(str_value) {
Some(v) => builder.append_value(v),
None => {
if eval_mode == EvalMode::Ansi {
return Err(invalid_value(arr.value(i), "STRING", type_name));
}
builder.append_null();
}
}
}
}
Ok(Arc::new(builder.finish()))
}
/// helper to parse floats from string inputs
fn parse_string_to_float<F>(s: &str) -> Option<F>
where
F: FromStr + num::Float,
{
// Handle +inf / -inf
if s.eq_ignore_ascii_case("inf")
|| s.eq_ignore_ascii_case("+inf")
|| s.eq_ignore_ascii_case("infinity")
|| s.eq_ignore_ascii_case("+infinity")
{
return Some(F::infinity());
}
if s.eq_ignore_ascii_case("-inf") || s.eq_ignore_ascii_case("-infinity") {
return Some(F::neg_infinity());
}
if s.eq_ignore_ascii_case("nan") {
return Some(F::nan());
}
// Remove D/F suffix if present
let pruned_float_str =
if s.ends_with("d") || s.ends_with("D") || s.ends_with('f') || s.ends_with('F') {
&s[..s.len() - 1]
} else {
s
};
// Rust's parse logic already handles scientific notations so we just rely on it
pruned_float_str.parse::<F>().ok()
}
pub(crate) fn spark_cast_utf8_to_boolean<OffsetSize>(
from: &dyn Array,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef>
where
OffsetSize: OffsetSizeTrait,
{
let array = from
.as_any()
.downcast_ref::<GenericStringArray<OffsetSize>>()
.unwrap();
let output_array = array
.iter()
.map(|value| match value {
Some(value) => match trim_all(value) {
v if is_true_string(v) => Ok(Some(true)),
v if is_false_string(v) => Ok(Some(false)),
_ if eval_mode == EvalMode::Ansi => Err(SparkError::CastInvalidValue {
value: value.to_string(),
from_type: "STRING".to_string(),
to_type: "BOOLEAN".to_string(),
}),
_ => Ok(None),
},
_ => Ok(None),
})
.collect::<Result<BooleanArray, _>>()?;
Ok(Arc::new(output_array))
}
/// Equivalent to `org.apache.spark.sql.catalyst.util.StringUtils.isTrueString`, minus the trim
/// that the caller has already applied.
///
/// Spark lowercases with `UTF8String.toLowerCase` before comparing, but every candidate is
/// ASCII, and no non-ASCII character lowercases into an ASCII one that would complete any of
/// them, so an ASCII-insensitive comparison gives the same answer without allocating.
#[inline]
fn is_true_string(trimmed: &str) -> bool {
["t", "true", "y", "yes", "1"]
.iter()
.any(|v| trimmed.eq_ignore_ascii_case(v))
}
/// Equivalent to `org.apache.spark.sql.catalyst.util.StringUtils.isFalseString`; see
/// [`is_true_string`] for why the comparison is ASCII-only.
#[inline]
fn is_false_string(trimmed: &str) -> bool {
["f", "false", "n", "no", "0"]
.iter()
.any(|v| trimmed.eq_ignore_ascii_case(v))
}
pub(crate) fn cast_string_to_decimal(
array: &ArrayRef,
to_type: &DataType,
precision: &u8,
scale: &i8,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
match to_type {
DataType::Decimal128(_, _) => {
cast_string_to_decimal128_impl(array, eval_mode, *precision, *scale)
}
DataType::Decimal256(_, _) => {
cast_string_to_decimal256_impl(array, eval_mode, *precision, *scale)
}
_ => Err(SparkError::Internal(format!(
"Unexpected type in cast_string_to_decimal: {:?}",
to_type
))),
}
}
fn cast_string_to_decimal128_impl(
array: &ArrayRef,
eval_mode: EvalMode,
precision: u8,
scale: i8,
) -> SparkResult<ArrayRef> {
let string_array = array
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| SparkError::Internal("Expected string array".to_string()))?;
let mut decimal_builder = Decimal128Builder::with_capacity(string_array.len());
for i in 0..string_array.len() {
if string_array.is_null(i) {
decimal_builder.append_null();
} else {
let str_value = string_array.value(i);
match parse_string_to_decimal(str_value, precision, scale) {
Ok(Some(decimal_value)) => {
decimal_builder.append_value(decimal_value);
}
Ok(None) => {
if eval_mode == EvalMode::Ansi {
return Err(invalid_value(
string_array.value(i),
"STRING",
&format!("DECIMAL({},{})", precision, scale),
));
}
decimal_builder.append_null();
}
Err(e) => {
if eval_mode == EvalMode::Ansi {
return Err(e);
}
decimal_builder.append_null();
}
}
}
}
Ok(Arc::new(
decimal_builder
.with_precision_and_scale(precision, scale)
.map_err(|e| {
if matches!(e, arrow::error::ArrowError::InvalidArgumentError(_))
&& e.to_string().contains("too large to store in a Decimal128")
{
// Fallback error handling
SparkError::NumericValueOutOfRange {
value: "overflow".to_string(),
precision,
scale,
}
} else {
SparkError::Arrow(Arc::new(e))
}
})?
.finish(),
))
}
fn cast_string_to_decimal256_impl(
array: &ArrayRef,
eval_mode: EvalMode,
precision: u8,
scale: i8,
) -> SparkResult<ArrayRef> {
let string_array = array
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| SparkError::Internal("Expected string array".to_string()))?;
let mut decimal_builder = PrimitiveBuilder::<Decimal256Type>::with_capacity(string_array.len());
for i in 0..string_array.len() {
if string_array.is_null(i) {
decimal_builder.append_null();
} else {
let str_value = string_array.value(i);
match parse_string_to_decimal(str_value, precision, scale) {
Ok(Some(decimal_value)) => {
// Convert i128 to i256
let i256_value = i256::from_i128(decimal_value);
decimal_builder.append_value(i256_value);
}
Ok(None) => {
if eval_mode == EvalMode::Ansi {
return Err(invalid_value(
str_value,
"STRING",
&format!("DECIMAL({},{})", precision, scale),
));
}
decimal_builder.append_null();
}
Err(e) => {
if eval_mode == EvalMode::Ansi {
return Err(e);
}
decimal_builder.append_null();
}
}
}
}
Ok(Arc::new(
decimal_builder
.with_precision_and_scale(precision, scale)
.map_err(|e| {
if matches!(e, arrow::error::ArrowError::InvalidArgumentError(_))
&& e.to_string().contains("too large to store in a Decimal128")
{
// Fallback error handling
SparkError::NumericValueOutOfRange {
value: "overflow".to_string(),
precision,
scale,
}
} else {
SparkError::Arrow(Arc::new(e))
}
})?
.finish(),
))
}
/// Normalize fullwidth Unicode digits (U+FF10–U+FF19) to their ASCII equivalents.
///
/// Spark's UTF8String parser treats fullwidth digits as numerically equivalent to
/// ASCII digits, e.g. "１２３.４５" parses as 123.45. Each fullwidth digit encodes
/// to exactly three UTF-8 bytes: [0xEF, 0xBC, 0x90+n] for digit n. The ASCII
/// equivalent is 0x30+n, so the conversion is: third_byte - 0x60.
///
/// All other bytes (ASCII or other multi-byte sequences) are passed through
/// unchanged, so the output is valid UTF-8 whenever the input is.
fn normalize_fullwidth_digits(s: &str) -> String {
let bytes = s.as_bytes();
let mut out = Vec::with_capacity(s.len());
let mut i = 0;
while i < bytes.len() {
if i + 2 < bytes.len()
&& bytes[i] == 0xEF
&& bytes[i + 1] == 0xBC
&& bytes[i + 2] >= 0x90
&& bytes[i + 2] <= 0x99
{
// e.g. 0x91 - 0x60 = 0x31 = b'1'
out.push(bytes[i + 2] - 0x60);
i += 3;
} else {
out.push(bytes[i]);
i += 1;
}
}
// SAFETY: we only replace valid 3-byte UTF-8 sequences [EF BC 9X] with a
// single ASCII byte; all other bytes are copied unchanged, preserving the
// UTF-8 invariant of the input.
unsafe { String::from_utf8_unchecked(out) }
}
/// Powers of ten that fit in an `i128` (`10^0` through `10^38`).
const POW10_I128: [i128; 39] = {
let mut table = [1i128; 39];
let mut i = 1;
while i < 39 {
table[i] = table[i - 1] * 10;
i += 1;
}
table
};
/// `10^exp`, or `None` when the exponent overflows an `i128` (exp >= 39).
#[inline]
pub(crate) fn pow10_i128(exp: u32) -> Option<i128> {
POW10_I128.get(exp as usize).copied()
}
/// Divide by a power of ten with HALF_UP rounding, matching `BigDecimal.setScale`: a tie
/// rounds away from zero.
///
/// `divisor` must be `10^n` for `n >= 1`, so `divisor / 2` is exact and a zero remainder
/// can never be mistaken for a tie.
#[inline]
pub(crate) fn div_round_half_up_i128(numerator: i128, divisor: i128) -> i128 {
debug_assert!(divisor >= 10);
let quotient = numerator / divisor;
let remainder = numerator % divisor;
if remainder.abs() >= divisor / 2 {
quotient + numerator.signum()
} else {
quotient
}
}
/// Accumulate an ASCII-digit slice into an `i128`, returning `None` on overflow.
///
/// The first 38 digits always fit (`i128::MAX` is ~1.7e38), so only the digits past
/// them need the per-digit overflow checks.
#[inline]
pub(crate) fn digits_to_i128(digits: &[u8]) -> Option<i128> {
let (head, tail) = digits.split_at(digits.len().min(38));
let mut value: i128 = 0;
for &d in head {
value = value * 10 + (d - b'0') as i128;
}
for &d in tail {
value = value.checked_mul(10)?.checked_add((d - b'0') as i128)?;
}
Some(value)
}
/// Values that Spark parses as NULL rather than as a decimal, matched case-insensitively.
const SPECIAL_DECIMAL_VALUES: [&str; 7] = [
"inf",
"+inf",
"-inf",
"infinity",
"+infinity",
"-infinity",
"nan",
];
/// True if `trimmed` is one of [`SPECIAL_DECIMAL_VALUES`].
#[inline]
fn is_special_value(trimmed: &str) -> bool {
// Every special value starts with `i`/`n`, or with a sign followed by `i`, so
// ordinary numeric input is ruled out after inspecting a single byte.
let bytes = trimmed.as_bytes();
let plausible = match bytes.first() {
Some(b'i' | b'I' | b'n' | b'N') => true,
Some(b'+' | b'-') => matches!(bytes.get(1), Some(b'i' | b'I')),
_ => false,
};
plausible
&& SPECIAL_DECIMAL_VALUES
.iter()
.any(|v| trimmed.eq_ignore_ascii_case(v))
}
/// Parse a decimal string into mantissa and scale
/// e.g., "123.45" -> (12345, 2), "-0.001" -> (-1, 3) , 0e50 -> (0,50) etc
/// Parse a string to decimal following Spark's behavior
fn parse_string_to_decimal(input_str: &str, precision: u8, scale: i8) -> SparkResult<Option<i128>> {
// Spark parses via `new java.math.BigDecimal(str.toString.trim)`, so the trim set is
// `String.trim`'s: every byte <= 0x20, which includes the null byte ("123\u0000" and
// "\u0000123" both parse as 123) but not 0x7F or any non-ASCII whitespace. Null bytes in
// the middle are not trimmed and will fail the digit validation in parse_decimal_str,
// producing NULL.
let trimmed = trim_java_string(input_str);
// Normalize fullwidth digits to ASCII. Fast path skips the allocation for
// pure-ASCII strings, which is the common case.
let normalized;
let trimmed = if trimmed.is_ascii() {
trimmed
} else {
normalized = normalize_fullwidth_digits(trimmed);
normalized.as_str()
};
if trimmed.is_empty() {
return Ok(None);
}
// Handle special values (inf, nan, etc.)
if is_special_value(trimmed) {
return Ok(None);
}
// validate and parse mantissa and exponent or bubble up the error
let (mantissa, exponent) = parse_decimal_str(trimmed, input_str, precision, scale)?;
// Early return mantissa 0, Spark checks if it fits digits and throw error in ansi
if mantissa == 0 {
if exponent < -37 {
return Err(SparkError::NumericOutOfRange {
value: input_str.to_string(),
});
}
return Ok(Some(0));
}
// scale adjustment
let target_scale = scale as i32;
let scale_adjustment = target_scale - exponent;
let scaled_value = if scale_adjustment >= 0 {
// Need to multiply (increase scale) but return None if scale is too high to fit i128
if scale_adjustment > 38 {
return Ok(None);
}
// Bounded above, so pow10_i128 always returns Some.
mantissa.checked_mul(pow10_i128(scale_adjustment as u32).unwrap())
} else {
// Need to divide (decrease scale)
let abs_scale_adjustment = (-scale_adjustment) as u32;
if abs_scale_adjustment > 38 {
return Ok(Some(0));
}
// Bounded above, so pow10_i128 always returns Some. The adjustment is at least 1
// here, so the divisor is a power of ten no smaller than 10.
let divisor = pow10_i128(abs_scale_adjustment).unwrap();
Some(div_round_half_up_i128(mantissa, divisor))
};
match scaled_value {
Some(value) => {
if is_validate_decimal_precision(value, precision) {
Ok(Some(value))
} else {
// Value ok but exceeds precision mentioned . THrow error
Err(SparkError::NumericValueOutOfRange {
value: trimmed.to_string(),
precision,
scale,
})
}
}
None => {
// Overflow when scaling raise exception
Err(SparkError::NumericValueOutOfRange {
value: trimmed.to_string(),
precision,
scale,
})
}
}
}
fn invalid_decimal_cast(value: &str, precision: u8, scale: i8) -> SparkError {
invalid_value(
value,
"STRING",
&format!("DECIMAL({},{})", precision, scale),
)
}
/// Parse a decimal string into mantissa and scale
/// e.g., "123.45" -> (12345, 2), "-0.001" -> (-1, 3) , 0e50 -> (0,50) etc
fn parse_decimal_str(
s: &str,
original_str: &str,
precision: u8,
scale: i8,
) -> SparkResult<(i128, i32)> {
let bytes = s.as_bytes();
let mut pos = 0;
let negative = match bytes.first() {
Some(b'-') => {
pos = 1;
true
}
Some(b'+') => {
pos = 1;
false
}
_ => false,
};
// Single validating pass over the mantissa: ASCII digits with at most one `.`,
// ending at the optional exponent marker. It also locates the integral/fractional
// split and the start of the exponent. `.`, `e` and `E` are ASCII, so scanning
// bytes can never land inside a multi-byte character and every index taken here is
// on a char boundary.
let digits_start = pos;
let mut dot_pos = None;
let mut exp_pos = None;
while pos < bytes.len() {
match bytes[pos] {
b'0'..=b'9' => pos += 1,
b'.' if dot_pos.is_none() => {
dot_pos = Some(pos);
pos += 1;
}
b'e' | b'E' => {
exp_pos = Some(pos);
break;
}
_ => return Err(invalid_decimal_cast(original_str, precision, scale)),
}
}
let exponent: i32 = match exp_pos {
Some(e_pos) => s[e_pos + 1..]
.parse()
.map_err(|_| invalid_decimal_cast(original_str, precision, scale))?,
None => 0,
};
// An empty integral part is valid (e.g. ".5" or "-.7e9"), as is an empty
// fractional part, but they cannot both be empty.
let mantissa_end = exp_pos.unwrap_or(pos);
let (integral_part, fractional_part): (&[u8], &[u8]) = match dot_pos {
Some(dot) => (&bytes[digits_start..dot], &bytes[dot + 1..mantissa_end]),
None => (&bytes[digits_start..mantissa_end], &[]),
};
if integral_part.is_empty() && fractional_part.is_empty() {
return Err(invalid_decimal_cast(original_str, precision, scale));
}
let integral_value = digits_to_i128(integral_part)
.ok_or_else(|| invalid_decimal_cast(original_str, precision, scale))?;
let fractional_scale = fractional_part.len() as i32;
let fractional_value = digits_to_i128(fractional_part)
.ok_or_else(|| invalid_decimal_cast(original_str, precision, scale))?;
// Combine: value = integral * 10^fractional_scale + fractional.
// A fractional_scale beyond 38 cannot fit in an i128, so pow10_i128 returns None and this
// maps to the invalid-decimal error path instead of panicking.
let mantissa = pow10_i128(fractional_scale as u32)
.and_then(|p| integral_value.checked_mul(p))
.and_then(|v| v.checked_add(fractional_value))
.ok_or_else(|| invalid_decimal_cast(original_str, precision, scale))?;
let final_mantissa = if negative { -mantissa } else { mantissa };
// final scale = fractional_scale - exponent
// For example : "1.23E-5" has fractional_scale=2, exponent=-5, so scale = 2 - (-5) = 7
let final_scale = fractional_scale - exponent;
Ok((final_mantissa, final_scale))
}
pub(crate) fn cast_string_to_date(
array: &ArrayRef,
to_type: &DataType,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<i32>>()
.expect("Expected a string array");
if to_type != &DataType::Date32 {
unreachable!("Invalid data type {:?} in cast from string", to_type);
}
let len = string_array.len();
let mut cast_array = PrimitiveArray::<Date32Type>::builder(len);
for i in 0..len {
let value = if string_array.is_null(i) {
None
} else {
match date_parser(string_array.value(i), eval_mode) {
Ok(Some(cast_value)) => Some(cast_value),
Ok(None) => None,
Err(e) => return Err(e),
}
};
match value {
Some(cast_value) => cast_array.append_value(cast_value),
None => cast_array.append_null(),
}
}
Ok(Arc::new(cast_array.finish()) as ArrayRef)
}
pub(crate) fn cast_string_to_timestamp(
array: &ArrayRef,
to_type: &DataType,
eval_mode: EvalMode,
timezone_str: &str,
is_spark4_plus: bool,
) -> SparkResult<ArrayRef> {
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<i32>>()
.expect("Expected a string array");
let tz = &Tz::from_str(timezone_str)
.map_err(|_| SparkError::Internal(format!("Invalid timezone string: {timezone_str}")))?;
let cast_array: ArrayRef = match to_type {
DataType::Timestamp(_, tz_opt) => {
let to_tz = tz_opt.as_deref().unwrap_or("UTC");
cast_utf8_to_timestamp!(
string_array,
eval_mode,
PrimitiveArray::<TimestampMicrosecondType>::builder(string_array.len())
.with_timezone(to_tz),
timestamp_parser,
tz,
is_spark4_plus
)?
}
_ => unreachable!("Invalid data type {:?} in cast from string", to_type),
};
Ok(cast_array)
}
pub(crate) fn cast_string_to_timestamp_ntz(
array: &ArrayRef,
eval_mode: EvalMode,
allow_time_zone: bool,
is_spark4_plus: bool,
) -> SparkResult<ArrayRef> {
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<i32>>()
.expect("Expected a string array");
let cast_array: ArrayRef = cast_utf8_to_timestamp!(
string_array,
eval_mode,
PrimitiveArray::<TimestampMicrosecondType>::builder(string_array.len()),
timestamp_ntz_parser,
allow_time_zone,
is_spark4_plus
)?;
Ok(cast_array)
}
pub(crate) fn cast_string_to_int<OffsetSize: OffsetSizeTrait>(
to_type: &DataType,
array: &ArrayRef,
eval_mode: EvalMode,
) -> SparkResult<ArrayRef> {
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<OffsetSize>>()
.expect("cast_string_to_int expected a string array");
// Select parse function once per batch based on eval_mode
let cast_array: ArrayRef =
match (to_type, eval_mode) {
(DataType::Int8, EvalMode::Legacy) => {
cast_utf8_to_int!(string_array, Int8Type, parse_string_to_i8_legacy)?
}
(DataType::Int8, EvalMode::Ansi) => {
cast_utf8_to_int!(string_array, Int8Type, parse_string_to_i8_ansi)?
}
(DataType::Int8, EvalMode::Try) => {
cast_utf8_to_int!(string_array, Int8Type, parse_string_to_i8_try)?
}
(DataType::Int16, EvalMode::Legacy) => {
cast_utf8_to_int!(string_array, Int16Type, parse_string_to_i16_legacy)?
}
(DataType::Int16, EvalMode::Ansi) => {
cast_utf8_to_int!(string_array, Int16Type, parse_string_to_i16_ansi)?
}
(DataType::Int16, EvalMode::Try) => {
cast_utf8_to_int!(string_array, Int16Type, parse_string_to_i16_try)?
}
(DataType::Int32, EvalMode::Legacy) => cast_utf8_to_int!(
string_array,
Int32Type,
|s| do_parse_string_to_int_legacy::<i32>(s, i32::MIN)
)?,
(DataType::Int32, EvalMode::Ansi) => {
cast_utf8_to_int!(string_array, Int32Type, |s| do_parse_string_to_int_ansi::<
i32,
>(
s, "INT", i32::MIN
))?
}
(DataType::Int32, EvalMode::Try) => {
cast_utf8_to_int!(
string_array,
Int32Type,
|s| do_parse_string_to_int_try::<i32>(s, i32::MIN)
)?
}
(DataType::Int64, EvalMode::Legacy) => cast_utf8_to_int!(
string_array,
Int64Type,
|s| do_parse_string_to_int_legacy::<i64>(s, i64::MIN)
)?,
(DataType::Int64, EvalMode::Ansi) => {
cast_utf8_to_int!(string_array, Int64Type, |s| do_parse_string_to_int_ansi::<
i64,
>(
s, "BIGINT", i64::MIN
))?
}
(DataType::Int64, EvalMode::Try) => {
cast_utf8_to_int!(
string_array,
Int64Type,
|s| do_parse_string_to_int_try::<i64>(s, i64::MIN)
)?
}
(dt, _) => unreachable!(
"{}",
format!("invalid integer type {dt} in cast from string")
),
};
Ok(cast_array)
}
/// Finalizes the result by applying the sign. Returns None if overflow would occur.
fn finalize_int_result<T: Integer + CheckedNeg + Copy>(result: T, negative: bool) -> Option<T> {
if negative {
Some(result)
} else {
result.checked_neg().filter(|&n| n >= T::zero())
}
}
/// Equivalent to
/// - org.apache.spark.unsafe.types.UTF8String.toInt(IntWrapper intWrapper, boolean allowDecimal)
/// - org.apache.spark.unsafe.types.UTF8String.toLong(LongWrapper longWrapper, boolean allowDecimal)
fn do_parse_string_to_int_legacy<T: Integer + CheckedSub + CheckedNeg + From<u8> + Copy>(
str: &str,
min_value: T,
) -> SparkResult<Option<T>> {
let trimmed_bytes = trim_all_bytes(str.as_bytes());
let (negative, digits) = match parse_sign(trimmed_bytes) {
Some(result) => result,
None => return Ok(None),
};
let mut result: T = T::zero();
let radix = T::from(10_u8);
let stop_value = min_value / radix;
let mut iter = digits.iter();
// Parse integer portion until '.' or end
for &ch in iter.by_ref() {
if ch == b'.' {
break;
}
if !ch.is_ascii_digit() {
return Ok(None);
}
if result < stop_value {
return Ok(None);
}
let v = result * radix;
let digit: T = T::from(ch - b'0');
match v.checked_sub(&digit) {
Some(x) if x <= T::zero() => result = x,
_ => return Ok(None),
}
}
// Validate decimal portion (digits only, values ignored)
for &ch in iter {
if !ch.is_ascii_digit() {
return Ok(None);
}
}
Ok(finalize_int_result(result, negative))
}
fn do_parse_string_to_int_ansi<T: Integer + CheckedSub + CheckedNeg + From<u8> + Copy>(
str: &str,
type_name: &str,
min_value: T,
) -> SparkResult<Option<T>> {
let error = || Err(invalid_value(str, "STRING", type_name));
let trimmed_bytes = trim_all_bytes(str.as_bytes());
let (negative, digits) = match parse_sign(trimmed_bytes) {
Some(result) => result,
None => return error(),
};
let mut result: T = T::zero();
let radix = T::from(10_u8);
let stop_value = min_value / radix;
for &ch in digits {
if ch == b'.' || !ch.is_ascii_digit() {
return error();
}
if result < stop_value {
return error();
}
let v = result * radix;
let digit: T = T::from(ch - b'0');
match v.checked_sub(&digit) {
Some(x) if x <= T::zero() => result = x,
_ => return error(),
}
}
finalize_int_result(result, negative)
.map(Some)
.ok_or_else(|| invalid_value(str, "STRING", type_name))
}
fn do_parse_string_to_int_try<T: Integer + CheckedSub + CheckedNeg + From<u8> + Copy>(
str: &str,
min_value: T,
) -> SparkResult<Option<T>> {
let trimmed_bytes = trim_all_bytes(str.as_bytes());
let (negative, digits) = match parse_sign(trimmed_bytes) {
Some(result) => result,
None => return Ok(None),
};
let mut result: T = T::zero();
let radix = T::from(10_u8);
let stop_value = min_value / radix;
for &ch in digits {
if ch == b'.' || !ch.is_ascii_digit() {
return Ok(None);
}
if result < stop_value {
return Ok(None);
}
let v = result * radix;
let digit: T = T::from(ch - b'0');
match v.checked_sub(&digit) {
Some(x) if x <= T::zero() => result = x,
_ => return Ok(None),
}
}
Ok(finalize_int_result(result, negative))
}
fn parse_string_to_i8_legacy(str: &str) -> SparkResult<Option<i8>> {
match do_parse_string_to_int_legacy::<i32>(str, i32::MIN)? {
Some(v) if v >= i8::MIN as i32 && v <= i8::MAX as i32 => Ok(Some(v as i8)),
_ => Ok(None),
}
}
fn parse_string_to_i8_ansi(str: &str) -> SparkResult<Option<i8>> {
match do_parse_string_to_int_ansi::<i32>(str, "TINYINT", i32::MIN)? {
Some(v) if v >= i8::MIN as i32 && v <= i8::MAX as i32 => Ok(Some(v as i8)),
_ => Err(invalid_value(str, "STRING", "TINYINT")),
}
}
fn parse_string_to_i8_try(str: &str) -> SparkResult<Option<i8>> {
match do_parse_string_to_int_try::<i32>(str, i32::MIN)? {
Some(v) if v >= i8::MIN as i32 && v <= i8::MAX as i32 => Ok(Some(v as i8)),
_ => Ok(None),
}
}
fn parse_string_to_i16_legacy(str: &str) -> SparkResult<Option<i16>> {
match do_parse_string_to_int_legacy::<i32>(str, i32::MIN)? {
Some(v) if v >= i16::MIN as i32 && v <= i16::MAX as i32 => Ok(Some(v as i16)),
_ => Ok(None),
}
}
fn parse_string_to_i16_ansi(str: &str) -> SparkResult<Option<i16>> {
match do_parse_string_to_int_ansi::<i32>(str, "SMALLINT", i32::MIN)? {
Some(v) if v >= i16::MIN as i32 && v <= i16::MAX as i32 => Ok(Some(v as i16)),
_ => Err(invalid_value(str, "STRING", "SMALLINT")),
}
}
fn parse_string_to_i16_try(str: &str) -> SparkResult<Option<i16>> {
match do_parse_string_to_int_try::<i32>(str, i32::MIN)? {
Some(v) if v >= i16::MIN as i32 && v <= i16::MAX as i32 => Ok(Some(v as i16)),
_ => Ok(None),
}
}
/// Parses sign and returns (is_negative, remaining_bytes after sign)
/// Returns None if invalid (empty input, or just "+" or "-")
fn parse_sign(bytes: &[u8]) -> Option<(bool, &[u8])> {
let (&first, rest) = bytes.split_first()?;
match first {
b'-' if !rest.is_empty() => Some((true, rest)),
b'+' if !rest.is_empty() => Some((false, rest)),
_ => Some((false, bytes)),
}
}
#[inline]
pub fn invalid_value(value: &str, from_type: &str, to_type: &str) -> SparkError {
SparkError::CastInvalidValue {
value: value.to_string(),
from_type: from_type.to_string(),
to_type: to_type.to_string(),
}
}
fn parse_to_timestamp_info(
value: &str,
timestamp_type: &str,
) -> SparkResult<Option<TimeStampInfo>> {
let (sign, date_part) = if let Some(stripped) = value.strip_prefix('-') {
(-1i32, stripped)
} else {
(1i32, value)
};
let mut parts = date_part.split(['T', ' ', '-', ':', '.']);
// The integer parser accepts a leading '+', which is not a segment separator.
let year = sign
* parts
.next()
.unwrap_or("")
.parse::<i32>()
.unwrap_or_default();
// Guard against years that cannot produce a valid i64 microsecond timestamp.
// The Long.MaxValue/MinValue boundaries correspond to years 294247 / -290308.
// We allow a slightly wider range and let parse_timestamp_to_micros perform the
// exact overflow check via checked arithmetic.
if !(-290309..=294248).contains(&year) {
return Ok(None);
}
let month = parts.next().map_or(1, |m| m.parse::<u32>().unwrap_or(1));
let day = parts.next().map_or(1, |d| d.parse::<u32>().unwrap_or(1));
let hour = parts.next().map_or(0, |h| h.parse::<u32>().unwrap_or(0));
let minute = parts.next().map_or(0, |m| m.parse::<u32>().unwrap_or(0));
let second = parts.next().map_or(0, |s| s.parse::<u32>().unwrap_or(0));
let microsecond = if let Some(ms) = parts.next() {
let Some(ms) = ms.get(..ms.len().min(6)) else {
return Ok(None);
};
let n = ms.len();
ms.parse::<u32>().unwrap_or(0) * 10u32.pow((6 - n) as u32)
} else {
0
};
let mut timestamp_info = TimeStampInfo::default();
let timestamp_info = match timestamp_type {
"year" => timestamp_info.with_year(year),
"month" => timestamp_info.with_year(year).with_month(month),
"day" => timestamp_info
.with_year(year)
.with_month(month)
.with_day(day),
"hour" => timestamp_info
.with_year(year)
.with_month(month)
.with_day(day)
.with_hour(hour),
"minute" => timestamp_info
.with_year(year)
.with_month(month)
.with_day(day)
.with_hour(hour)
.with_minute(minute),
"second" => timestamp_info
.with_year(year)
.with_month(month)
.with_day(day)
.with_hour(hour)
.with_minute(minute)
.with_second(second),
"microsecond" => timestamp_info
.with_year(year)
.with_month(month)
.with_day(day)
.with_hour(hour)
.with_minute(minute)
.with_second(second)
.with_microsecond(microsecond),
_ => {
return Err(SparkError::InvalidInputInCastToDatetime {
value: value.to_string(),
from_type: "STRING".to_string(),
to_type: "TIMESTAMP".to_string(),
})
}
};
Ok(Some(timestamp_info.to_owned()))
}
fn get_timestamp_values<T: TimeZone>(
value: &str,
timestamp_type: &str,
tz: &T,
) -> SparkResult<Option<i64>> {
match parse_to_timestamp_info(value, timestamp_type)? {
Some(info) => parse_timestamp_to_micros(&info, tz),
None => Ok(None),
}
}
/// Howard Hinnant's algorithm: proleptic Gregorian days since 1970-01-01 for any i64 year.
/// Works correctly for positive and negative years via Euclidean floor division.
/// Spark uses Java's equivalent [LocalDate.toEpochDay](https://github.com/openjdk/jdk/blob/cddee6d6eb3e048635c380a32bd2f6ebfd2c18b5/src/java.base/share/classes/java/time/LocalDate.java#L1954)
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
let (y, m) = if m <= 2 { (y - 1, m + 9) } else { (y, m - 3) };
let era = if y >= 0 { y / 400 } else { (y - 399) / 400 };
let yoe = y - era * 400; // year of era [0, 399]
let doy = (153 * m + 2) / 5 + d - 1; // day of year [0, 365]
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // day of era [0, 146096]
era * 146097 + doe - 719468
}
fn is_leap_year(year: i64) -> bool {
year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)
}
/// Days since 1970-01-01 for a proleptic Gregorian year/month/day, or `None` when the
/// combination is not a real calendar date. Unlike `NaiveDate::from_ymd_opt`, this accepts
/// any year that fits in `i64`.
pub(crate) fn ymd_to_epoch_day(year: i64, month: i64, day: i64) -> Option<i64> {
const DAYS_IN_MONTH: [i64; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
let mut max_day = *DAYS_IN_MONTH.get(usize::try_from(month.checked_sub(1)?).ok()?)?;
if month == 2 && is_leap_year(year) {
max_day = 29;
}
if day < 1 || day > max_day {
return None;
}
Some(days_from_civil(year, month, day))
}
fn parse_timestamp_to_micros<T: TimeZone>(
timestamp_info: &TimeStampInfo,
tz: &T,
) -> SparkResult<Option<i64>> {
// Build NaiveDateTime explicitly so we can pattern-match LocalResult variants and
// handle the DST spring-forward gap case.
let naive_date_opt = NaiveDate::from_ymd_opt(
timestamp_info.year,
timestamp_info.month,
timestamp_info.day,
);
// NaiveTime is used for the common path; also validates hour/min/sec.
let naive_time = match NaiveTime::from_hms_opt(
timestamp_info.hour,
timestamp_info.minute,
timestamp_info.second,
) {
Some(t) => t,
None => return Ok(None), // invalid time components
};
if let Some(naive_date) = naive_date_opt {
let local_naive = naive_date.and_time(naive_time);
// Resolve local datetime to UTC, handling DST transitions.
// We compute base_micros with second precision (local_naive has no sub-second component),
// then add microseconds at the end to avoid calling with_nanosecond(), which internally
// calls from_local_datetime().single() and returns None for ambiguous (fall-back) times.
let base_micros: Option<i64> = match tz.from_local_datetime(&local_naive) {
// Unambiguous local time.
LocalResult::Single(dt) => Some(dt.timestamp_micros()),
// DST fall-back overlap: Spark picks the earlier UTC instant (pre-transition offset).
LocalResult::Ambiguous(earlier, _) => Some(earlier.timestamp_micros()),
// DST spring-forward gap: the local time does not exist.
// Java's ZonedDateTime.of() advances by the gap length, which is equivalent to
// utc = local_naive − pre_gap_offset
LocalResult::None => {
let probe = local_naive - chrono::Duration::hours(3);
let pre_offset = match tz.from_local_datetime(&probe) {
LocalResult::Single(dt) => dt.offset().fix(),
LocalResult::Ambiguous(dt, _) => dt.offset().fix(),
LocalResult::None => return Ok(None),
};
let offset_secs = pre_offset.local_minus_utc() as i64;
let utc_naive = local_naive - chrono::Duration::seconds(offset_secs);
Some(utc_naive.and_utc().timestamp_micros())
}
};
Ok(base_micros.map(|m| m + timestamp_info.microsecond as i64))
} else {
// NaiveDate::from_ymd_opt returned None. This means either:
// (a) invalid calendar date (Feb 29 on non-leap year, month 13, etc.)
// (b) year outside chrono's representable range (> 262143 or < -262144)
//
// For case (b) we fall back to Howard Hinnant's direct arithmetic, which works
// for any year that fits in i64. This covers the Long.MaxValue / Long.MinValue
// boundary timestamps (year 294247 / -290308).
let year = timestamp_info.year as i64;
if (-262144..=262143).contains(&year) {
// Year is in chrono's range but date was rejected -> truly invalid date.
return Ok(None);
}
// Validate month and day manually for extreme years.
let days =
match ymd_to_epoch_day(year, timestamp_info.month as i64, timestamp_info.day as i64) {
Some(days) => days,
None => return Ok(None),
};
// Compute the timezone offset using epoch as a surrogate probe point.
// Extreme-year timestamps are only valid with a UTC-like fixed offset (any DST
// zone would overflow). Using epoch gives us the standard offset.
let epoch_probe = NaiveDate::from_ymd_opt(1970, 1, 1)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
let tz_offset_secs: i64 = match tz.from_local_datetime(&epoch_probe) {
LocalResult::Single(dt) => dt.offset().fix().local_minus_utc() as i64,
LocalResult::Ambiguous(dt, _) => dt.offset().fix().local_minus_utc() as i64,
LocalResult::None => 0,
};
// Compute seconds since epoch via direct calendar arithmetic.
// Use i128 for the intermediate multiply-by-1_000_000 step: the seconds value can be
// just outside the i64 range while the final microseconds result is still within range
// (e.g., Long.MinValue boundary: seconds = -9_223_372_036_855, result = i64::MIN).
let time_secs = timestamp_info.hour as i64 * 3600
+ timestamp_info.minute as i64 * 60
+ timestamp_info.second as i64;
let total_secs = days
.checked_mul(86400)
.and_then(|s| s.checked_add(time_secs))
.and_then(|s| s.checked_sub(tz_offset_secs));
let utc_micros = total_secs.and_then(|s| {
let micros128 = s as i128 * 1_000_000 + timestamp_info.microsecond as i128;
i64::try_from(micros128).ok()
});
Ok(utc_micros)
}
}
fn local_datetime_to_micros(timestamp_info: &TimeStampInfo) -> SparkResult<Option<i64>> {
let year = timestamp_info.year as i64;
let days = match ymd_to_epoch_day(year, timestamp_info.month as i64, timestamp_info.day as i64)
{
Some(days) => days,
None => return Ok(None),
};
if timestamp_info.hour >= 24 || timestamp_info.minute >= 60 || timestamp_info.second >= 60 {
return Ok(None);
}
let time_secs = timestamp_info.hour as i64 * 3600
+ timestamp_info.minute as i64 * 60
+ timestamp_info.second as i64;
let total_secs = days
.checked_mul(86400)
.and_then(|s| s.checked_add(time_secs));
let micros = total_secs.and_then(|s| {
let micros128 = s as i128 * 1_000_000 + timestamp_info.microsecond as i128;
i64::try_from(micros128).ok()
});
Ok(micros)
}
fn timestamp_parser<T: TimeZone>(
value: &str,
eval_mode: EvalMode,
tz: &T,
is_spark4_plus: bool,
) -> SparkResult<Option<i64>> {
// `SparkDateTimeUtils.getTrimmedStart`/`getTrimmedEnd` trim the same byte set as
// `UTF8String.trimAll`.
let (start, end) = trim_all_range(value.as_bytes());
let trimmed = &value[start..end];
// A value that trims to nothing has no segments, so Spark treats it as malformed like any
// other: null, or CAST_INVALID_INPUT under ANSI.
//
// Spark 4.0+ also rejects leading padding for ALL T-prefixed time-only strings
// (T<h>, T<h>:<m>, T<h>:<m>:<s>, T<h>:<m>:<s>.<f>), but accepts trailing padding: it only
// treats the `T` as a time-only marker at raw byte 0. Spark 3.x trims first, so leading
// padding is accepted there.
// Check the prefix, not the base patterns: a zone suffix can hide a time-only match.
if trimmed.is_empty() || (is_spark4_plus && start > 0 && trimmed.starts_with('T')) {
return if eval_mode == EvalMode::Ansi {
Err(SparkError::InvalidInputInCastToDatetime {
value: value.to_string(),
from_type: "STRING".to_string(),
to_type: "TIMESTAMP".to_string(),
})
} else {
Ok(None)
};
}
let value = trimmed;
// Classify the shape once: the result decides both whether an offset suffix may be
// stripped and which parse routine runs below.
let pattern = classify_timestamp_pattern(value);
// Only attempt offset-suffix extraction when the value does not already match a
// base pattern. This prevents the '-' in plain date strings like "2015-03-18"
// from being misidentified as a negative-offset sign.
if pattern.is_none() {
if let Some((stripped, suffix_tz)) = extract_offset_suffix(value) {
// Spark applies Java String.trim to the zone, not Unicode whitespace trimming.
let stripped = stripped.trim_end_matches(|c: char| c <= '\u{20}');
// A zone suffix is only meaningful after the seconds segment. Otherwise fall
// through with the unstripped value, which no base pattern matches, so it is
// reported as malformed (null, or CAST_INVALID_INPUT under ANSI) like Spark does.
let stripped_pattern = classify_timestamp_pattern(stripped);
if stripped_pattern.is_some_and(TimestampPattern::ends_with_seconds_segment) {
return parse_timestamp_pattern(stripped, stripped_pattern, eval_mode, &suffix_tz);
}
}
}
parse_timestamp_pattern(value, pattern, eval_mode, tz)
}
/// Parses the portion of an offset string AFTER any "UTC"/"GMT"/"UT" prefix (or the
/// full bare +/- offset including its sign character). Returns the offset in whole seconds,
/// or `None` for any malformed, out-of-range, or trailing-garbage input.
///
/// Accepted formats (H = 1–2 digit hour, M = 1–2 digit minute):
/// "" -> 0 (bare "UTC" / "GMT" / "UT")
/// "+H" -> +H*3600 (hour-only, e.g. "+0" from "UTC+0")
/// "+HH" -> same
/// "+HHMM" -> +H*3600+M*60 (4 digits, no colon)
/// "+H:M" -> same (with colon, any digit count 1-2 each)
/// "+HH:MM" -> same
/// (negative with '-' analogously)
///
/// Hours must be 0–18 and minutes 0–59, with a maximum absolute offset of 18:00.
/// A trailing colon ("+8:") is rejected.
fn parse_sign_offset(s: &str) -> Option<i32> {
if s.is_empty() {
return Some(0);
}
let (sign, rest) = match s.as_bytes().first() {
Some(&b'+') => (1i32, &s[1..]),
Some(&b'-') => (-1i32, &s[1..]),
_ => return None,
};
// Validate before slicing: malformed date segments can reach this helper, and a
// byte range such as rest[..2] must not split a non-ASCII digit's UTF-8 encoding.
if rest.is_empty() || !rest.bytes().all(|b| b.is_ascii_digit() || b == b':') {
return None;
}
let (h, m) = if let Some(colon_pos) = rest.find(':') {
let h_str = &rest[..colon_pos];
let m_str = &rest[colon_pos + 1..];
if !(1..=2).contains(&h_str.len()) || !(1..=2).contains(&m_str.len()) {
return None;
}
let h: i32 = h_str.parse().ok()?;
// Note: "+HH:MM:SS" (with seconds) is not handled; Spark accepts it but it is rare.
let m: i32 = m_str.parse().ok()?;
(h, m)
} else {
match rest.len() {
1 | 2 => (rest.parse::<i32>().ok()?, 0),
4 => (
rest[..2].parse::<i32>().ok()?,
rest[2..].parse::<i32>().ok()?,
),
_ => return None,
}
};
if !(0..=18).contains(&h) || !(0..=59).contains(&m) || (h == 18 && m != 0) {
return None;
}
Some(sign * (h * 3600 + m * 60))
}
/// Constructs a [`Tz`] from an offset measured in seconds.
/// E.g. `+7*3600 + 30*60` -> `"+07:30"`.
fn tz_from_offset_secs(secs: i32) -> Option<Tz> {
let abs = secs.abs();
let h = abs / 3600;
let m = (abs % 3600) / 60;
let sign = if secs >= 0 { '+' } else { '-' };
Tz::from_str(&format!("{}{:02}:{:02}", sign, h, m)).ok()
}
/// Returns the last (rightmost) byte position where `needle` starts inside `haystack`.
fn rfind_str(haystack: &str, needle: &str) -> Option<usize> {
let hb = haystack.as_bytes();
let nb = needle.as_bytes();
if nb.len() > hb.len() {
return None;
}
(0..=(hb.len() - nb.len()))
.rev()
.find(|&i| hb[i..].starts_with(nb))
}
/// If `value` ends with a recognised timezone suffix, returns `(datetime_prefix, Tz)`.
/// Returns `None` when no suffix is found.
///
/// Recognised forms (in matching priority order):
/// Z -> UTC+0
/// UTC / " UTC" -> UTC+0 (or UTC +/- offset, e.g. "UTC+0", " UTC+07:30")
/// GMT / " GMT" -> UTC+0 (or GMT +/- offset)
/// UT / " UT" -> UTC+0 (or UT +/- offset)
/// Named IANA zone -> e.g. " Europe/Moscow"
/// Bare +/-offset -> e.g. "+07:30", "-1:0", "+0730"
///
/// **The caller must ensure the value does not already match a base timestamp pattern.**
/// Without that guard a bare '-' in "2015-03-18" would be misread as a -18:00 offset.
fn extract_offset_suffix(value: &str) -> Option<(&str, Tz)> {
// 1. Z suffix
if let Some(stripped) = value.strip_suffix('Z') {
return Some((stripped, tz_from_offset_secs(0)?));
}
// 2. Named text-prefix forms: "UTC", "GMT", "UT" (optionally space-prefixed),
// each optionally followed by a bare +/-offset.
// Longest first so " UTC" is tried before " UT", etc.
for prefix in &[" UTC", "UTC", " GMT", "GMT", " UT", "UT"] {
if let Some(pos) = rfind_str(value, prefix) {
let offset_str = &value[pos + prefix.len()..];
if let Some(secs) = parse_sign_offset(offset_str) {
return Some((&value[..pos], tz_from_offset_secs(secs)?));
}
}
}
// 3. Java SHORT_IDS fixed-offset abbreviations recognised by ZoneId.of() via SHORT_IDS map.
// Only three have purely fixed offsets (no '/'):
// EST -> -05:00 (-18 000 s)
// MST -> -07:00 (-25 200 s)
// HST -> -10:00 (-36 000 s)
// These may appear with or without a leading space; no sub-offset is allowed after them.
for (abbr, offset_secs) in &[
(" EST", -18_000i32),
("EST", -18_000),
(" MST", -25_200),
("MST", -25_200),
(" HST", -36_000),
("HST", -36_000),
] {
if let Some(pos) = rfind_str(value, abbr) {
if pos + abbr.len() == value.len() {
return Some((&value[..pos], tz_from_offset_secs(*offset_secs)?));
}
}
}
// 4. Named IANA timezone: a space followed by a slash-containing word at the end.
// e.g. "2015-03-18T12:03:17.123456 Europe/Moscow"
if let Some(space_pos) = value.rfind(' ') {
let tz_name = &value[space_pos + 1..];
if tz_name.contains('/') {
if let Ok(tz) = Tz::from_str(tz_name) {
return Some((&value[..space_pos], tz));
}
}
}
// 5. Bare +/-offset: find the rightmost '+' or '-' and try to parse everything
// from that position to the end as a complete valid offset.
let last_sign = {
let p = value.rfind('+');
let m = value.rfind('-');
match (p, m) {
(Some(p), Some(m)) => Some(p.max(m)),
(a, b) => a.or(b),
}
};
if let Some(pos) = last_sign {
let offset_str = &value[pos..];
if let Some(secs) = parse_sign_offset(offset_str) {
return Some((&value[..pos], tz_from_offset_secs(secs)?));
}
}
None
}
/// The timestamp string shapes the parser recognises. The shapes are mutually exclusive, so at
/// most one can apply to any given string.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum TimestampPattern {
Year,
Month,
Day,
Hour,
Minute,
Second,
Microsecond,
TimeOnlyH,
TimeOnlyHm,
TimeOnlyHms,
TimeOnlyHmsu,
BareHm,
BareHms,
BareHmsu,
}
impl TimestampPattern {
/// True for the shapes that carry no date component: `T12`, `T12:34`, `12:34`, ...
fn is_time_only(self) -> bool {
!matches!(
self,
Self::Year
| Self::Month
| Self::Day
| Self::Hour
| Self::Minute
| Self::Second
| Self::Microsecond
)
}
/// True for the shapes that end in a seconds or fraction segment. Spark's
/// `parseTimestampString` only captures a zone id when its byte scanner hits a non-digit
/// while inside those two segments, so a suffix such as `Z`, `+05:30` or ` UTC` is legal
/// after `hh:mm:ss` or `hh:mm:ss.f*` but makes a date-only, hour-only or hour:minute value
/// malformed ("2020-10-01Z" and "2020-01-01T12:34Z" are both null).
fn ends_with_seconds_segment(self) -> bool {
matches!(
self,
Self::Second
| Self::Microsecond
| Self::TimeOnlyHms
| Self::TimeOnlyHmsu
| Self::BareHms
| Self::BareHmsu
)
}
/// Every shape, in declaration order.
#[cfg(test)]
const ALL: [TimestampPattern; 14] = [
Self::Year,
Self::Month,
Self::Day,
Self::Hour,
Self::Minute,
Self::Second,
Self::Microsecond,
Self::TimeOnlyH,
Self::TimeOnlyHm,
Self::TimeOnlyHms,
Self::TimeOnlyHmsu,
Self::BareHm,
Self::BareHms,
Self::BareHmsu,
];
/// The regular expression [`classify_timestamp_pattern`] is equivalent to for this shape,
/// which the classifier tests compare it against.
#[cfg(test)]
fn regex_str(self) -> &'static str {
match self {
Self::Year => r"^[+-]?[0-9]{4,6}$",
Self::Month => r"^[+-]?[0-9]{4,6}-[0-9]{1,2}$",
Self::Day => r"^[+-]?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}$",
Self::Hour => r"^[+-]?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}$",
Self::Minute => r"^[+-]?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}$",
Self::Second => {
r"^[+-]?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$"
}
Self::Microsecond => {
r"^[+-]?[0-9]{4,6}-[0-9]{1,2}-[0-9]{1,2}[T ][0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]*$"
}
Self::TimeOnlyH => r"^T[0-9]{1,2}$",
Self::TimeOnlyHm => r"^T[0-9]{1,2}:[0-9]{1,2}$",
Self::TimeOnlyHms => r"^T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$",
Self::TimeOnlyHmsu => r"^T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]*$",
Self::BareHm => r"^[0-9]{1,2}:[0-9]{1,2}$",
Self::BareHms => r"^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$",
Self::BareHmsu => r"^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]*$",
}
}
}
/// Returns the shape `value` has, or `None` when it matches none of them.
///
/// The shapes transcribe the per-segment digit rules of Spark's
/// `SparkDateTimeUtils.parseTimestampString` (`isValidDigits`): the year takes an optional
/// '+' or '-' sign followed by 4-6 digits (`maxDigitsYear = 6`, so "0002020-01-01" is
/// malformed for a timestamp even though `stringToDate`, ported by `date_parser`, allows 7),
/// month/day/hour/minute/second take 1-2 digits each, and the fraction takes any number of
/// digits including none ("12:34:56." is valid), of which only the first six are kept. All
/// digits must be ASCII, matching Spark's byte scanner and the numeric parsers used after
/// shape recognition, so a string with any non-ASCII byte has no shape.
///
/// The shapes are classified by a single left-to-right byte scan rather than by matching
/// [`TimestampPattern::regex_str`] one pattern at a time, which the classifier tests check it
/// is equivalent to.
fn classify_timestamp_pattern(value: &str) -> Option<TimestampPattern> {
let bytes = value.as_bytes();
// `T`-prefixed time-only shapes.
if let [b'T', rest @ ..] = bytes {
return classify_time(rest, true);
}
let (signed, rest) = match bytes {
[b'+' | b'-', rest @ ..] => (true, rest),
_ => (false, bytes),
};
let digits = digit_run(rest, MAX_YEAR_DIGITS + 1);
let after_digits = &rest[digits..];
match after_digits.first() {
// Year only.
None => (4..=MAX_YEAR_DIGITS)
.contains(&digits)
.then_some(TimestampPattern::Year),
// A year of 4-6 digits followed by the date separator.
Some(b'-') if (4..=MAX_YEAR_DIGITS).contains(&digits) => {
classify_date_tail(&after_digits[1..])
}
// Bare time-only shapes take a 1-2 digit hour and no sign.
Some(b':') if !signed && (1..=2).contains(&digits) => classify_time(rest, false),
_ => None,
}
}
/// The most year digits Spark's `isValidDigits` accepts (`maxDigitsYear`).
const MAX_YEAR_DIGITS: usize = 6;
/// The most digits Spark's `isValidDigits` accepts in a month, day, hour, minute or second.
const MAX_SEGMENT_DIGITS: usize = 2;
/// Number of leading ASCII digits in `bytes`, counting no more than `limit`.
///
/// The fixed-width segments pass one more than their widest valid length, which is enough to
/// reject an overlong run. Stopping there keeps a long malformed value, such as thousands of
/// digits followed by `Z`, from being scanned end to end, and then again after the zone is
/// stripped; the regexes this replaced also gave up after a bounded prefix.
fn digit_run(bytes: &[u8], limit: usize) -> usize {
bytes
.iter()
.take(limit)
.take_while(|b| b.is_ascii_digit())
.count()
}
/// `bytes` past a leading 1-2 digit segment, or `None` when it does not start with one.
///
/// Taking the whole digit run is equivalent to the regexes' `[0-9]{1,2}`: every segment is
/// followed by a separator or the end of the string, so a run of three or more digits can
/// never match, and counting to three is enough to tell.
fn skip_segment(bytes: &[u8]) -> Option<&[u8]> {
let digits = digit_run(bytes, MAX_SEGMENT_DIGITS + 1);
(1..=MAX_SEGMENT_DIGITS)
.contains(&digits)
.then(|| &bytes[digits..])
}
/// Classifies `[0-9]{1,2}(:[0-9]{1,2}(:[0-9]{1,2}(\.[0-9]*)?)?)?`, the time-only shapes. A bare
/// hour with no minutes is only a shape when the `T` prefix was present.
fn classify_time(bytes: &[u8], t_prefixed: bool) -> Option<TimestampPattern> {
use TimestampPattern::*;
let bytes = skip_segment(bytes)?;
if bytes.is_empty() {
return t_prefixed.then_some(TimeOnlyH);
}
let bytes = skip_segment(bytes.strip_prefix(b":")?)?;
if bytes.is_empty() {
return Some(if t_prefixed { TimeOnlyHm } else { BareHm });
}
let bytes = skip_segment(bytes.strip_prefix(b":")?)?;
if bytes.is_empty() {
return Some(if t_prefixed { TimeOnlyHms } else { BareHms });
}
// A fraction may be any length, so it is the one run that has to be scanned in full.
let fraction = bytes.strip_prefix(b".")?;
fraction
.iter()
.all(u8::is_ascii_digit)
.then_some(if t_prefixed { TimeOnlyHmsu } else { BareHmsu })
}
/// Classifies `[0-9]{1,2}(-[0-9]{1,2}([T ][0-9]{1,2}(:[0-9]{1,2}(:[0-9]{1,2}(\.[0-9]*)?)?)?)?)?`,
/// the part of a date-time shape that follows the year and its `-` separator.
fn classify_date_tail(bytes: &[u8]) -> Option<TimestampPattern> {
use TimestampPattern::*;
let bytes = skip_segment(bytes)?;
if bytes.is_empty() {
return Some(Month);
}
let bytes = skip_segment(bytes.strip_prefix(b"-")?)?;
if bytes.is_empty() {
return Some(Day);
}
let ([b'T', bytes @ ..] | [b' ', bytes @ ..]) = bytes else {
return None;
};
let bytes = skip_segment(bytes)?;
if bytes.is_empty() {
return Some(Hour);
}
let bytes = skip_segment(bytes.strip_prefix(b":")?)?;
if bytes.is_empty() {
return Some(Minute);
}
let bytes = skip_segment(bytes.strip_prefix(b":")?)?;
if bytes.is_empty() {
return Some(Second);
}
let fraction = bytes.strip_prefix(b".")?;
fraction
.iter()
.all(u8::is_ascii_digit)
.then_some(Microsecond)
}
/// Parses `value` according to the shape already determined for it by
/// [`classify_timestamp_pattern`].
///
/// Both T-separator and space-separator date-time forms are supported. Negative years are
/// handled by `get_timestamp_values` detecting a leading '-'.
fn parse_timestamp_pattern<T: TimeZone>(
value: &str,
pattern: Option<TimestampPattern>,
eval_mode: EvalMode,
tz: &T,
) -> SparkResult<Option<i64>> {
let timestamp = match pattern {
Some(TimestampPattern::Year) => get_timestamp_values(value, "year", tz)?,
Some(TimestampPattern::Month) => get_timestamp_values(value, "month", tz)?,
Some(TimestampPattern::Day) => get_timestamp_values(value, "day", tz)?,
Some(TimestampPattern::Hour) => get_timestamp_values(value, "hour", tz)?,
Some(TimestampPattern::Minute) => get_timestamp_values(value, "minute", tz)?,
Some(TimestampPattern::Second) => get_timestamp_values(value, "second", tz)?,
Some(TimestampPattern::Microsecond) => get_timestamp_values(value, "microsecond", tz)?,
// Listed out rather than caught by a `Some(_)` arm on purpose: a catch-all would route
// any future non-time-only shape into the time-only parser and produce a wrong value
// silently. Spelling out the variants makes adding one a compile error here.
Some(
TimestampPattern::TimeOnlyH
| TimestampPattern::TimeOnlyHm
| TimestampPattern::TimeOnlyHms
| TimestampPattern::TimeOnlyHmsu
| TimestampPattern::BareHm
| TimestampPattern::BareHms
| TimestampPattern::BareHmsu,
) => parse_str_to_time_only_timestamp(value, tz)?,
None => None,
};
if timestamp.is_none() {
return if eval_mode == EvalMode::Ansi {
Err(SparkError::InvalidInputInCastToDatetime {
value: value.to_string(),
from_type: "STRING".to_string(),
to_type: "TIMESTAMP".to_string(),
})
} else {
Ok(None)
};
}
Ok(timestamp)
}
fn timestamp_ntz_parser(
value: &str,
eval_mode: EvalMode,
allow_time_zone: bool,
_is_spark4_plus: bool,
) -> SparkResult<Option<i64>> {
// Trimmed as in `timestamp_parser`. NTZ rejects time-only strings outright, so it needs no
// Spark 4 check for padding before a `T`.
let value = trim_all(value);
// Classify the shape once and reuse it for the time-only rejection, the offset-suffix
// guard, and the parse dispatch.
let pattern = classify_timestamp_pattern(value);
// Reject a value that trims to nothing, and time-only patterns: NTZ requires a date
// component
if value.is_empty() || pattern.is_some_and(TimestampPattern::is_time_only) {
return invalid_ntz_timestamp(value, eval_mode);
}
// If no direct match, try stripping a timezone suffix. Spark only recognises a zone after
// the seconds segment; a suffix anywhere else leaves the unstripped value, which no base
// pattern matches, so the inner parser reports it as malformed.
let (value_to_parse, pattern) = if pattern.is_none() {
match extract_offset_suffix(value) {
Some((stripped, _tz)) => {
// Spark applies Java String.trim to the zone, not Unicode whitespace trimming.
let stripped = stripped.trim_end_matches(|c: char| c <= '\u{20}');
let stripped_pattern = classify_timestamp_pattern(stripped);
if !stripped_pattern.is_some_and(TimestampPattern::ends_with_seconds_segment) {
(value, None)
} else if allow_time_zone {
(stripped, stripped_pattern)
} else {
return invalid_ntz_timestamp(value, eval_mode);
}
}
None => (value, None),
}
} else {
(value, pattern)
};
timestamp_ntz_parser_inner(value_to_parse, pattern, eval_mode)
}
/// ANSI mode raises on an unparseable TIMESTAMP_NTZ value; the other modes return null.
fn invalid_ntz_timestamp(value: &str, eval_mode: EvalMode) -> SparkResult<Option<i64>> {
if eval_mode == EvalMode::Ansi {
Err(SparkError::InvalidInputInCastToDatetime {
value: value.to_string(),
from_type: "STRING".to_string(),
to_type: "TIMESTAMP_NTZ".to_string(),
})
} else {
Ok(None)
}
}
fn timestamp_ntz_parser_inner(
value: &str,
pattern: Option<TimestampPattern>,
eval_mode: EvalMode,
) -> SparkResult<Option<i64>> {
let timestamp_type = match pattern {
Some(TimestampPattern::Year) => "year",
Some(TimestampPattern::Month) => "month",
Some(TimestampPattern::Day) => "day",
Some(TimestampPattern::Hour) => "hour",
Some(TimestampPattern::Minute) => "minute",
Some(TimestampPattern::Second) => "second",
Some(TimestampPattern::Microsecond) => "microsecond",
// Time-only shapes carry no date, so NTZ cannot represent them.
Some(_) | None => return invalid_ntz_timestamp(value, eval_mode),
};
if let Some(info) = parse_to_timestamp_info(value, timestamp_type)? {
if let Some(timestamp) = local_datetime_to_micros(&info)? {
return Ok(Some(timestamp));
}
}
invalid_ntz_timestamp(value, eval_mode)
}
fn parse_str_to_time_only_timestamp<T: TimeZone>(value: &str, tz: &T) -> SparkResult<Option<i64>> {
// The 'T' is optional in the time format; strip it if specified.
let time_part = value.strip_prefix('T').unwrap_or(value);
// Parse time components: hour[:minute[:second[.fraction]]]
// Use splitn(3) so "12:34:56.789" splits into ["12", "34", "56.789"].
let colon_parts: Vec<&str> = time_part.splitn(3, ':').collect();
let hour: u32 = colon_parts[0].parse().unwrap_or(0);
let minute: u32 = colon_parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0);
let (second, nanosecond) = if let Some(sec_frac) = colon_parts.get(2) {
let dot_idx = sec_frac.find('.');
let sec: u32 = sec_frac[..dot_idx.unwrap_or(sec_frac.len())]
.parse()
.unwrap_or(0);
let ns: u32 = if let Some(dot) = dot_idx {
let frac = &sec_frac[dot + 1..];
// Interpret up to 6 digits as microseconds, padding with trailing zeros.
let Some(trimmed) = frac.get(..frac.len().min(6)) else {
return Ok(None);
};
let padded = format!("{:0<6}", trimmed);
padded.parse::<u32>().unwrap_or(0) * 1000
} else {
0
};
(sec, ns)
} else {
(0, 0)
};
let datetime = tz.from_utc_datetime(&chrono::Utc::now().naive_utc());
let result = datetime
.with_timezone(tz)
.with_hour(hour)
.and_then(|dt| dt.with_minute(minute))
.and_then(|dt| dt.with_second(second))
.and_then(|dt| dt.with_nanosecond(nanosecond))
.map(|dt| dt.timestamp_micros());
Ok(result)
}
//a string to date parser - port of spark's SparkDateTimeUtils#stringToDate.
fn date_parser(date_str: &str, eval_mode: EvalMode) -> SparkResult<Option<i32>> {
// local functions
/// Decodes a run of ASCII digits, or `None` if any byte is not a digit.
fn decode_digits(bytes: &[u8]) -> Option<i64> {
bytes.iter().try_fold(0i64, |acc, b| {
b.is_ascii_digit().then(|| acc * 10 + (b - b'0') as i64)
})
}
fn is_valid_digits(segment: i32, digits: usize) -> bool {
// Years are bounded by `resolve_epoch_day` below. We allow up to 7 digits to support
// leading-zero year strings like "0002020" (= year 2020), matching Spark's isValidDigits.
let max_digits_year = 7;
// year (segment 0) can be between 4 to 7 digits,
// month and day (segment 1 and 2) can be between 1 to 2 digits
(segment == 0 && digits >= 4 && digits <= max_digits_year)
|| (segment != 0 && digits > 0 && digits <= 2)
}
fn return_result(date_str: &str, eval_mode: EvalMode) -> SparkResult<Option<i32>> {
if eval_mode == EvalMode::Ansi {
Err(SparkError::InvalidInputInCastToDatetime {
value: date_str.to_string(),
from_type: "STRING".to_string(),
to_type: "DATE".to_string(),
})
} else {
Ok(None)
}
}
/// Turns parsed year/month/day segments into an epoch day. Shared by both parsing paths so
/// that the decision of what a segment triple *means* lives in exactly one place.
fn resolve_epoch_day(
year: i64,
month: i64,
day: i64,
date_str: &str,
eval_mode: EvalMode,
) -> SparkResult<Option<i32>> {
// Spark builds a `LocalDate` and narrows its epoch day to an `Int`, so an invalid
// calendar date or an epoch day that overflows `i32` both yield `None` there, which
// `stringToDateAnsi` turns into CAST_INVALID_INPUT.
let Some(days) = ymd_to_epoch_day(year, month, day).and_then(|d| i32::try_from(d).ok())
else {
return return_result(date_str, eval_mode);
};
// Spark accepts years beyond what chrono can represent, and downstream date kernels
// cannot handle those values, so Comet keeps returning null for them in every eval mode
// rather than raising. This is a Comet limitation, not a malformed input.
//
// The bound is chrono's representable year range: `NaiveDate::MIN` is `-262143-01-01`
// and `NaiveDate::MAX` is `262142-12-31`
// (https://docs.rs/chrono/latest/chrono/naive/struct.NaiveDate.html#associatedconstant.MIN).
if !(-262143..=262142).contains(&year) {
return Ok(None);
}
Ok(Some(days))
}
// end local functions
if date_str.is_empty() {
return return_result(date_str, eval_mode);
}
let bytes = date_str.as_bytes();
// `SparkDateTimeUtils.getTrimmedStart`/`getTrimmedEnd` trim the same byte set as
// `UTF8String.trimAll`.
let (start, str_end_trimmed) = trim_all_range(bytes);
let mut j = start;
if j == str_end_trimmed {
return return_result(date_str, eval_mode);
}
// Fast path for the canonical `yyyy-mm-dd` form, which only skips the byte-scanning loop
// below. Any other shape (including a leading sign, which makes the first byte a
// non-digit) falls through to the general parser.
let trimmed = &bytes[j..str_end_trimmed];
if trimmed.len() == 10 && trimmed[4] == b'-' && trimmed[7] == b'-' {
if let (Some(year), Some(month), Some(day)) = (
decode_digits(&trimmed[..4]),
decode_digits(&trimmed[5..7]),
decode_digits(&trimmed[8..10]),
) {
return resolve_epoch_day(year, month, day, date_str, eval_mode);
}
}
//values of date segments year, month and day defaulting to 1
let mut date_segments = [1, 1, 1];
let mut sign = 1;
let mut current_segment = 0;
let mut current_segment_value = Wrapping(0);
let mut current_segment_digits = 0;
// assign a sign to the date; both '-' and '+' are accepted (Spark stringToDate line 357-360)
if bytes[j] == b'-' {
sign = -1;
j += 1;
} else if bytes[j] == b'+' {
// sign remains 1 (positive)
j += 1;
}
//loop to the end of string until we have processed 3 segments,
//exit loop on encountering any space ' ' or 'T' after the 3rd segment
while j < str_end_trimmed && (current_segment < 3 && !(bytes[j] == b' ' || bytes[j] == b'T')) {
let b = bytes[j];
if current_segment < 2 && b == b'-' {
//check for validity of year and month segments if current byte is separator
if !is_valid_digits(current_segment, current_segment_digits) {
return return_result(date_str, eval_mode);
}
//if valid update corresponding segment with the current segment value.
date_segments[current_segment as usize] = current_segment_value.0;
current_segment_value = Wrapping(0);
current_segment_digits = 0;
current_segment += 1;
} else if !b.is_ascii_digit() {
return return_result(date_str, eval_mode);
} else {
//increment value of current segment by the next digit
let parsed_value = Wrapping((b - b'0') as i32);
current_segment_value = current_segment_value * Wrapping(10) + parsed_value;
current_segment_digits += 1;
}
j += 1;
}
//check for validity of last segment
if !is_valid_digits(current_segment, current_segment_digits) {
return return_result(date_str, eval_mode);
}
if current_segment < 2 && j < str_end_trimmed {
// For the `yyyy` and `yyyy-[m]m` formats, entire input must be consumed.
return return_result(date_str, eval_mode);
}
date_segments[current_segment as usize] = current_segment_value.0;
resolve_epoch_day(
(sign * date_segments[0]) as i64,
date_segments[1] as i64,
date_segments[2] as i64,
date_str,
eval_mode,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cast::cast_array;
use crate::SparkCastOptions;
use arrow::array::{DictionaryArray, Int32Array, StringArray};
use arrow::datatypes::TimeUnit;
use datafusion::common::Result as DataFusionResult;
use regex::Regex;
/// Test helper that wraps the mode-specific parse functions
fn cast_string_to_i8(str: &str, eval_mode: EvalMode) -> SparkResult<Option<i8>> {
match eval_mode {
EvalMode::Legacy => parse_string_to_i8_legacy(str),
EvalMode::Ansi => parse_string_to_i8_ansi(str),
EvalMode::Try => parse_string_to_i8_try(str),
}
}
/// Alphabet of the characters that appear in the timestamp shapes, used to generate
/// classifier test inputs.
const SHAPE_ALPHABET: [u8; 9] = *b"019-:.T +";
/// The classifier replaced a sequential scan of [`TimestampPattern::regex_str`]. The two
/// must agree on every input, so this compares them over an exhaustive enumeration of
/// short strings over [`SHAPE_ALPHABET`] plus every single-character mutation of a set
/// of realistic timestamp strings.
#[test]
fn test_timestamp_pattern_classifier_matches_regexes() {
let regexes: Vec<Regex> = TimestampPattern::ALL
.iter()
.map(|p| Regex::new(p.regex_str()).unwrap())
.collect();
let check = |value: &str| {
let expected = regexes
.iter()
.position(|r| r.is_match(value))
.map(|i| TimestampPattern::ALL[i]);
assert_eq!(
classify_timestamp_pattern(value),
expected,
"classifying {value:?}"
);
};
// Exhaustive over every string of up to 5 shape characters.
let base = SHAPE_ALPHABET.len();
let mut buf = Vec::with_capacity(5);
for len in 0..=5u32 {
for n in 0..base.pow(len) {
buf.clear();
let mut n = n;
for _ in 0..len {
buf.push(SHAPE_ALPHABET[n % base]);
n /= base;
}
check(std::str::from_utf8(&buf).unwrap());
}
}
// Longer realistic shapes, and every one-character deletion, substitution, and
// insertion applied to them.
for seed in [
"2020-01-01",
"2020-01-01T12:34:56",
"2020-01-01 12:34:56.123456",
"-0002020-1-1T1",
"0002020-01-01 12:34",
"T12:34:56.7",
"12:34:56",
"202001",
"2020-01-01T12:34:56.123456+07:30",
"+2020-1-2T3:4:5.",
"-002020-1-2 3:4",
"T1:2:3.",
"1:2:3.4",
"0000000002020-01-01T12:34:56Z",
"2020-01-01T12:34:56.123456789012",
] {
check(seed);
for i in 0..seed.len() {
let mut deleted = seed.to_string();
deleted.remove(i);
check(&deleted);
for c in SHAPE_ALPHABET.map(char::from) {
let mut substituted = seed.to_string();
substituted.replace_range(i..i + 1, &c.to_string());
check(&substituted);
let mut inserted = seed.to_string();
inserted.insert(i, c);
check(&inserted);
}
}
}
// Spark's byte scanner only accepts ASCII digits, so no shape admits a non-ASCII one.
check("٢٠٢٠");
check("٢٠٢٠-٠١-٠١");
check("2020-01-01T12:34:56é");
}
/// Anchors the per-segment digit rules the classifier transcribes from Spark's
/// `isValidDigits`, which are easy to lose when hand-writing it.
#[test]
fn test_timestamp_pattern_segment_rules() {
use TimestampPattern::*;
// The year takes 4-6 digits, with or without a separator after it.
assert_eq!(classify_timestamp_pattern("012020"), Some(Year));
assert_eq!(classify_timestamp_pattern("0119704"), None);
assert_eq!(classify_timestamp_pattern("002020-01"), Some(Month));
assert_eq!(classify_timestamp_pattern("0002020-01"), None);
assert_eq!(classify_timestamp_pattern("202-01"), None);
// Either sign, but only one, and only on a year.
assert_eq!(classify_timestamp_pattern("+2020"), Some(Year));
assert_eq!(classify_timestamp_pattern("-2020-1-1"), Some(Day));
assert_eq!(classify_timestamp_pattern("+-2020"), None);
assert_eq!(classify_timestamp_pattern("-1:2"), None);
assert_eq!(classify_timestamp_pattern("+T1"), None);
// Every other segment takes 1-2 digits, wherever it sits.
assert_eq!(classify_timestamp_pattern("2020-1-1T1"), Some(Hour));
assert_eq!(classify_timestamp_pattern("2020-1-1 1:2"), Some(Minute));
assert_eq!(classify_timestamp_pattern("2020-1-1T1:2:3"), Some(Second));
assert_eq!(classify_timestamp_pattern("2020-001-01"), None);
assert_eq!(classify_timestamp_pattern("2020-01-01T12:345"), None);
// Bare time-only shapes take no lone hour; the `T` prefix allows one.
assert_eq!(classify_timestamp_pattern("1:2"), Some(BareHm));
assert_eq!(classify_timestamp_pattern("12"), None);
assert_eq!(classify_timestamp_pattern("T12"), Some(TimeOnlyH));
// A fraction may be empty.
assert_eq!(classify_timestamp_pattern("T1:2:3."), Some(TimeOnlyHmsu));
assert_eq!(classify_timestamp_pattern("1:2:3."), Some(BareHmsu));
assert_eq!(
classify_timestamp_pattern("2020-01-01 12:34:56."),
Some(Microsecond)
);
// An overlong run is rejected however long it is, while a fraction may be any length.
let long = "1".repeat(8192);
assert_eq!(classify_timestamp_pattern(&long), None);
assert_eq!(classify_timestamp_pattern(&format!("{long}Z")), None);
assert_eq!(classify_timestamp_pattern(&format!("-{long}-01")), None);
assert_eq!(classify_timestamp_pattern(&format!("2020-{long}")), None);
assert_eq!(classify_timestamp_pattern(&format!("T1:{long}")), None);
assert_eq!(
classify_timestamp_pattern(&format!("2020-01-01 12:34:56.{long}")),
Some(Microsecond)
);
assert_eq!(
classify_timestamp_pattern(&format!("1:2:3.{long}")),
Some(BareHmsu)
);
}
#[test]
fn test_digits_to_i128_boundary() {
// 38 nines is under i128::MAX (~1.7e38 vs 9.99e37); the head-only path parses it.
let d38 = "9".repeat(38);
assert_eq!(
digits_to_i128(d38.as_bytes()),
Some(99_999_999_999_999_999_999_999_999_999_999_999_999_i128)
);
// 39 nines overflows i128 in the tail-checked step.
let d39 = "9".repeat(39);
assert_eq!(digits_to_i128(d39.as_bytes()), None);
// 40 characters that reduce to a small value once the leading zero(s) are seen: the
// head-only accumulator must not lose bits on 38 zeros followed by two digits.
let z38_plus = format!("{}42", "0".repeat(38));
assert_eq!(digits_to_i128(z38_plus.as_bytes()), Some(42));
}
#[test]
fn test_parse_string_to_decimal_boundary() {
// 38-digit integral parses (fits i128).
let s38 = "9".repeat(38);
assert!(parse_string_to_decimal(&s38, 38, 0).unwrap().is_some());
// 39-digit integral overflows i128, so returns the invalid_decimal_cast error.
let s39 = "9".repeat(39);
assert!(parse_string_to_decimal(&s39, 38, 0).is_err());
// Very long fractional part now returns Err via the invalid_decimal_cast path instead
// of panicking on 10_i128.pow(fractional_scale).
let over_long = format!("0.{}", "0".repeat(40));
assert!(parse_string_to_decimal(&over_long, 38, 10).is_err());
}
#[test]
#[cfg_attr(miri, ignore)] // test takes too long with miri
fn test_cast_string_to_timestamp() {
let array: ArrayRef = Arc::new(StringArray::from(vec![
Some("2020-01-01T12:34:56.123456"),
Some("T2"),
Some("0100-01-01T12:34:56.123456"),
Some("10000-01-01T12:34:56.123456"),
// 7-digit year-only strings must return null (Spark returns null for these)
Some("0119704"),
Some("2024001"),
]));
let tz = &Tz::from_str("UTC").unwrap();
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<i32>>()
.expect("Expected a string array");
let eval_mode = EvalMode::Legacy;
let result = cast_utf8_to_timestamp!(
&string_array,
eval_mode,
PrimitiveArray::<TimestampMicrosecondType>::builder(string_array.len())
.with_timezone("UTC"),
timestamp_parser,
tz,
true
)
.unwrap();
assert_eq!(
result.data_type(),
&DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()))
);
assert_eq!(result.len(), 6);
// 7-digit year-only strings must be null
assert!(result.is_null(4), "0119704 should be null");
assert!(result.is_null(5), "2024001 should be null");
}
#[test]
fn test_cast_string_to_timestamp_ansi_error() {
// In ANSI mode, an invalid timestamp string must produce an error rather than null.
let array: ArrayRef = Arc::new(StringArray::from(vec![
Some("2020-01-01T12:34:56.123456"),
Some("not_a_timestamp"),
]));
let tz = &Tz::from_str("UTC").unwrap();
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<i32>>()
.expect("Expected a string array");
let eval_mode = EvalMode::Ansi;
let result = cast_utf8_to_timestamp!(
&string_array,
eval_mode,
PrimitiveArray::<TimestampMicrosecondType>::builder(string_array.len())
.with_timezone("UTC"),
timestamp_parser,
tz,
true
);
assert!(
result.is_err(),
"ANSI mode should return Err for an invalid timestamp string"
);
}
#[test]
fn test_cast_string_to_timestamp_ansi_error_trimmed_value() {
// The error value in InvalidInputInCastToDatetime must match the raw input
// (including trailing whitespace) to match Spark's CAST_INVALID_INPUT behavior.
let array: ArrayRef = Arc::new(StringArray::from(vec![
Some("91\n3 "), // trailing spaces after a newline in the middle
]));
let tz = &Tz::from_str("UTC").unwrap();
let string_array = array
.as_any()
.downcast_ref::<GenericStringArray<i32>>()
.expect("Expected a string array");
let eval_mode = EvalMode::Ansi;
let result = cast_utf8_to_timestamp!(
&string_array,
eval_mode,
PrimitiveArray::<TimestampMicrosecondType>::builder(string_array.len())
.with_timezone("UTC"),
timestamp_parser,
tz,
true
);
match result {
Err(SparkError::InvalidInputInCastToDatetime { value, .. }) => {
assert_eq!(
value, "91\n3 ",
"ANSI error value should match the raw (untrimmed) input to match Spark behavior"
);
}
other => panic!("Expected InvalidInputInCastToDatetime error, got {other:?}"),
}
}
/// The codepoint matrix from
/// <https://github.com/apache/datafusion-comet/issues/5149>: the ASCII control bytes and
/// DELETE, plus the non-ASCII codepoints that are whitespace to Unicode but that no Spark
/// cast trims. `CometNativeCastSuite` runs the same matrix with Spark itself as the oracle.
fn trim_pads() -> Vec<String> {
let mut pads: Vec<String> = (0x00u8..=0x20).map(|b| String::from(b as char)).collect();
pads.push("\u{7f}".to_string());
pads.extend(
[
"\u{85}", "\u{a0}", "\u{1680}", "\u{2000}", "\u{2005}", "\u{200a}", "\u{2028}",
"\u{2029}", "\u{202f}", "\u{205f}", "\u{3000}",
]
.iter()
.map(|s| s.to_string()),
);
pads
}
/// Asserts that the cast to `to_type` trims each [`trim_pads`] entry exactly when `regime`
/// -- the trim helper that Spark's cast to `to_type` uses -- trims it: a value in every eval
/// mode when it is trimmed, NULL (or an ANSI error) when it is not. Interior padding, padding
/// on its own and the empty string must never parse.
fn assert_trim_parity(to_type: &DataType, valid: &str, regime: fn(&str) -> &str) {
let split = valid.char_indices().nth(1).map(|(i, _)| i).unwrap();
// The empty string reaches the same empty-slice branch that fully-trimmed padding does.
let mut cases = vec![("empty".to_string(), String::new(), false)];
for pad in trim_pads() {
// The regime trims this padding iff trimming the padding alone leaves nothing.
let trimmed = regime(&pad).is_empty();
cases.extend([
(format!("leading {pad:?}"), format!("{pad}{valid}"), trimmed),
(
format!("trailing {pad:?}"),
format!("{valid}{pad}"),
trimmed,
),
(
format!("both ends {pad:?}"),
format!("{pad}{valid}{pad}"),
trimmed,
),
(
format!("interior {pad:?}"),
format!("{}{pad}{}", &valid[..split], &valid[split..]),
false,
),
(format!("only {pad:?}"), pad.clone(), false),
]);
}
for (position, input, expect_value) in cases {
for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
let array: ArrayRef = Arc::new(StringArray::from(vec![Some(input.as_str())]));
let options = SparkCastOptions::new(eval_mode, "UTC", false);
let result = cast_array(array, to_type, &options);
let context = format!("cast {input:?} ({position}) to {to_type} in {eval_mode:?}");
if expect_value {
let array = result.unwrap_or_else(|e| panic!("{context}: {e}"));
assert!(!array.is_null(0), "{context}: expected a value, got NULL");
} else if eval_mode == EvalMode::Ansi {
assert!(result.is_err(), "{context}: expected an error");
} else {
let array = result.unwrap_or_else(|e| panic!("{context}: {e}"));
assert!(array.is_null(0), "{context}: expected NULL");
}
}
}
}
#[test]
fn test_cast_string_to_boolean_trim_parity() {
assert_trim_parity(&DataType::Boolean, "true", trim_all);
}
#[test]
fn test_cast_string_to_int_trim_parity() {
for to_type in [
DataType::Int8,
DataType::Int16,
DataType::Int32,
DataType::Int64,
] {
assert_trim_parity(&to_type, "12", trim_all);
}
}
#[test]
fn test_cast_string_to_date_trim_parity() {
assert_trim_parity(&DataType::Date32, "2020-01-01", trim_all);
}
/// Float, double and decimal all use the narrower `String.trim` set, which keeps `0x7F`.
#[test]
fn test_cast_string_to_float_and_decimal_trim_parity() {
for to_type in [
DataType::Float32,
DataType::Float64,
DataType::Decimal128(10, 2),
] {
assert_trim_parity(&to_type, "1.5", trim_java_string);
}
}
/// Timestamps trim the `trimAll` set too, through `SparkDateTimeUtils.getTrimmedStart` /
/// `getTrimmedEnd`.
#[test]
fn test_cast_string_to_timestamp_trim_parity() {
for to_type in [
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
DataType::Timestamp(TimeUnit::Microsecond, None),
] {
assert_trim_parity(&to_type, "2020-01-01 12:34:56", trim_all);
}
}
#[test]
fn test_cast_string_to_timestamp_ntz() {
// Helper to reduce boilerplate
fn parse(s: &str, allow_tz: bool) -> Option<i64> {
timestamp_ntz_parser(s, EvalMode::Legacy, allow_tz, false).unwrap()
}
// Basic: "2020-01-01 12:34:56" -> local micros
// days_from_civil(2020,1,1) = 18262; 18262*86400 = 1577836800
// + 12*3600 + 34*60 + 56 = 45296; total = 1577882096s
assert_eq!(
parse("2020-01-01 12:34:56", true),
Some(1_577_882_096_000_000)
);
assert_eq!(
parse("2020-01-01T12:34:56", true),
Some(1_577_882_096_000_000)
);
// With microseconds
assert_eq!(
parse("2020-01-01 12:34:56.123456", true),
Some(1_577_882_096_123_456)
);
// Date only
assert_eq!(parse("2020-01-01", true), Some(1_577_836_800_000_000));
// Timezone discarded (allow_time_zone=true): same result as without TZ
assert_eq!(
parse("2020-01-01T12:34:56Z", true),
Some(1_577_882_096_000_000)
);
assert_eq!(
parse("2020-01-01T12:34:56+05:30", true),
Some(1_577_882_096_000_000)
);
assert_eq!(
parse("2020-01-01T12:34:56-08:00", true),
Some(1_577_882_096_000_000)
);
// Space-separated offset (e.g. "2021-11-22 10:54:27 +08:00")
assert_eq!(
parse("2021-11-22 10:54:27 +08:00", true),
parse("2021-11-22 10:54:27", true)
);
// Timezone rejected (allow_time_zone=false)
assert_eq!(parse("2020-01-01T12:34:56Z", false), None);
assert_eq!(parse("2020-01-01T12:34:56+05:30", false), None);
// Time-only rejected
assert_eq!(parse("T12:34:56", true), None);
assert_eq!(parse("12:34", true), None);
assert_eq!(parse("T2", true), None);
// Invalid -> None in Legacy
assert_eq!(parse("invalid", true), None);
assert_eq!(parse("", true), None);
// Invalid -> Error in ANSI
assert!(timestamp_ntz_parser("invalid", EvalMode::Ansi, true, false).is_err());
assert!(timestamp_ntz_parser("T12:34", EvalMode::Ansi, true, false).is_err());
// Invalid -> None in Try
assert_eq!(
timestamp_ntz_parser("invalid", EvalMode::Try, true, false).unwrap(),
None
);
// DST gap time works for NTZ (pure arithmetic, no DST)
// days_from_civil(2024,3,10) * 86400 + 2*3600 + 30*60 = 1710037800s
assert_eq!(
parse("2024-03-10 02:30:00", true),
Some(1_710_037_800_000_000)
);
// Invalid leap day -> None
assert_eq!(parse("2023-02-29 00:00:00", true), None);
// Valid leap day
assert!(parse("2020-02-29 00:00:00", true).is_some());
}
#[test]
fn test_cast_string_to_timestamp_ntz_array() {
let array: ArrayRef = Arc::new(StringArray::from(vec![
Some("2020-01-01T12:34:56.123456"),
Some("T2"),
Some("2020-01-01"),
None,
Some("invalid"),
Some("2020-06-15T12:30:00Z"),
]));
let result = cast_string_to_timestamp_ntz(&array, EvalMode::Legacy, true, false).unwrap();
let ts_array = result
.as_any()
.downcast_ref::<PrimitiveArray<TimestampMicrosecondType>>()
.unwrap();
assert_eq!(ts_array.len(), 6);
assert!(!ts_array.is_null(0)); // valid
assert!(ts_array.is_null(1)); // time-only -> null
assert!(!ts_array.is_null(2)); // date-only -> valid
assert!(ts_array.is_null(3)); // null input
assert!(ts_array.is_null(4)); // invalid -> null
assert!(!ts_array.is_null(5)); // TZ discarded -> valid
// TZ discarded: "2020-06-15T12:30:00Z" should give same micros as "2020-06-15T12:30:00"
assert_eq!(
ts_array.value(5),
timestamp_ntz_parser("2020-06-15T12:30:00", EvalMode::Legacy, true, false)
.unwrap()
.unwrap()
);
}
#[test]
fn test_cast_string_to_timestamp_ntz_ansi_error() {
let array: ArrayRef = Arc::new(StringArray::from(vec![Some("invalid")]));
let result = cast_string_to_timestamp_ntz(&array, EvalMode::Ansi, true, false);
assert!(result.is_err());
match result.unwrap_err() {
SparkError::InvalidInputInCastToDatetime { to_type, .. } => {
assert_eq!(to_type, "TIMESTAMP_NTZ");
}
other => panic!("Expected InvalidInputInCastToDatetime, got {other:?}"),
}
}
#[test]
fn test_cast_string_to_timestamp_ntz_ansi_invalid_date() {
// 2023-02-29 is parseable but invalid (not a leap year).
// In ANSI mode this must error, not return NULL.
let result = timestamp_ntz_parser("2023-02-29", EvalMode::Ansi, false, false);
assert!(
result.is_err(),
"ANSI mode should error on invalid date 2023-02-29"
);
match result.unwrap_err() {
SparkError::InvalidInputInCastToDatetime { to_type, .. } => {
assert_eq!(to_type, "TIMESTAMP_NTZ");
}
other => panic!("Expected InvalidInputInCastToDatetime, got {other:?}"),
}
// In Legacy mode, same input should return None (null).
let result = timestamp_ntz_parser("2023-02-29", EvalMode::Legacy, false, false);
assert_eq!(result.unwrap(), None);
}
#[test]
fn test_cast_string_to_timestamp_ntz_out_of_range_year() {
for value in ["294249-01-01", "294249-01-01 00:00:00", "-290310-01-01"] {
let array: ArrayRef = Arc::new(StringArray::from(vec![value]));
match cast_string_to_timestamp_ntz(&array, EvalMode::Ansi, true, false) {
Err(SparkError::InvalidInputInCastToDatetime {
value: actual,
from_type,
to_type,
}) => {
assert_eq!(actual, value);
assert_eq!(from_type, "STRING");
assert_eq!(to_type, "TIMESTAMP_NTZ");
}
other => panic!("Expected ANSI cast error for {value}, got {other:?}"),
}
for mode in [EvalMode::Legacy, EvalMode::Try] {
let result = cast_string_to_timestamp_ntz(&array, mode, true, false).unwrap();
assert!(result.is_null(0), "Expected NULL for {value} in {mode:?}");
}
}
}
#[test]
fn test_cast_dict_string_to_timestamp() -> DataFusionResult<()> {
// prepare input data
let keys = Int32Array::from(vec![0, 1]);
let values: ArrayRef = Arc::new(StringArray::from(vec![
Some("2020-01-01T12:34:56.123456"),
Some("T2"),
]));
let dict_array = Arc::new(DictionaryArray::new(keys, values));
let timezone = "UTC".to_string();
// test casting string dictionary array to timestamp array
let cast_options = SparkCastOptions::new(EvalMode::Legacy, &timezone, false);
let result = cast_array(
dict_array,
&DataType::Timestamp(TimeUnit::Microsecond, Some(timezone.clone().into())),
&cast_options,
)?;
assert_eq!(
*result.data_type(),
DataType::Timestamp(TimeUnit::Microsecond, Some(timezone.into()))
);
assert_eq!(result.len(), 2);
Ok(())
}
#[test]
fn extreme_year_boundary_test() {
let tz = &Tz::from_str("UTC").unwrap();
// Long.MaxValue = 9223372036854775807 μs -> 294247-01-10T04:00:54.775807Z
assert_eq!(
timestamp_parser("294247-01-10T04:00:54.775807Z", EvalMode::Legacy, tz, true).unwrap(),
Some(i64::MAX),
);
// Long.MinValue = -9223372036854775808 μs -> -290308-12-21T19:59:05.224192Z
assert_eq!(
timestamp_parser("-290308-12-21T19:59:05.224192Z", EvalMode::Legacy, tz, true).unwrap(),
Some(i64::MIN),
);
// One beyond Long.MaxValue -> null (overflow)
assert_eq!(
timestamp_parser("294247-01-10T04:00:54.775808Z", EvalMode::Legacy, tz, true).unwrap(),
None,
);
// One before Long.MinValue -> null (overflow)
assert_eq!(
timestamp_parser("-290308-12-21T19:59:05.224191Z", EvalMode::Legacy, tz, true).unwrap(),
None,
);
}
#[test]
fn test_leading_whitespace_t_hm() {
let tz = &Tz::from_str("UTC").unwrap();
// Spark 4.0+ rejects leading whitespace for ALL T-prefixed time-only patterns, and the
// ISO control characters Spark trims along with it count as whitespace here too.
for ws_input in &[
" T2:30",
"\tT2:30",
"\nT2:30",
" T2",
"\tT2",
"\nT2",
"\tT1:2:3 +08:00",
" T1:2:3.4 +08:00",
"\u{1}T2",
"\u{7f}T2:30",
] {
for mode in [EvalMode::Legacy, EvalMode::Try] {
assert!(
timestamp_parser(ws_input, mode, tz, true)
.unwrap()
.is_none(),
"'{ws_input}' should be null in {mode:?} mode on Spark 4.0+"
);
}
// In ANSI mode the same inputs must raise an error (not silently return null).
assert!(
timestamp_parser(ws_input, EvalMode::Ansi, tz, true).is_err(),
"'{ws_input}' should error in ANSI mode on Spark 4.0+"
);
// Spark 3.x trims all whitespace first, so leading whitespace is valid.
assert!(
timestamp_parser(ws_input, EvalMode::Legacy, tz, false)
.unwrap()
.is_some(),
"'{ws_input}' should be valid in Legacy mode on Spark 3.x"
);
}
// Without leading whitespace, these must be valid on all versions.
for ok_input in &[
"T2:30",
"T2",
"T1:2:3 +08:00",
"T1:2:3.4 +08:00",
"T2\u{1}",
"T2:30\u{7f}",
] {
assert!(
timestamp_parser(ok_input, EvalMode::Legacy, tz, true)
.unwrap()
.is_some(),
"'{ok_input}' should be valid"
);
}
// Non-ASCII whitespace is never trimmed, so it is malformed on every version.
for spark4 in [false, true] {
assert!(
timestamp_parser("\u{3000}T2", EvalMode::Legacy, tz, spark4)
.unwrap()
.is_none(),
"'\\u{{3000}}T2' should be null (spark4={spark4})"
);
}
}
#[test]
fn plus_sign_year_test() {
let tz = &Tz::from_str("UTC").unwrap();
for input in [
"7528",
"00463",
"79821",
"2976",
"0000",
"002020",
"2020-1",
"2020-1-2",
"2020-1-2T3",
"2020-1-2 3:4",
"2020-1-2T3:4:5",
"2020-1-2 3:4:5.",
"2020-1-2T3:4:5.123456789",
"2020-1-2T3:4:5Z",
"2020-1-2T3:4:5+05:30",
"2020-1-2T3:4:5-08:00",
"2020-1-2T3:4:5 UTC",
"294247-01-10T04:00:54.775807",
] {
let signed = format!(" +{input} ");
for mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
for (actual, expected) in [
(
timestamp_parser(&signed, mode, tz, true),
timestamp_parser(input, mode, tz, true),
),
(
timestamp_ntz_parser(&signed, mode, true, true),
timestamp_ntz_parser(input, mode, true, true),
),
] {
let expected = expected.unwrap();
assert!(expected.is_some(), "{input:?}");
assert_eq!(actual.unwrap(), expected, "{signed:?} in {mode:?}");
}
}
}
for input in [
"+",
"++2020",
"+-2020",
"-+2020",
"--2020",
"+020",
"+0002020",
"+ 2020",
"+２０２０",
"+2020-+1",
"+2020--1",
"+2020-1-+2",
"+12:12:12",
"+T12:12:12",
"+2020Z",
"+2020-1-2Z",
"+2020-1-2T3:4Z",
"+294247-01-10T04:00:54.775808",
] {
for mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
for result in [
timestamp_parser(input, mode, tz, true),
timestamp_ntz_parser(input, mode, true, true),
] {
if mode == EvalMode::Ansi {
assert!(result.is_err(), "{input:?} in {mode:?}");
} else {
assert_eq!(result.unwrap(), None, "{input:?} in {mode:?}");
}
}
}
}
}
#[test]
#[cfg_attr(miri, ignore)] // test takes too long with miri
fn timestamp_parser_test() {
let tz = &Tz::from_str("UTC").unwrap();
// write for all formats
assert_eq!(
timestamp_parser("2020", EvalMode::Legacy, tz, true).unwrap(),
Some(1577836800000000) // this is in milliseconds
);
assert_eq!(
timestamp_parser("2020-01", EvalMode::Legacy, tz, true).unwrap(),
Some(1577836800000000)
);
assert_eq!(
timestamp_parser("2020-01-01", EvalMode::Legacy, tz, true).unwrap(),
Some(1577836800000000)
);
assert_eq!(
timestamp_parser("2020-01-01T12", EvalMode::Legacy, tz, true).unwrap(),
Some(1577880000000000)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882040000000)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882096000000)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.123456", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882096123456)
);
assert_eq!(
timestamp_parser("0100", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011459200000000)
);
assert_eq!(
timestamp_parser("0100-01", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011459200000000)
);
assert_eq!(
timestamp_parser("0100-01-01", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011459200000000)
);
assert_eq!(
timestamp_parser("0100-01-01T12", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011416000000000)
);
assert_eq!(
timestamp_parser("0100-01-01T12:34", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011413960000000)
);
assert_eq!(
timestamp_parser("0100-01-01T12:34:56", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011413904000000)
);
assert_eq!(
timestamp_parser("0100-01-01T12:34:56.123456", EvalMode::Legacy, tz, true).unwrap(),
Some(-59011413903876544)
);
assert_eq!(
timestamp_parser("10000", EvalMode::Legacy, tz, true).unwrap(),
Some(253402300800000000)
);
assert_eq!(
timestamp_parser("10000-01", EvalMode::Legacy, tz, true).unwrap(),
Some(253402300800000000)
);
assert_eq!(
timestamp_parser("10000-01-01", EvalMode::Legacy, tz, true).unwrap(),
Some(253402300800000000)
);
assert_eq!(
timestamp_parser("10000-01-01T12", EvalMode::Legacy, tz, true).unwrap(),
Some(253402344000000000)
);
assert_eq!(
timestamp_parser("10000-01-01T12:34", EvalMode::Legacy, tz, true).unwrap(),
Some(253402346040000000)
);
assert_eq!(
timestamp_parser("10000-01-01T12:34:56", EvalMode::Legacy, tz, true).unwrap(),
Some(253402346096000000)
);
assert_eq!(
timestamp_parser("10000-01-01T12:34:56.123456", EvalMode::Legacy, tz, true).unwrap(),
Some(253402346096123456)
);
// Space separator (same values as T separator)
assert_eq!(
timestamp_parser("2020-01-01 12", EvalMode::Legacy, tz, true).unwrap(),
Some(1577880000000000)
);
assert_eq!(
timestamp_parser("2020-01-01 12:34", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882040000000)
);
assert_eq!(
timestamp_parser("2020-01-01 12:34:56", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882096000000)
);
assert_eq!(
timestamp_parser("2020-01-01 12:34:56.123456", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882096123456)
);
// Z suffix (UTC)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56Z", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882096000000)
);
// Positive offset suffix
assert_eq!(
timestamp_parser("2020-01-01T12:34:56+05:30", EvalMode::Legacy, tz, true).unwrap(),
Some(1577862296000000) // 12:34:56 UTC+5:30 = 07:04:56 UTC
);
// T-prefixed time-only with colon
assert!(timestamp_parser("T12:34", EvalMode::Legacy, tz, true)
.unwrap()
.is_some());
assert!(timestamp_parser("T12:34:56", EvalMode::Legacy, tz, true)
.unwrap()
.is_some());
assert!(
timestamp_parser("T12:34:56.123456", EvalMode::Legacy, tz, true)
.unwrap()
.is_some()
);
// Bare time-only (hour:minute without T prefix)
assert!(timestamp_parser("12:34", EvalMode::Legacy, tz, true)
.unwrap()
.is_some());
assert!(timestamp_parser("12:34:56", EvalMode::Legacy, tz, true)
.unwrap()
.is_some());
// Negative year
assert!(timestamp_parser("-0001", EvalMode::Legacy, tz, true)
.unwrap()
.is_some());
assert!(
timestamp_parser("-0001-01-01T12:34:56", EvalMode::Legacy, tz, true)
.unwrap()
.is_some()
);
}
#[test]
#[cfg_attr(miri, ignore)]
fn timestamp_parser_fraction_scaling_test() {
let tz = &Tz::from_str("UTC").unwrap();
// Base: "2020-01-01T12:34:56" = 1577882096000000 µs (confirmed by timestamp_parser_test)
let base = 1577882096000000i64;
// 3-digit fraction: ".123" -> 123_000 µs
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.123", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 123_000)
);
// 1-digit fraction: ".1" -> 100_000 µs
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.1", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 100_000)
);
// 4-digit fraction: ".1000" -> 100_000 µs (trailing zeros not extra precision)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.1000", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 100_000)
);
// 5-digit fraction: ".12312" -> 123_120 µs
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.12312", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 123_120)
);
// 6-digit fraction (exact): unchanged
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.123456", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 123_456)
);
// >6 digits: truncated to 6
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.123456789", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 123_456)
);
// Fraction after Z-stripped offset
assert_eq!(
timestamp_parser("2020-01-01T12:34:56.123Z", EvalMode::Legacy, tz, true).unwrap(),
Some(base + 123_000)
);
}
#[test]
#[cfg_attr(miri, ignore)]
fn timestamp_parser_tz_offset_formats_test() {
let tz = &Tz::from_str("UTC").unwrap();
// All of these represent 2020-01-01T12:34:56 UTC = 1577882096000000 µs.
let utc = 1577882096000000i64;
// +05:30 offset -> UTC = 12:34:56 − 5h30m = 07:04:56 UTC = 1577862296000000 µs
let plus530 = 1577862296000000i64;
// +/-HHMM (no colon)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56+0000", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56+0530", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
// +/-H:MM (single-digit hour)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56+5:30", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56+0:00", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
// +/-H:M (single-digit both)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56+5:3", EvalMode::Legacy, tz, true).unwrap(),
Some(1577863916000000) // 12:34:56 − 5h3m = 07:31:56 UTC = 1577836800+27116
);
// bare UTC / " UTC"
assert_eq!(
timestamp_parser("2020-01-01T12:34:56UTC", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 UTC", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
// UTC+offset
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 UTC+5:30", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
// UTC+0 (single-digit zero)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 UTC+0", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
// GMT+/-HH:MM (no space)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56GMT+00:00", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56GMT+05:30", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
// " GMT+/-..." (space-prefixed)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 GMT+05:30", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
// " GMT+/-HHMM" (space + GMT + no colon)
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 GMT+0530", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
// " UT+/-HH:MM"
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 UT+05:30", EvalMode::Legacy, tz, true).unwrap(),
Some(plus530)
);
// Bare "UT" (no leading space) — Spark accepts "UT" as a UTC alias.
assert_eq!(
timestamp_parser("2020-01-01T12:34:56UT", EvalMode::Legacy, tz, true).unwrap(),
Some(utc)
);
// Java SHORT_IDS: EST (-05:00), MST (-07:00), HST (-10:00)
// 2020-01-01T12:34:56 EST = 2020-01-01T17:34:56 UTC = 1577896496 s
let est_utc = utc + 5 * 3600 * 1_000_000;
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 EST", EvalMode::Legacy, tz, true).unwrap(),
Some(est_utc)
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56EST", EvalMode::Legacy, tz, true).unwrap(),
Some(est_utc)
);
// 2020-01-01T12:34:56 MST = 2020-01-01T19:34:56 UTC
let mst_utc = utc + 7 * 3600 * 1_000_000;
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 MST", EvalMode::Legacy, tz, true).unwrap(),
Some(mst_utc)
);
// 2020-01-01T12:34:56 HST = 2020-01-01T22:34:56 UTC
let hst_utc = utc + 10 * 3600 * 1_000_000;
assert_eq!(
timestamp_parser("2020-01-01T12:34:56 HST", EvalMode::Legacy, tz, true).unwrap(),
Some(hst_utc)
);
// Named IANA zone " Europe/Moscow" (UTC+3 in winter 2020)
// 2020-01-01T12:34:56 Europe/Moscow = 2020-01-01T09:34:56 UTC = 1577871296000000 µs
assert_eq!(
timestamp_parser(
"2020-01-01T12:34:56 Europe/Moscow",
EvalMode::Legacy,
tz,
true
)
.unwrap(),
Some(1577871296000000)
);
// Plain date strings must NOT be affected by the offset-extraction logic.
assert_eq!(
timestamp_parser("2020-01-01", EvalMode::Legacy, tz, true).unwrap(),
Some(1577836800000000)
);
// Invalid offset formats -> null
assert_eq!(
timestamp_parser("2020-01-01T12:34:56-8:", EvalMode::Legacy, tz, true).unwrap(),
None
);
assert_eq!(
timestamp_parser("2020-01-01T12:34:56-20:0", EvalMode::Legacy, tz, true).unwrap(),
None // h=20 > 18 invalid
);
// Positive year-sign prefix is accepted for timestamps (see plus_sign_year_test)
assert_eq!(
timestamp_parser("+2020-01-01T12:34:56", EvalMode::Legacy, tz, true).unwrap(),
Some(1577882096000000)
);
}
#[test]
#[cfg_attr(miri, ignore)]
fn timestamp_parser_dst_test() {
// DST spring-forward: America/New_York springs forward 2020-03-08 02:00 -> 03:00.
// 02:30 does not exist; Spark advances to 03:30 EDT (UTC-4) = 07:30 UTC.
// 2020-03-08T07:30:00Z = 1577836800 + 67*86400 + 27000 = 1583652600 seconds.
let ny_tz = &Tz::from_str("America/New_York").unwrap();
assert_eq!(
timestamp_parser("2020-03-08 02:30:00", EvalMode::Legacy, ny_tz, true).unwrap(),
Some(1583652600000000)
);
// Just before gap: 01:59:59 EST (UTC-5) = 06:59:59 UTC = 1583650799 seconds.
assert_eq!(
timestamp_parser("2020-03-08 01:59:59", EvalMode::Legacy, ny_tz, true).unwrap(),
Some(1583650799000000)
);
// Just after gap: 03:00:00 EDT (UTC-4) = 07:00:00 UTC = 1583650800 seconds.
assert_eq!(
timestamp_parser("2020-03-08 03:00:00", EvalMode::Legacy, ny_tz, true).unwrap(),
Some(1583650800000000)
);
// DST fall-back: 2020-11-01 02:00 EDT -> 01:00 EST. Ambiguous: [01:00, 02:00).
// Spark picks the earlier UTC instant (pre-transition = EDT = UTC-4).
// 01:30 EDT (UTC-4) = 05:30 UTC.
// 2020-11-01 = 2020-01-01 + 305 days = 1577836800 + 305*86400 = 1604188800 seconds.
assert_eq!(
timestamp_parser("2020-11-01 01:30:00", EvalMode::Legacy, ny_tz, true).unwrap(),
Some(1604208600000000) // 1604188800 + 5*3600 + 30*60 = 1604208600
);
}
/// Days from 1970-01-01 for a proleptic-Gregorian civil date (Howard Hinnant's algorithm).
///
/// Used to compute the expected values of the ported Spark tests below independently of
/// the parser under test. Handles the full year range Spark supports, including negative
/// years and years beyond `chrono`'s limits, which the `Long.MinValue`/`MaxValue`
/// boundary cases need.
fn days_from_civil(year: i64, month: i64, day: i64) -> i64 {
let y = year - i64::from(month <= 2);
// Truncating division, as the algorithm requires - not Rust's `%`-consistent one.
let era = if y >= 0 { y } else { y - 399 } / 400;
let year_of_era = y - era * 400; // [0, 399]
let day_of_year = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
era * 146097 + day_of_era - 719468
}
/// Microseconds since the epoch for a UTC civil date-time. The Rust equivalent of the
/// `DateTimeTestUtils.date(..., zid = UTC)` helper the Spark tests below use.
///
/// Computed in `i128` because the `Long.MinValue` boundary case below is exactly
/// `i64::MIN` microseconds, and its intermediate whole-second product overflows `i64`.
fn utc_micros(
year: i64,
month: i64,
day: i64,
hour: i64,
minute: i64,
second: i64,
micros: i64,
) -> i64 {
let seconds = i128::from(days_from_civil(year, month, day)) * 86_400
+ i128::from(hour) * 3_600
+ i128::from(minute) * 60
+ i128::from(second);
i64::try_from(seconds * 1_000_000 + i128::from(micros))
.expect("expected value should fit in i64")
}
/// Reinterprets a UTC-computed instant as a wall-clock time in a fixed zone
/// `offset_seconds` east of UTC. The equivalent of `date(..., zid = getZoneId("+07:30"))`.
fn at_offset(utc_micros: i64, offset_seconds: i64) -> i64 {
utc_micros - offset_seconds * 1_000_000
}
/// Asserts `value` parses to `expected` in every eval mode, under session timezone `tz`.
///
/// Mirrors `checkStringToTimestamp` in Spark's `DateTimeUtilsSuite`, extended over the
/// eval modes Comet has: a `None` expectation additionally requires ANSI mode to raise
/// rather than return null.
fn check_string_to_timestamp(value: &str, expected: Option<i64>, tz: &str) {
let tz = &Tz::from_str(tz).unwrap();
for is_spark4_plus in [false, true] {
for eval_mode in [EvalMode::Legacy, EvalMode::Try] {
assert_eq!(
timestamp_parser(value, eval_mode, tz, is_spark4_plus).unwrap(),
expected,
"{value:?} in {eval_mode:?} (spark4={is_spark4_plus})"
);
}
let ansi = timestamp_parser(value, EvalMode::Ansi, tz, is_spark4_plus);
match expected {
Some(_) => assert_eq!(
ansi.unwrap(),
expected,
"{value:?} in ANSI (spark4={is_spark4_plus})"
),
None => assert!(
ansi.is_err(),
"{value:?} should raise in ANSI (spark4={is_spark4_plus})"
),
}
}
}
/// Port of `test("string to timestamp")` in Spark's `DateTimeUtilsSuite`.
///
/// Spark runs its version over `ALL_TIMEZONES`; the session-timezone dimension is covered
/// separately by `spark_string_to_timestamp_session_timezone_test` below, so this pins the
/// exact expected microseconds under a UTC session timezone.
#[test]
#[cfg_attr(miri, ignore)]
fn spark_string_to_timestamp_test() {
let check =
|value: &str, expected: Option<i64>| check_string_to_timestamp(value, expected, "UTC");
check(
"1969-12-31 16:00:00",
Some(utc_micros(1969, 12, 31, 16, 0, 0, 0)),
);
check("0001", Some(utc_micros(1, 1, 1, 0, 0, 0, 0)));
check("2015-03", Some(utc_micros(2015, 3, 1, 0, 0, 0, 0)));
for value in ["2015-03-18", "2015-03-18 ", " 2015-03-18", " 2015-03-18 "] {
check(value, Some(utc_micros(2015, 3, 18, 0, 0, 0, 0)));
}
let expected = Some(utc_micros(2015, 3, 18, 12, 3, 17, 0));
check("2015-03-18 12:03:17", expected);
check("2015-03-18T12:03:17", expected);
// When the string carries a timezone, that zone wins over the session timezone.
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 0),
-(13 * 3600 + 53 * 60),
));
check("2015-03-18T12:03:17-13:53", expected);
check("2015-03-18T12:03:17GMT-13:53", expected);
check("2015-03-18T12:03:17-1353", expected);
let expected = Some(utc_micros(2015, 3, 18, 12, 3, 17, 0));
check("2015-03-18T12:03:17Z", expected);
check("2015-03-18 12:03:17Z", expected);
check("2015-03-18 12:03:17UTC", expected);
let expected = Some(at_offset(utc_micros(2015, 3, 18, 12, 3, 17, 0), -3600));
check("2015-03-18T12:03:17-1:0", expected);
check("2015-03-18T12:03:17-01:00", expected);
check("2015-03-18T12:03:17GMT-01:00", expected);
check("2015-03-18T12:03:17-0100", expected);
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 0),
7 * 3600 + 30 * 60,
));
check("2015-03-18T12:03:17+07:30", expected);
check("2015-03-18T12:03:17 GMT+07:30", expected);
check("2015-03-18T12:03:17+0730", expected);
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 0),
7 * 3600 + 3 * 60,
));
check("2015-03-18T12:03:17+07:03", expected);
check("2015-03-18T12:03:17GMT+07:03", expected);
check("2015-03-18T12:03:17+0703", expected);
// Strings including milliseconds.
let expected = Some(utc_micros(2015, 3, 18, 12, 3, 17, 123_000));
check("2015-03-18 12:03:17.123", expected);
check("2015-03-18T12:03:17.123", expected);
let expected = Some(utc_micros(2015, 3, 18, 12, 3, 17, 456_000));
check("2015-03-18T12:03:17.456Z", expected);
check("2015-03-18 12:03:17.456Z", expected);
check("2015-03-18 12:03:17.456 UTC", expected);
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 123_000),
-3600,
));
check("2015-03-18T12:03:17.123-1:0", expected);
check("2015-03-18T12:03:17.123-01:00", expected);
check("2015-03-18T12:03:17.123 GMT-01:00", expected);
check("2015-03-18T12:03:17.123-0100", expected);
let plus_730 = 7 * 3600 + 30 * 60;
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 123_000),
plus_730,
));
check("2015-03-18T12:03:17.123+07:30", expected);
check("2015-03-18T12:03:17.123 GMT+07:30", expected);
check("2015-03-18T12:03:17.123+0730", expected);
check("2015-03-18T12:03:17.123GMT+07:30", expected);
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 123_121),
plus_730,
));
check("2015-03-18T12:03:17.123121+7:30", expected);
check("2015-03-18T12:03:17.123121 GMT+0730", expected);
let expected = Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 123_120),
plus_730,
));
check("2015-03-18T12:03:17.12312+7:30", expected);
check("2015-03-18T12:03:17.12312 UT+07:30", expected);
check("2015-03-18T12:03:17.12312+0730", expected);
// Time-only strings are anchored to the current date, so assert against a
// today-relative expectation rather than a fixed instant.
//
// "Today" here is the current date *in the string's own zone*, not in UTC: the parser
// resolves the anchor with `tz.from_utc_datetime(Utc::now())`, and for a string carrying
// its own offset that `tz` is the offset from the string. Using the UTC date instead
// makes every `+07:30` case fail by 24 hours after 16:30 UTC, i.e. for part of each day.
let today_micros = |hour: i64, min: i64, sec: i64, micros: i64, offset: i64| {
use chrono::Datelike;
let today = (chrono::Utc::now() + chrono::Duration::seconds(offset)).date_naive();
Some(at_offset(
utc_micros(
today.year() as i64,
today.month() as i64,
today.day() as i64,
hour,
min,
sec,
micros,
),
offset,
))
};
check("18:12:15", today_micros(18, 12, 15, 0, 0));
// Extreme offsets in both directions, so that the zone date differs from the UTC date
// whatever hour CI runs at: +14:00 crosses from 10:00 UTC onward, -12:00 crosses before
// 12:00 UTC, and together they cover every hour. Anchoring the expectation to the UTC
// date instead of the string's own zone fails one of these by exactly 24 hours.
check("T18:12:15+14:00", today_micros(18, 12, 15, 0, 14 * 3600));
check("T18:12:15-12:00", today_micros(18, 12, 15, 0, -12 * 3600));
let expected = today_micros(18, 12, 15, 123_120, plus_730);
check("T18:12:15.12312+7:30", expected);
check("T18:12:15.12312 UTC+07:30", expected);
check("T18:12:15.12312+0730", expected);
check("18:12:15.12312+7:30", expected);
check("18:12:15.12312 GMT+07:30", expected);
check("18:12:15.12312+0730", expected);
// A 4-digit fraction is truncated to microseconds, not read as extra precision.
check(
"2011-05-06 07:08:09.1000",
Some(utc_micros(2011, 5, 6, 7, 8, 9, 100_000)),
);
for invalid in [
"238",
"2015-03-18 123142",
"2015-03-18T123123",
"2015-03-18X",
"2015/03/18",
"2015.03.18",
"20150318",
"2015-031-8",
"015-01-18",
"2015-03-18T12:03.17-20:0",
"2015-03-18T12:03.17-0:70",
"2015-03-18T12:03.17-1:0:0",
"1999 08 01",
"1999-08 01",
"1999 08",
"",
" ",
"+",
"T",
"2015-03-18T",
"12::",
"2015-03-18T12:03:17-8:",
"2015-03-18T12:03:17-8:30:",
] {
check(invalid, None);
}
// Fractional seconds beyond microsecond precision are truncated, not rounded.
let expected = Some(utc_micros(2015, 3, 18, 12, 3, 17, 123_456));
check("2015-03-18T12:03:17.123456789+0:00", expected);
check("2015-03-18T12:03:17.123456789 UTC+0", expected);
check("2015-03-18T12:03:17.123456789GMT+00:00", expected);
// Named zone suffix. Europe/Moscow was UTC+3 in March 2015.
check(
"2015-03-18T12:03:17.123456 Europe/Moscow",
Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 123_456),
3 * 3600,
)),
);
// Whitespace and control-character permutations. Spark trims leading and trailing
// whitespace *and ISO control* characters before parsing a timestamp
// (`SparkDateTimeUtils.getTrimmedStart`/`getTrimmedEnd`), so a value padded with U+0003
// and spaces in any order still parses.
let expected = Some(utc_micros(2015, 3, 18, 12, 3, 17, 0));
for value in permute_with_whitespace_and_control(&["2015-03-18 12:03:17"]) {
debug_assert!(value.contains('\u{0003}'));
check(&value, expected);
}
// Values Spark rejects are rejected here too, control characters or not.
for value in permute_with_whitespace_and_control(&[
"INVALID_INPUT",
"\t",
"",
"2015-03-18\u{0003}12:03:17",
"2015-03-18 12:",
"2015-03-18 123",
]) {
check(&value, None);
}
// Whitespace-only padding parses too.
for value in [
"2015-03-18 12:03:17",
" 2015-03-18 12:03:17",
"2015-03-18 12:03:17 ",
" 2015-03-18 12:03:17 ",
"\t2015-03-18 12:03:17\t",
] {
check(value, expected);
}
}
/// Port of `permuteWithWhitespaceAndControl` in Spark's `DateTimeUtilsSuite`: every
/// permutation of the input interleaved with two control characters and two spaces.
///
/// Spark permutes `Seq(input, "", "", " ", " ")`, which - because Scala's
/// `permutations` deduplicates equal elements - yields 30 distinct strings per input.
fn permute_with_whitespace_and_control(values: &[&str]) -> Vec<String> {
let mut out = Vec::new();
for value in values {
let parts = [*value, "\u{0003}", "\u{0003}", " ", " "];
let mut seen = std::collections::HashSet::new();
// All 5! orderings; the set collapses the duplicate-element ones, matching Scala.
for indices in permutations_of_5() {
let joined: String = indices.iter().map(|&i| parts[i]).collect();
if seen.insert(joined.clone()) {
out.push(joined);
}
}
}
out
}
/// Every ordering of `[0, 1, 2, 3, 4]`.
fn permutations_of_5() -> Vec<[usize; 5]> {
let mut out = Vec::with_capacity(120);
let mut items = [0, 1, 2, 3, 4];
permute(&mut items, 0, &mut out);
fn permute(items: &mut [usize; 5], k: usize, out: &mut Vec<[usize; 5]>) {
if k == 5 {
out.push(*items);
return;
}
for i in k..5 {
items.swap(k, i);
permute(items, k + 1, out);
items.swap(k, i);
}
}
out
}
/// Port of `test("SPARK-35780: support full range of timestamp string")`.
#[test]
#[cfg_attr(miri, ignore)]
fn spark_string_to_timestamp_full_range_test() {
let check =
|value: &str, expected: Option<i64>| check_string_to_timestamp(value, expected, "UTC");
check(
"-1969-12-31 16:00:00",
Some(utc_micros(-1969, 12, 31, 16, 0, 0, 0)),
);
check(
"02015-03-18 16:00:00",
Some(utc_micros(2015, 3, 18, 16, 0, 0, 0)),
);
check("000001", Some(utc_micros(1, 1, 1, 0, 0, 0, 0)));
check("-000001", Some(utc_micros(-1, 1, 1, 0, 0, 0, 0)));
check("00238", Some(utc_micros(238, 1, 1, 0, 0, 0, 0)));
check(
"99999-03-01T12:03:17",
Some(utc_micros(99999, 3, 1, 12, 3, 17, 0)),
);
// A leading sign is a year sign, so it is rejected on a time-only string.
check("+12:12:12", None);
check("-12:12:12", None);
check("", None);
check(" ", None);
check("+", None);
// i64::MAX microseconds, and one microsecond past it.
let max = utc_micros(294_247, 1, 10, 4, 0, 54, 775_807);
assert_eq!(
max,
i64::MAX,
"the boundary case should be exactly i64::MAX"
);
check("294247-01-10T04:00:54.775807Z", Some(max));
check("294247-01-10T04:00:54.775808Z", None);
// i64::MIN microseconds, and one microsecond before it.
let min = utc_micros(-290_308, 12, 21, 19, 59, 5, 224_192);
assert_eq!(
min,
i64::MIN,
"the boundary case should be exactly i64::MIN"
);
check("-290308-12-21T19:59:05.224192Z", Some(min));
check("-290308-12-21T19:59:05.224191Z", None);
// Overflow of a single segment.
for invalid in [
"4294967297",
"2021-4294967297-11",
"4294967297:30:00",
"2021-11-4294967297T12:30:00",
"2021-01-01T12:4294967297:00",
"2021-01-01T12:30:4294967297",
"2021-01-01T12:30:4294967297.123456",
"2021-01-01T12:30:4294967297+07:30",
"2021-01-01T12:30:4294967297UTC",
"2021-01-01T12:30:4294967297+4294967297:30",
] {
check(invalid, None);
}
}
/// The session-timezone dimension of Spark's `ALL_TIMEZONES` loop: a string with no zone
/// of its own is read in the session timezone, and one that carries a zone ignores it.
#[test]
#[cfg_attr(miri, ignore)]
fn spark_string_to_timestamp_session_timezone_test() {
// (session timezone, its offset for each date used below)
let cases: &[(&str, i64, i64, i64)] = &[
// zone, Dec 1969, Mar 1 2015, Mar 18 2015 / May 2011
("UTC", 0, 0, 0),
// Los Angeles is PST (-8) until DST starts 2015-03-08, PDT (-7) after.
("America/Los_Angeles", -8 * 3600, -8 * 3600, -7 * 3600),
// Tokyo has no DST.
("Asia/Tokyo", 9 * 3600, 9 * 3600, 9 * 3600),
];
for &(zone, off_1969, off_mar_1, off_mar_18) in cases {
let check = |value: &str, expected: Option<i64>| {
check_string_to_timestamp(value, expected, zone)
};
check(
"1969-12-31 16:00:00",
Some(at_offset(utc_micros(1969, 12, 31, 16, 0, 0, 0), off_1969)),
);
check(
"2015-03",
Some(at_offset(utc_micros(2015, 3, 1, 0, 0, 0, 0), off_mar_1)),
);
check(
"2015-03-18",
Some(at_offset(utc_micros(2015, 3, 18, 0, 0, 0, 0), off_mar_18)),
);
check(
"2015-03-18 12:03:17",
Some(at_offset(utc_micros(2015, 3, 18, 12, 3, 17, 0), off_mar_18)),
);
check(
"2015-03-18T12:03:17.123",
Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 123_000),
off_mar_18,
)),
);
check(
"2011-05-06 07:08:09.1000",
Some(at_offset(
utc_micros(2011, 5, 6, 7, 8, 9, 100_000),
off_mar_18,
)),
);
// A zone in the string overrides the session timezone entirely.
check(
"2015-03-18T12:03:17Z",
Some(utc_micros(2015, 3, 18, 12, 3, 17, 0)),
);
check(
"2015-03-18T12:03:17+07:30",
Some(at_offset(
utc_micros(2015, 3, 18, 12, 3, 17, 0),
7 * 3600 + 30 * 60,
)),
);
}
}
/// Port of `test("SPARK-37326: stringToTimestampWithoutTimeZone with allowTimeZone")`.
#[test]
#[cfg_attr(miri, ignore)]
fn spark_string_to_timestamp_ntz_allow_time_zone_test() {
// With a zone allowed, the zone is parsed and then discarded: the local date-time is
// kept as written, not shifted.
assert_eq!(
timestamp_ntz_parser("2021-11-22 10:54:27 +08:00", EvalMode::Legacy, true, true)
.unwrap(),
Some(utc_micros(2021, 11, 22, 10, 54, 27, 0))
);
// With a zone disallowed, the same string is rejected.
assert_eq!(
timestamp_ntz_parser("2021-11-22 10:54:27 +08:00", EvalMode::Legacy, false, true)
.unwrap(),
None
);
}
/// Port of the `stringToTimestamp` half of `test("SPARK-15379: special invalid date
/// string")` - dates that are well-formed but do not exist.
#[test]
#[cfg_attr(miri, ignore)]
fn spark_special_invalid_date_string_timestamp_test() {
for value in [
"2015-02-29 00:00:00",
"2015-04-31 00:00:00",
"2015-02-29",
"2015-04-31",
] {
check_string_to_timestamp(value, None, "UTC");
}
}
// 2020-01-01T00:00:00Z, 2020-01-01T12:34:56Z and the same wall clock at +05:30, in micros.
const JAN1_2020: i64 = 1577836800000000;
const JAN1_2020_123456: i64 = 1577882096000000;
const JAN1_2020_123456_PLUS_0530: i64 = 1577862296000000;
/// Inputs Spark's `parseTimestampString` accepts that the fixed 2-digit shapes rejected:
/// 1-2 digit month/day/hour/minute/second, an empty fraction (also before a zone), and
/// 6-digit years (issue #5674). Values are UTC micros, identical for TIMESTAMP_NTZ.
const SPARK_SEGMENT_RULE_VALID: &[(&str, i64)] = &[
("2020-10-1", 1_601_510_400_000_000),
("2020-12-1", 1_606_780_800_000_000),
("2020-1", JAN1_2020),
("2020-1-1", JAN1_2020),
("2020-1-1T1", JAN1_2020 + 3600 * 1_000_000),
("2020-1-1 1:2", JAN1_2020 + 3720 * 1_000_000),
("2020-01-01 12:34:5", JAN1_2020 + 45245 * 1_000_000),
("2020-1-1T1:2:3.4", JAN1_2020 + 3723 * 1_000_000 + 400_000),
("2020-01-01 12:34:56.", JAN1_2020_123456),
("002020-01-01 00:00:00", JAN1_2020),
];
/// Inputs Spark rejects: non-ASCII segment digits, a zone suffix anywhere but after the
/// seconds segment, more than six year digits, and more than two digits in any other segment.
const SPARK_SEGMENT_RULE_INVALID: &[&str] = &[
"2020-01-01 12:34:56.1٢٢٢",
"T1:2:3.1٢٢٢",
"2020-1-1T٢",
"2020-1-1T1:2:3.٢",
"٢020-1-1",
"2020-٢",
"2020-01-٢",
"2020-1\u{967}",
"2020-\u{967}1",
"2020-01-1\u{967}",
"2020-01-\u{967}1",
"2020-01-01 12:34:56 +08:000",
"2020-01-01 12:34:56 +008:00",
"2020-01-01 12:34:56 +18:01",
"2020-01-01 12:34:56 -18:01",
"2020-01-01 12:34:56+08:000",
"2020-01-01 12:34:56+008:00",
"2020-01-01 12:34:56+18:01",
"2020-01-01 12:34:56 UTC+08:000",
"2020-01-01 12:34:56 GMT+008:00",
"2020-01-01 12:34:56 UT+18:01",
"2020-01-01T1:٢",
"2020-01-01T1:2:٣",
"2020-1-1T1:2:3.٢Z",
"T٢",
"T1:٢",
"T1:2:٣",
"T1:2:3.٢",
"1:٢",
"1:2:٣",
"1:2:3.٢",
"2020Z",
"2020-10-01Z",
"2020-01-01+05:30",
"2020-01-01-08:00",
"2020-10-01 UTC",
"2020-01-01T12Z",
"2020-01-01 12 UTC",
"2020-01-01T12:34Z",
"2020-01-01 12:34 UTC",
"2020-01-01T12:34:Z",
"0002020-01-01",
"0002020-01-01 00:00:00",
"-0002020-01-01",
"2020-001-01",
"2020-01-001",
"2020-01-01T123",
"2020-01-01T12:345",
"2020-01-01T12:34:567",
];
#[test]
#[cfg_attr(miri, ignore)]
fn timestamp_zone_whitespace_matches_java_trim() {
let tz = &Tz::from_str("UTC").unwrap();
for whitespace in [
" ", "\t", "\n", "\u{1}", "\u{b}", "\u{c}", "\u{7f}", "\u{a0}", "\u{2009}", "\u{3000}",
] {
let valid = whitespace.chars().all(|c| c <= '\u{20}');
for (suffix, offset) in [("+08:00", 28_800_000_000), ("UTC", 0), ("Z", 0)] {
for (fraction, micros) in [("", 0), (".123", 123_000)] {
let input = format!("2020-01-01 12:34:56{fraction}{whitespace}{suffix}");
for mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
for spark4 in [false, true] {
for (result, expected) in [
(
timestamp_parser(&input, mode, tz, spark4),
JAN1_2020_123456 + micros - offset,
),
(
timestamp_ntz_parser(&input, mode, true, spark4),
JAN1_2020_123456 + micros,
),
] {
if valid {
assert_eq!(
result.unwrap(),
Some(expected),
"{input:?}, {mode:?}"
);
} else if mode == EvalMode::Ansi {
assert!(
matches!(
result,
Err(SparkError::InvalidInputInCastToDatetime { .. })
),
"{input:?}"
);
} else {
assert_eq!(result.unwrap(), None, "{input:?}, {mode:?}");
}
}
let no_zone = timestamp_ntz_parser(&input, mode, false, spark4);
if mode == EvalMode::Ansi {
assert!(no_zone.is_err(), "{input:?}");
} else {
assert_eq!(no_zone.unwrap(), None, "{input:?}");
}
}
}
}
}
}
}
#[test]
fn timestamp_numeric_offset_validation() {
for (input, seconds) in [
("", 0),
("+0", 0),
("-00", 0),
("+8", 28_800),
("+08", 28_800),
("+0800", 28_800),
("+8:0", 28_800),
("+08:0", 28_800),
("+8:00", 28_800),
("+17:59", 64_740),
("+18:00", 64_800),
("-18:00", -64_800),
("+1800", 64_800),
("-1800", -64_800),
] {
assert_eq!(parse_sign_offset(input), Some(seconds), "{input:?}");
}
for input in [
"+08:000",
"+008:00",
"+18:01",
"-18:01",
"+18:59",
"+19",
"-1900",
"+8:",
"+1:+1",
"-1:-1",
"+1\u{967}",
"+\u{967}1",
"+1٢",
"++1",
] {
assert_eq!(parse_sign_offset(input), None, "{input:?}");
}
}
#[test]
#[cfg_attr(miri, ignore)]
fn timestamp_parser_spark_segment_rules_test() {
let tz = &Tz::from_str("UTC").unwrap();
// Exercise the decoders without their regex gates: malformed UTF-8 boundaries
// must not panic even if the accepted patterns change later.
assert!(
parse_to_timestamp_info("2020-01-01 12:34:56.1٢٢٢", "microsecond")
.unwrap()
.is_none()
);
assert_eq!(
parse_str_to_time_only_timestamp("T1:2:3.1٢٢٢", tz).unwrap(),
None
);
for mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
assert_eq!(
timestamp_parser("2021-11-22 10:54:27 +08:00", mode, tz, true).unwrap(),
Some(1_637_549_667_000_000)
);
assert_eq!(
timestamp_ntz_parser("2021-11-22 10:54:27 +08:00", mode, true, true).unwrap(),
Some(1_637_578_467_000_000)
);
}
for &(input, expected) in SPARK_SEGMENT_RULE_VALID {
for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
assert_eq!(
timestamp_parser(input, eval_mode, tz, true).unwrap(),
Some(expected),
"{input:?} in {eval_mode:?}"
);
}
}
// An empty fraction may still be followed by a zone.
assert_eq!(
timestamp_parser("2020-01-01 12:34:56.Z", EvalMode::Legacy, tz, true).unwrap(),
Some(JAN1_2020_123456)
);
assert_eq!(
timestamp_parser("2020-01-01 12:34:56.+05:30", EvalMode::Legacy, tz, true).unwrap(),
Some(JAN1_2020_123456_PLUS_0530)
);
// Time-only shapes may carry a zone after their seconds segment but not before it.
for input in ["T12:34:56Z", "12:34:56+05:30", "T1:2:3.Z"] {
assert!(
timestamp_parser(input, EvalMode::Ansi, tz, true)
.unwrap()
.is_some(),
"{input:?}"
);
}
for input in
SPARK_SEGMENT_RULE_INVALID
.iter()
.copied()
.chain(["T12Z", "12:34Z", "T12:34 UTC"])
{
for eval_mode in [EvalMode::Legacy, EvalMode::Try] {
assert_eq!(
timestamp_parser(input, eval_mode, tz, true).unwrap(),
None,
"{input:?} in {eval_mode:?}"
);
}
assert!(
timestamp_parser(input, EvalMode::Ansi, tz, true).is_err(),
"{input:?} in Ansi"
);
}
// Shapes that were already accepted keep their exact values.
let la_offset = 8 * 3600 * 1_000_000; // America/Los_Angeles is UTC-8 in January
for (input, expected) in [
("2020-01-01", JAN1_2020),
("2020-01-01 12:34:56", JAN1_2020_123456),
("2020-01-01T12:34:56.123456", JAN1_2020_123456 + 123456),
("2020-01-01T12:34:56Z", JAN1_2020_123456),
("2020-01-01T12:34:56.123Z", JAN1_2020_123456 + 123000),
("2020-01-01T12:34:56+05:30", JAN1_2020_123456_PLUS_0530),
("2020-01-01T12:34:56 UTC", JAN1_2020_123456),
("2020-01-01T12:34:56 UTC+5:30", JAN1_2020_123456_PLUS_0530),
(
"2020-01-01T12:34:56 America/Los_Angeles",
JAN1_2020_123456 + la_offset,
),
("-0001-01-01T12:34:56", -62198709904000000),
] {
for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
assert_eq!(
timestamp_parser(input, eval_mode, tz, true).unwrap(),
Some(expected),
"{input:?} in {eval_mode:?}"
);
}
}
// `date_parser` ports `stringToDate`, whose `maxDigitsYear` is 7, so a date cast keeps
// accepting the 7-digit year that a timestamp cast rejects.
for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
assert_eq!(
date_parser("0002020-01-01", eval_mode).unwrap(),
Some(18262)
);
}
}
#[test]
#[cfg_attr(miri, ignore)]
fn timestamp_ntz_parser_spark_segment_rules_test() {
for allow_time_zone in [true, false] {
for &(input, expected) in SPARK_SEGMENT_RULE_VALID {
for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
assert_eq!(
timestamp_ntz_parser(input, eval_mode, allow_time_zone, false).unwrap(),
Some(expected),
"{input:?} in {eval_mode:?}, allow_time_zone={allow_time_zone}"
);
}
}
for &input in SPARK_SEGMENT_RULE_INVALID {
for eval_mode in [EvalMode::Legacy, EvalMode::Try] {
assert_eq!(
timestamp_ntz_parser(input, eval_mode, allow_time_zone, false).unwrap(),
None,
"{input:?} in {eval_mode:?}, allow_time_zone={allow_time_zone}"
);
}
assert!(
timestamp_ntz_parser(input, EvalMode::Ansi, allow_time_zone, false).is_err(),
"{input:?} in Ansi, allow_time_zone={allow_time_zone}"
);
}
}
// A zone after an empty fraction is discarded when allowed and rejected otherwise.
assert_eq!(
timestamp_ntz_parser("2020-01-01 12:34:56.Z", EvalMode::Legacy, true, false).unwrap(),
Some(JAN1_2020_123456)
);
assert_eq!(
timestamp_ntz_parser("2020-01-01 12:34:56.Z", EvalMode::Legacy, false, false).unwrap(),
None
);
assert!(
timestamp_ntz_parser("2020-01-01 12:34:56.Z", EvalMode::Ansi, false, false).is_err()
);
}
#[test]
#[cfg_attr(miri, ignore)]
fn test_cast_string_to_timestamp_spark_segment_rules_array() {
// The reproducer from issue #5674, through the batch entry points.
let inputs = vec![
Some("2020-1-1"),
Some("2020-01-01 12:34:5"),
Some("2020-01-01 12:34:56."),
Some("2020-10-01Z"),
Some("0002020-01-01 00:00:00"),
];
let expected = [
Some(JAN1_2020),
Some(JAN1_2020 + 45245 * 1_000_000),
Some(JAN1_2020_123456),
None,
None,
];
let array: ArrayRef = Arc::new(StringArray::from(inputs));
let to_type = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()));
let tz_result =
cast_string_to_timestamp(&array, &to_type, EvalMode::Legacy, "UTC", true).unwrap();
let ntz_result =
cast_string_to_timestamp_ntz(&array, EvalMode::Legacy, true, false).unwrap();
for result in [&tz_result, &ntz_result] {
let result = result
.as_any()
.downcast_ref::<PrimitiveArray<TimestampMicrosecondType>>()
.unwrap();
let actual: Vec<Option<i64>> = result.iter().collect();
assert_eq!(actual, expected);
}
// Under ANSI the first malformed row fails the batch and names the raw input.
let tz_err =
cast_string_to_timestamp(&array, &to_type, EvalMode::Ansi, "UTC", true).unwrap_err();
let ntz_err =
cast_string_to_timestamp_ntz(&array, EvalMode::Ansi, true, false).unwrap_err();
for (err, expected_type) in [(tz_err, "TIMESTAMP"), (ntz_err, "TIMESTAMP_NTZ")] {
match err {
SparkError::InvalidInputInCastToDatetime {
value,
from_type,
to_type,
} => {
assert_eq!(value, "2020-10-01Z");
assert_eq!(from_type, "STRING");
assert_eq!(to_type, expected_type);
}
other => panic!("Expected InvalidInputInCastToDatetime, got {other:?}"),
}
}
}
/// Asserts every date parses to null in legacy and try mode. When `expect_ansi_error` is set,
/// ANSI mode must raise CAST_INVALID_INPUT; otherwise ANSI mode must also return null.
fn assert_dates(dates: &[&str], expect_ansi_error: bool) {
for &date in dates {
for eval_mode in [EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
if expect_ansi_error && eval_mode == EvalMode::Ansi {
assert!(date_parser(date, eval_mode).is_err(), "{date}");
} else {
assert_eq!(date_parser(date, eval_mode).unwrap(), None, "{date}");
}
}
}
}
/// Malformed input: null in legacy and try mode, CAST_INVALID_INPUT in ANSI mode.
fn assert_null_or_ansi_error(dates: &[&str]) {
assert_dates(dates, true);
}
/// Input Spark parses successfully but Comet cannot represent: null in every eval mode.
fn assert_null_in_all_modes(dates: &[&str]) {
assert_dates(dates, false);
}
#[test]
fn date_parser_test() {
for date in &[
"2020",
"2020-01",
"2020-01-01",
"+2020-01-01", // Spark accepts '+' year prefix on dates
"02020-01-01",
"002020-01-01",
"0002020-01-01",
"2020-1-1",
"2020-01-01 ",
"2020-01-01T",
] {
for eval_mode in &[EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] {
assert_eq!(date_parser(date, *eval_mode).unwrap(), Some(18262));
}
}
//dates in invalid formats
assert_null_or_ansi_error(&[
"abc",
"",
"not_a_date",
"3/",
"3/12",
"3/12/2020",
"3/12/2002 T",
"202",
"2020-010-01",
"2020-10-010",
"2020-10-010T",
"--262143-12-31",
"--262143-12-31 ",
]);
for date in &["-3638-5"] {
for eval_mode in &[EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
assert_eq!(date_parser(date, *eval_mode).unwrap(), Some(-2048160));
}
}
//Naive Date only supports years 262142 AD to 262143 BC. Spark parses these fine, so
//they are a Comet limitation rather than malformed input and stay null in ANSI mode.
assert_null_in_all_modes(&[
"-262144-1-1",
"262143-01-1",
"262143-1-1",
"262143-01-1 ",
"262143-01-01T ",
"262143-1-01T 1234",
"-0973250",
]);
//years whose epoch day overflows i32 are rejected by Spark too (localDateToDays uses
//Math.toIntExact), so ANSI mode must raise rather than return null
assert_null_or_ansi_error(&["9999999-01-01", "-9999999-01-01"]);
// Canonical `yyyy-mm-dd` shape with invalid calendar dates exercises the fast path.
// Spark's LocalDate.of rejects these, so ANSI mode must raise (issue #5012).
assert_null_or_ansi_error(&[
"2020-02-30",
"2021-02-29",
"2020-13-01",
"2020-00-15",
"2020-04-31",
"2020-01-00",
]);
// Same invalid calendar dates in non-canonical shapes take the general parser path.
assert_null_or_ansi_error(&["2020-2-30", "2020-13-1", "2020-4-31 ", "2020-02-30T"]);
// Valid leap day flows through the fast path.
assert_eq!(
date_parser("2020-02-29", EvalMode::Legacy).unwrap(),
Some(18321)
);
}
#[test]
fn test_cast_string_to_date() {
let array: ArrayRef = Arc::new(StringArray::from(vec![
Some("2020"),
Some("2020-01"),
Some("2020-01-01"),
Some("2020-01-01T"),
]));
let result = cast_string_to_date(&array, &DataType::Date32, EvalMode::Legacy).unwrap();
let date32_array = result
.as_any()
.downcast_ref::<arrow::array::Date32Array>()
.unwrap();
assert_eq!(date32_array.len(), 4);
date32_array
.iter()
.for_each(|v| assert_eq!(v.unwrap(), 18262));
}
#[test]
fn test_cast_string_array_with_valid_dates() {
let array_with_invalid_date: ArrayRef = Arc::new(StringArray::from(vec![
Some("-262143-12-31"),
Some("\n -262143-12-31 "),
Some("-262143-12-31T \t\n"),
Some("\n\t-262143-12-31T\r"),
Some("-262143-12-31T 123123123"),
Some("\r\n-262143-12-31T \r123123123"),
Some("\n -262143-12-31T \n\t"),
]));
for eval_mode in &[EvalMode::Legacy, EvalMode::Try, EvalMode::Ansi] {
let result =
cast_string_to_date(&array_with_invalid_date, &DataType::Date32, *eval_mode)
.unwrap();
let date32_array = result
.as_any()
.downcast_ref::<arrow::array::Date32Array>()
.unwrap();
assert_eq!(result.len(), 7);
date32_array
.iter()
.for_each(|v| assert_eq!(v.unwrap(), -96464928));
}
}
#[test]
fn test_cast_string_array_with_invalid_dates() {
let array_with_invalid_date: ArrayRef = Arc::new(StringArray::from(vec![
Some("2020"),
Some("2020-01"),
Some("2020-01-01"),
//4 invalid dates
Some("2020-010-01T"),
Some("202"),
Some(" 202 "),
Some("\n 2020-\r8 "),
Some("2020-01-01T"),
// Overflows i32
Some("-4607172990231812908"),
]));
for eval_mode in &[EvalMode::Legacy, EvalMode::Try] {
let result =
cast_string_to_date(&array_with_invalid_date, &DataType::Date32, *eval_mode)
.unwrap();
let date32_array = result
.as_any()
.downcast_ref::<arrow::array::Date32Array>()
.unwrap();
assert_eq!(
date32_array.iter().collect::<Vec<_>>(),
vec![
Some(18262),
Some(18262),
Some(18262),
None,
None,
None,
None,
Some(18262),
None
]
);
}
let result =
cast_string_to_date(&array_with_invalid_date, &DataType::Date32, EvalMode::Ansi);
match result {
Err(e) => assert!(
e.to_string().contains(
"[CAST_INVALID_INPUT] The value '2020-010-01T' of the type \"STRING\" cannot be cast to \"DATE\" because it is malformed")
),
_ => panic!("Expected error"),
}
}
#[test]
fn test_cast_string_as_i8() {
// basic
assert_eq!(
cast_string_to_i8("127", EvalMode::Legacy).unwrap(),
Some(127_i8)
);
assert_eq!(cast_string_to_i8("128", EvalMode::Legacy).unwrap(), None);
assert!(cast_string_to_i8("128", EvalMode::Ansi).is_err());
// decimals
assert_eq!(
cast_string_to_i8("0.2", EvalMode::Legacy).unwrap(),
Some(0_i8)
);
assert_eq!(
cast_string_to_i8(".", EvalMode::Legacy).unwrap(),
Some(0_i8)
);
// TRY should always return null for decimals
assert_eq!(cast_string_to_i8("0.2", EvalMode::Try).unwrap(), None);
assert_eq!(cast_string_to_i8(".", EvalMode::Try).unwrap(), None);
// ANSI mode should throw error on decimal
assert!(cast_string_to_i8("0.2", EvalMode::Ansi).is_err());
assert!(cast_string_to_i8(".", EvalMode::Ansi).is_err());
}
}