IMPALA-8706: ISO:SQL:2016 datetime patterns - Milestone 4
This patch adds ISO 8601 week-based date format tokens on top
of what was introduced in IMPALA-8703, IMPALA-8704 and
IMPALA-8705.
The ISO 8601 week-based date tokens may be used for both datetime
to string and string to datetime conversion.
The ISO 8601 week-based date tokens are as follows:
- IYYY: 4-digit ISO 8601 week-numbering year.
Week-numbering year is the year relating to the ISO
8601 week number (IW), which is the full week (Monday
to Sunday) which contains January 4 of the Gregorian
year.
Behaves similarly to YYYY in that for datetime to
string conversion, prefix digits for 1, 2, and 3-digit
inputs are obtained from current ISO 8601
week-numbering year.
- IYY: Last 3 digits of ISO 8601 week-numbering year.
Behaves similarly to YYY in that for datetime to string
conversion, prefix digit is obtained from current ISO
8601 week-numbering year and can accept 1 or 2-digit
input.
- IY: Last 2 digits of ISO 8601 week-numbering year.
Behaves similarly to YY in that for datetime to string
conversion, prefix digits are obtained from current ISO
8601 week-numbering year and can accept 1-digit input.
- I: Last digit of ISO 8601 week-numbering year.
Behaves similarly to Y in that for datetime to string
conversion, prefix digits are obtained from current ISO
8601 week-numbering year.
- IW: ISO 8601 week of year (1-53).
Begins on the Monday closest to January 1 of the year.
For string to datetime conversion, if the input ISO
8601 week does not exist in the input year, an error
will be thrown.
Note that IW is different from the other week-related
tokens WW and W (implemented in IMPALA-8705). With WW
and W weeks start with the first day of the
year/month. ISO 8601 weeks on the other hand always
start with Monday.
- ID: ISO 8601 day of week (1-7). 1 means Monday and 7 means
Sunday.
When doing string to datetime conversion, the ISO 8601 week-based
tokens are meant to be used together and not mixed with other ISO
SQL date tokens. E.g. 'YYYY-IW-ID' is an invalid format string.
The only exceptions are the day name tokens (DAY and DY) which
may be used instead of ID with the rest of the ISO 8601
week-based date tokens. E.g. 'IYYY-IW-DAY' is a valid format
string.
Change-Id: I89a8c1b98742391cb7b331840d216558dbca362b
Reviewed-on: http://gerrit.cloudera.org:8080/14852
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Gabor Kaszab <gaborkaszab@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
diff --git a/be/src/exprs/date-functions-ir.cc b/be/src/exprs/date-functions-ir.cc
index 8e16fb5..a8638ca 100644
--- a/be/src/exprs/date-functions-ir.cc
+++ b/be/src/exprs/date-functions-ir.cc
@@ -95,7 +95,7 @@
if (d_val.is_null) return IntVal::null();
DateValue dv = DateValue::FromDateVal(d_val);
- int yweek = dv.WeekOfYear();
+ int yweek = dv.Iso8601WeekOfYear();
if (yweek == -1) return IntVal::null();
return IntVal(yweek);
}
diff --git a/be/src/runtime/date-parse-util.cc b/be/src/runtime/date-parse-util.cc
index 579a895..1f820f5 100644
--- a/be/src/runtime/date-parse-util.cc
+++ b/be/src/runtime/date-parse-util.cc
@@ -137,6 +137,7 @@
int year, month, day;
if (!date.ToYearMonthDay(&year, &month, &day)) return "";
+ DCHECK(date.IsValid());
string result;
result.reserve(dt_ctx.fmt_out_len);
for (const DateTimeFormatToken& tok: dt_ctx.toks) {
@@ -144,11 +145,7 @@
switch (tok.type) {
case YEAR:
case ROUND_YEAR: {
- num_val = year;
- if (tok.len < 4) {
- int adjust_factor = std::pow(10, tok.len);
- num_val %= adjust_factor;
- }
+ num_val = AdjustYearToLength(year, tok.len);
break;
}
case QUARTER_OF_YEAR: {
@@ -191,6 +188,20 @@
result.append(FormatTextToken(tok));
break;
}
+ case ISO8601_WEEK_NUMBERING_YEAR: {
+ num_val = AdjustYearToLength(date.Iso8601WeekNumberingYear(), tok.len);
+ break;
+ }
+ case ISO8601_WEEK_OF_YEAR: {
+ num_val = date.Iso8601WeekOfYear();
+ break;
+ }
+ case ISO8601_DAY_OF_WEEK: {
+ // WeekDay() returns 0 for Monday and 6 for Sunday.
+ // We need to output 1 for Monday and 7 for Sunday.
+ num_val = date.WeekDay() + 1;
+ break;
+ }
default: DCHECK(false) << "Unknown date format token";
}
if (num_val > -1) {
diff --git a/be/src/runtime/date-test.cc b/be/src/runtime/date-test.cc
index 404c831..3072dc5 100644
--- a/be/src/runtime/date-test.cc
+++ b/be/src/runtime/date-test.cc
@@ -824,10 +824,10 @@
EXPECT_EQ(366, DateValue(2000, 12, 31).DayOfYear());
}
-TEST(DateTest, WeekOfYear) {
+TEST(DateTest, Iso8601WeekOfYear) {
// Test that it returns -1 for invalid dates.
DateValue invalid_dv;
- EXPECT_EQ(-1, invalid_dv.WeekOfYear());
+ EXPECT_EQ(-1, invalid_dv.Iso8601WeekOfYear());
// Iterate through days of 2019.
// 2019-01-01 is Tuesday and 2019-12-31 is Tuesday too.
@@ -836,36 +836,144 @@
for (DateValue dv = jan1;
dv <= DateValue(2019, 12, 29);
dv = dv.AddDays(1)) {
- EXPECT_EQ(weekday_offset / 7 + 1, dv.WeekOfYear());
+ EXPECT_EQ(weekday_offset / 7 + 1, dv.Iso8601WeekOfYear());
++weekday_offset;
}
// Year 2015 has 53 weeks. 2015-12-31 is Thursday.
- EXPECT_EQ(53, DateValue(2015, 12, 31).WeekOfYear());
+ EXPECT_EQ(53, DateValue(2015, 12, 31).Iso8601WeekOfYear());
// 2019-12-30 (Monday) and 2019-12-31 (Tuesday) belong to year 2020.
- EXPECT_EQ(1, DateValue(2019, 12, 30).WeekOfYear());
- EXPECT_EQ(1, DateValue(2019, 12, 31).WeekOfYear());
- EXPECT_EQ(1, DateValue(2020, 1, 1).WeekOfYear());
- EXPECT_EQ(1, DateValue(2020, 1, 5).WeekOfYear());
- EXPECT_EQ(2, DateValue(2020, 1, 6).WeekOfYear());
+ EXPECT_EQ(1, DateValue(2019, 12, 30).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(2019, 12, 31).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(2020, 1, 1).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(2020, 1, 5).Iso8601WeekOfYear());
+ EXPECT_EQ(2, DateValue(2020, 1, 6).Iso8601WeekOfYear());
// 0002-01-01 is Tuesday. Test days around 0002-01-01.
- EXPECT_EQ(51, DateValue(1, 12, 23).WeekOfYear());
- EXPECT_EQ(52, DateValue(1, 12, 30).WeekOfYear());
- EXPECT_EQ(1, DateValue(1, 12, 31).WeekOfYear());
- EXPECT_EQ(1, DateValue(2, 1, 1).WeekOfYear());
- EXPECT_EQ(1, DateValue(2, 1, 6).WeekOfYear());
- EXPECT_EQ(2, DateValue(2, 1, 7).WeekOfYear());
+ EXPECT_EQ(51, DateValue(1, 12, 23).Iso8601WeekOfYear());
+ EXPECT_EQ(52, DateValue(1, 12, 30).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(1, 12, 31).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(2, 1, 1).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(2, 1, 6).Iso8601WeekOfYear());
+ EXPECT_EQ(2, DateValue(2, 1, 7).Iso8601WeekOfYear());
// 0001-01-01 is Monday. Test days around 0001-01-01.
- EXPECT_EQ(1, DateValue(1, 1, 1).WeekOfYear());
- EXPECT_EQ(1, DateValue(1, 1, 2).WeekOfYear());
- EXPECT_EQ(2, DateValue(1, 1, 8).WeekOfYear());
+ EXPECT_EQ(1, DateValue(1, 1, 1).Iso8601WeekOfYear());
+ EXPECT_EQ(1, DateValue(1, 1, 2).Iso8601WeekOfYear());
+ EXPECT_EQ(2, DateValue(1, 1, 8).Iso8601WeekOfYear());
// 9999-12-31 is Friday. Test days around 9999-12-31.
- EXPECT_EQ(52, DateValue(9999, 12, 31).WeekOfYear());
- EXPECT_EQ(52, DateValue(9999, 12, 27).WeekOfYear());
- EXPECT_EQ(51, DateValue(9999, 12, 26).WeekOfYear());
+ EXPECT_EQ(52, DateValue(9999, 12, 31).Iso8601WeekOfYear());
+ EXPECT_EQ(52, DateValue(9999, 12, 27).Iso8601WeekOfYear());
+ EXPECT_EQ(51, DateValue(9999, 12, 26).Iso8601WeekOfYear());
+}
+
+TEST(DateTest, Iso8601WeekNumberingYear) {
+ // Test that it returns -1 for invalid dates.
+ DateValue invalid_dv;
+ EXPECT_EQ(-1, invalid_dv.Iso8601WeekNumberingYear());
+
+ // Iterate through days of 2019.
+ // 2019-01-01 is Tuesday and 2019-12-29 is Sunday.
+ DateValue jan1(2019, 1, 1);
+ for (DateValue dv = jan1;
+ dv <= DateValue(2019, 12, 29);
+ dv = dv.AddDays(1)) {
+ EXPECT_EQ(2019, dv.Iso8601WeekNumberingYear());
+ }
+ // 2019-12-30 (Monday) and 2019-12-31 (Tuesday) belong to year 2020.
+ EXPECT_EQ(2020, DateValue(2019, 12, 30).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2020, DateValue(2019, 12, 31).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2020, DateValue(2020, 1, 1).Iso8601WeekNumberingYear());
+
+ // 2015-01-01 is Thursday and 2015-12-31 is Thursday too.
+ // Both days belong to year 2015.
+ EXPECT_EQ(2015, DateValue(2015, 1, 1).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2015, DateValue(2015, 12, 31).Iso8601WeekNumberingYear());
+
+ // 2040-01-01 is Sunday and 2040-12-31 is Monday.
+ // Neither days belong to year 2040.
+ EXPECT_EQ(2039, DateValue(2040, 1, 1).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2041, DateValue(2040, 12, 31).Iso8601WeekNumberingYear());
+
+ // 0002-01-01 is Tuesday. Test days around 0002-01-01.
+ EXPECT_EQ(1, DateValue(1, 12, 29).Iso8601WeekNumberingYear());
+ EXPECT_EQ(1, DateValue(1, 12, 30).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2, DateValue(1, 12, 31).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2, DateValue(2, 1, 1).Iso8601WeekNumberingYear());
+ EXPECT_EQ(2, DateValue(2, 1, 2).Iso8601WeekNumberingYear());
+ // 0001-01-01 is Monday. Test days around 0001-01-01.
+ EXPECT_EQ(1, DateValue(1, 1, 1).Iso8601WeekNumberingYear());
+ EXPECT_EQ(1, DateValue(1, 1, 2).Iso8601WeekNumberingYear());
+
+ // 9999-12-31 is Friday. Test days around 9999-12-31.
+ EXPECT_EQ(9999, DateValue(9999, 12, 30).Iso8601WeekNumberingYear());
+ EXPECT_EQ(9999, DateValue(9999, 12, 31).Iso8601WeekNumberingYear());
+}
+
+TEST(DateTest, CreateFromIso8601WeekBasedDateVals) {
+ // Invalid week numbering year.
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(-1, 1, 1).IsValid());
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(0, 1, 1).IsValid());
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(10000, 1, 1).IsValid());
+
+ // Test invalid week of year.
+ // Year 2020 has 53 weeks.
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(2020, 54, 1).IsValid());
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(2020, 0, 1).IsValid());
+ // Year 2019 has 52 weeks.
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(2019, 53, 1).IsValid());
+
+ // Test invalid week day.
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(2020, 1, 0).IsValid());
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(2020, 1, 8).IsValid());
+
+ // 0001-01-01 is Monday. It belongs to the first week of year 1.
+ // Test days around 0001-01-01.
+ EXPECT_EQ(DateValue(1, 1, 1), DateValue::CreateFromIso8601WeekBasedDateVals(1, 1, 1));
+ EXPECT_EQ(DateValue(1, 1, 2), DateValue::CreateFromIso8601WeekBasedDateVals(1, 1, 2));
+ EXPECT_EQ(DateValue(1, 1, 7), DateValue::CreateFromIso8601WeekBasedDateVals(1, 1, 7));
+ EXPECT_EQ(DateValue(1, 1, 8), DateValue::CreateFromIso8601WeekBasedDateVals(1, 2, 1));
+ // 0001-12-30 is Sunday, belongs to week 52 of year 1.
+ EXPECT_EQ(DateValue(1, 12, 30),
+ DateValue::CreateFromIso8601WeekBasedDateVals(1, 52, 7));
+ // 0001-12-31 is Monday, belongs to year 2.
+ EXPECT_EQ(DateValue(1, 12, 31), DateValue::CreateFromIso8601WeekBasedDateVals(2, 1, 1));
+ EXPECT_EQ(DateValue(2, 1, 1), DateValue::CreateFromIso8601WeekBasedDateVals(2, 1, 2));
+
+ // Test 2020 ISO 8601 week numbering year.
+ // 2019-12-30 is Monday, belongs to week 1 of year 2020.
+ // 2021-01-03 is Sunday, belongs to week 53 of year 2020.
+ int week_of_year = 1, day_of_week = 1;
+ for (DateValue dv(2019, 12, 30);
+ dv <= DateValue(2021, 1, 3);
+ dv = dv.AddDays(1)) {
+ DateValue dv_iso8601 = DateValue::CreateFromIso8601WeekBasedDateVals(2020,
+ week_of_year, day_of_week);
+ EXPECT_TRUE(dv_iso8601.IsValid());
+ EXPECT_EQ(dv, dv_iso8601);
+
+ if (day_of_week == 7) ++week_of_year;
+ day_of_week = day_of_week % 7 + 1;
+ }
+ EXPECT_EQ(54, week_of_year);
+ EXPECT_EQ(1, day_of_week);
+
+ // 9998-12-31 is Thursday, belongs to week 53 of year 9998.
+ EXPECT_EQ(DateValue(9998, 12, 31),
+ DateValue::CreateFromIso8601WeekBasedDateVals(9998, 53, 4));
+ EXPECT_EQ(DateValue(9999, 1, 1),
+ DateValue::CreateFromIso8601WeekBasedDateVals(9998, 53, 5));
+ EXPECT_EQ(DateValue(9999, 1, 2),
+ DateValue::CreateFromIso8601WeekBasedDateVals(9998, 53, 6));
+ EXPECT_EQ(DateValue(9999, 1, 3),
+ DateValue::CreateFromIso8601WeekBasedDateVals(9998, 53, 7));
+ EXPECT_EQ(DateValue(9999, 1, 4),
+ DateValue::CreateFromIso8601WeekBasedDateVals(9999, 1, 1));
+ // 9999-12-31 is Friday, belongs to week 52 of year 9999.
+ EXPECT_EQ(DateValue(9999, 12, 31),
+ DateValue::CreateFromIso8601WeekBasedDateVals(9999, 52, 5));
+ EXPECT_FALSE(DateValue::CreateFromIso8601WeekBasedDateVals(9999, 52, 6).IsValid());
}
TEST(DateTest, LastDay) {
diff --git a/be/src/runtime/date-value.cc b/be/src/runtime/date-value.cc
index d3fd4a0..ed783b1 100644
--- a/be/src/runtime/date-value.cc
+++ b/be/src/runtime/date-value.cc
@@ -58,6 +58,71 @@
}
}
+namespace {
+
+inline int32_t CalcFirstDayOfYearSinceEpoch(int year) {
+ int m400 = year % 400;
+ int m100 = m400 % 100;
+ int m4 = m100 % 4;
+
+ return (year - EPOCH_YEAR) * 365
+ + ((year - EPOCH_YEAR / 4 * 4 + ((m4 != 0) ? 4 - m4 : 0)) / 4 - 1)
+ - ((year - EPOCH_YEAR / 100 * 100 + ((m100 != 0) ? 100 - m100 : 0)) / 100 - 1)
+ + ((year - EPOCH_YEAR / 400 * 400 + ((m400 != 0) ? 400 - m400 : 0)) / 400 - 1);
+}
+
+// Returns the Monday that belongs to the first ISO 8601 week of 'year'. That is:
+// - If Jan 1 falls on Monday, Tuesday, Wednesday or Thursday, then the week of Jan 1 is
+// the first ISO 8601 week of 'year'. Therefore the Monday of the first ISO 8601 week in
+// 'year' is the first Monday before Jan 2.
+// - If Jan 1 falls on Friday, Saturday or Sunday, then the week of Jan 1 is the last ISO
+// 8601 week of the previous year. Therefore the Monday of the first ISO 8601 week in
+// 'year' is the first Monday following Jan 1.
+inline cctz::civil_day GetFirstIso8601MondayOfYear(int year) {
+ cctz::civil_day jan1 = cctz::civil_day(year, 1, 1);
+ if (cctz::get_weekday(jan1) <= cctz::weekday::thursday) {
+ // Get the previous Monday if 'jan1' is not already a Monday.
+ return cctz::next_weekday(jan1, cctz::weekday::monday) - 7;
+ } else {
+ // Get the next Monday.
+ return cctz::next_weekday(jan1, cctz::weekday::monday);
+ }
+}
+
+// Returns Sunday that belongs to the last ISO 8601 week of 'year'. That is:
+// - If Dec 31 falls on Thursday, Friday, Saturday or Sunday, then the week of Dec 31
+// is the last ISO 8601 week of 'year'. Therefore the Sunday of the last ISO 8601 week
+// in 'year' is the first Sunday after Dec 30.
+// - If Dec 31 falls on Monday, Tuesday or Wednesday, then the week of Dec 31 is the first
+// ISO 8601 week of the next year. Therefore the Sunday of the last ISO 8601 week in
+// 'year' is the first Sunday previous to Dec 31.
+inline cctz::civil_day GetLastIso8601SundayOfYear(int year) {
+ cctz::civil_day dec31 = cctz::civil_day(year, 12, 31);
+ if (cctz::get_weekday(dec31) >= cctz::weekday::thursday) {
+ // Get the next Sunday if 'dec31' is not already a Sunday.
+ return cctz::prev_weekday(dec31, cctz::weekday::sunday) + 7;
+ } else {
+ // Get the previous Sunday.
+ return cctz::prev_weekday(dec31, cctz::weekday::sunday);
+ }
+}
+
+}
+
+DateValue DateValue::CreateFromIso8601WeekBasedDateVals(int year,
+ int week_of_year, int day_of_week) {
+ if (year >= MIN_YEAR && year <= MAX_YEAR
+ && week_of_year >= 1 && week_of_year <= 53
+ && 1 <= day_of_week && day_of_week <= 7) {
+ const cctz::civil_day first_monday = GetFirstIso8601MondayOfYear(year);
+ const cctz::civil_day last_sunday = GetLastIso8601SundayOfYear(year);
+
+ const cctz::civil_day today = first_monday + (week_of_year - 1) * 7 + day_of_week - 1;
+ if (today <= last_sunday) return DateValue(today - EPOCH_DATE);
+ }
+ return DateValue();
+}
+
DateValue DateValue::ParseSimpleDateFormat(const char* str, int len,
bool accept_time_toks) {
DateValue dv;
@@ -87,21 +152,6 @@
return DateParser::Format(dt_ctx, *this);
}
-namespace {
-
-inline int32_t CalcFirstDayOfYearSinceEpoch(int year) {
- int m400 = year % 400;
- int m100 = m400 % 100;
- int m4 = m100 % 4;
-
- return (year - EPOCH_YEAR) * 365
- + ((year - EPOCH_YEAR / 4 * 4 + ((m4 != 0) ? 4 - m4 : 0)) / 4 - 1)
- - ((year - EPOCH_YEAR / 100 * 100 + ((m100 != 0) ? 100 - m100 : 0)) / 100 - 1)
- + ((year - EPOCH_YEAR / 400 * 400 + ((m400 != 0) ? 400 - m400 : 0)) / 400 - 1);
-}
-
-}
-
bool DateValue::ToYear(int* year) const {
DCHECK(year != nullptr);
if (UNLIKELY(!IsValid())) return false;
@@ -144,7 +194,7 @@
// f(year) = - ((m4 != 0) ? (4 - m4) * 100 : 0)
// + ((m100 != 0) ? (100 - m100) * 4 : 0)
// - ((m400 != 0) ? 400 - m400 : 0)
- // and 'year' is in the [0, 9999] range, then it follows that (B):
+ // and 'year' is in the [1, 9999] range, then it follows that (B):
// f(year) must fall into the [-591, 288] range.
//
// Finally, if we put (A) and (B) together we can conclude that 'year' must fall into
@@ -206,38 +256,24 @@
return static_cast<int>(cctz::get_yearday(cd));
}
-int DateValue::WeekOfYear() const {
+int DateValue::Iso8601WeekOfYear() const {
if (UNLIKELY(!IsValid())) return -1;
const cctz::civil_day today = EPOCH_DATE + days_since_epoch_;
-
- cctz::civil_day jan1 = cctz::civil_day(today.year(), 1, 1);
- cctz::civil_day first_monday;
- if (cctz::get_weekday(jan1) <= cctz::weekday::thursday) {
- // Get the previous Monday if 'jan1' is not already a Monday.
- first_monday = cctz::next_weekday(jan1, cctz::weekday::monday) - 7;
- } else {
- // Get the next Monday.
- first_monday = cctz::next_weekday(jan1, cctz::weekday::monday);
- }
-
- cctz::civil_day dec31 = cctz::civil_day(today.year(), 12, 31);
- cctz::civil_day last_sunday;
- if (cctz::get_weekday(dec31) >= cctz::weekday::thursday) {
- // Get the next Sunday if 'dec31' is not already a Sunday.
- last_sunday = cctz::prev_weekday(dec31, cctz::weekday::sunday) + 7;
- } else {
- // Get the previous Sunday.
- last_sunday = cctz::prev_weekday(dec31, cctz::weekday::sunday);
- }
+ const cctz::civil_day first_monday = GetFirstIso8601MondayOfYear(today.year());
+ const cctz::civil_day last_sunday = GetLastIso8601SundayOfYear(today.year());
// 0001-01-01 is Monday in the proleptic Gregorian calendar.
// 0001-01-01 belongs to year 0001.
+ // 9999-12-31 is Friday in the proleptic Gregorian calendar.
+ // 9999-12-31 belongs to year 9999.
if (today >= first_monday && today <= last_sunday) {
return (today - first_monday) / 7 + 1;
} else if (today > last_sunday) {
+ DCHECK(today.year() >= MIN_YEAR && today.year() < MAX_YEAR);
return 1;
} else {
- // today < first_monday && today.year() > 0
+ DCHECK(today < first_monday);
+ DCHECK(today.year() > MIN_YEAR && today.year() <= MAX_YEAR);
cctz::civil_day prev_jan1 = cctz::civil_day(today.year() - 1, 1, 1);
cctz::civil_day prev_first_monday;
if (cctz::get_weekday(prev_jan1) <= cctz::weekday::thursday) {
@@ -251,6 +287,28 @@
}
}
+int DateValue::Iso8601WeekNumberingYear() const {
+ if (UNLIKELY(!IsValid())) return -1;
+ const cctz::civil_day today = EPOCH_DATE + days_since_epoch_;
+ const cctz::civil_day first_monday = GetFirstIso8601MondayOfYear(today.year());
+ const cctz::civil_day last_sunday = GetLastIso8601SundayOfYear(today.year());
+
+ // 0001-01-01 is Monday in the proleptic Gregorian calendar.
+ // 0001-01-01 belongs to year 0001.
+ // 9999-12-31 is Friday in the proleptic Gregorian calendar.
+ // 9999-12-31 belongs to year 9999.
+ if (today >= first_monday && today <= last_sunday) {
+ return today.year();
+ } else if (today > last_sunday) {
+ DCHECK(today.year() >= MIN_YEAR && today.year() < MAX_YEAR);
+ return today.year() + 1;
+ } else {
+ DCHECK(today < first_monday);
+ DCHECK(today.year() > MIN_YEAR && today.year() <= MAX_YEAR);
+ return today.year() - 1;
+ }
+}
+
DateValue DateValue::AddDays(int64_t days) const {
if (UNLIKELY(!IsValid()
|| days < MIN_DAYS_SINCE_EPOCH - days_since_epoch_
diff --git a/be/src/runtime/date-value.h b/be/src/runtime/date-value.h
index 2719466..e7c48f5 100644
--- a/be/src/runtime/date-value.h
+++ b/be/src/runtime/date-value.h
@@ -71,6 +71,15 @@
DateValue(int64_t year, int64_t month, int64_t day);
+ /// Creates a DateValue instance from the ISO 8601 week-based date values:
+ /// 'year' is expected to be in the [1, 9999] range.
+ /// 'week_of_year' is expected to be in the [1, 53] range.
+ /// 'day_of_week' is expected to be in the [1, 7] range.
+ /// If any of the parameters is out of range or 'year' has less than 'week_of_year'
+ /// ISO 8601 weeks, returns an invalid DateValue instance.
+ static DateValue CreateFromIso8601WeekBasedDateVals(int week_numbering_year,
+ int week_of_year, int day_of_week);
+
bool IsValid() const {
return days_since_epoch_ != INVALID_DAYS_SINCE_EPOCH;
}
@@ -99,10 +108,20 @@
/// Otherwise, return -1.
int DayOfYear() const;
- /// Returns the week corresponding to his date. Returned value is in the [1, 53] range.
- /// Weeks start with Monday. Each week's year is the Gregorian year in which the
- /// Thursday falls.
- int WeekOfYear() const;
+ /// Returns the ISO 8601 week corresponding to this date.
+ /// If this DateValue instance is invalid, the return value is -1; otherwise the return
+ /// value is in the [1, 53] range.
+ /// ISO 8601 weeks start with Monday. Each ISO 8601 week's year is the Gregorian year in
+ /// which the Thursday falls.
+ int Iso8601WeekOfYear() const;
+
+ /// Returns the year that the current ISO 8601 week belongs to.
+ /// If this DateValue instance is invalid, the return value is -1; otherwise the return
+ /// value is in the [current_year - 1, current_year + 1] range, which always falls into
+ /// the [1, 9999] range.
+ /// ISO 8601 weeks start with Monday. Each ISO 8601 week's year is the Gregorian year in
+ /// which the Thursday falls.
+ int Iso8601WeekNumberingYear() const;
/// If this DateValue instance valid, add 'days' days to it and return the result.
/// Otherwise, return an invalid DateValue instance.
diff --git a/be/src/runtime/datetime-iso-sql-format-parser.cc b/be/src/runtime/datetime-iso-sql-format-parser.cc
index bd7fb5e..2069256 100644
--- a/be/src/runtime/datetime-iso-sql-format-parser.cc
+++ b/be/src/runtime/datetime-iso-sql-format-parser.cc
@@ -30,11 +30,14 @@
bool IsoSqlFormatParser::ParseDateTime(const char* input_str, int input_len,
const DateTimeFormatContext& dt_ctx, DateTimeParseResult* result) {
DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(dt_ctx.current_time != nullptr);
+ DCHECK(dt_ctx.current_time->HasDateAndTime());
DCHECK(result != nullptr);
DCHECK(result->hour == 0);
if (input_str == nullptr || input_len <= 0) return false;
int day_in_year = -1;
+ Iso8601WeekBasedDateParseResult iso8601_week_based_date_values;
const char* current_pos = input_str;
const char* end_pos = input_str + input_len;
@@ -87,7 +90,8 @@
return false;
}
if (token_len < 4) {
- PrefixYearFromCurrentYear(token_len, dt_ctx.current_time, result);
+ result->year = PrefixYearFromCurrentYear(result->year, token_len,
+ dt_ctx.current_time->date().year());
}
break;
}
@@ -95,9 +99,13 @@
if (!ParseAndValidate(current_pos, token_len, 0, 9999, &result->year)) {
return false;
}
- if (token_len == 2) GetRoundYear(dt_ctx.current_time, result);
+ if (token_len == 2) {
+ result->year = RoundYearFromCurrentYear(result->year,
+ dt_ctx.current_time->date().year());
+ }
if (token_len == 3 || token_len == 1) {
- PrefixYearFromCurrentYear(token_len, dt_ctx.current_time, result);
+ result->year = PrefixYearFromCurrentYear(result->year, token_len,
+ dt_ctx.current_time->date().year());
}
break;
}
@@ -115,6 +123,14 @@
}
break;
}
+ case DAY_NAME:
+ case DAY_NAME_SHORT: {
+ if (!ParseDayNameToken(*tok, current_pos, &token_end_pos, dt_ctx.fx_modifier,
+ &iso8601_week_based_date_values.day_of_week)) {
+ return false;
+ }
+ break;
+ }
case DAY_IN_MONTH: {
if (!ParseAndValidate(current_pos, token_len, 1, 31, &result->day)) return false;
break;
@@ -194,6 +210,32 @@
if (toupper(*current_pos) != toupper(*tok->val)) return false;
break;
}
+ case ISO8601_WEEK_NUMBERING_YEAR: {
+ if (!ParseAndValidate(current_pos, token_len, 0, 9999,
+ &iso8601_week_based_date_values.year)) {
+ return false;
+ }
+ if (token_len < 4) {
+ iso8601_week_based_date_values.year = PrefixYearFromCurrentYear(
+ iso8601_week_based_date_values.year, token_len,
+ GetIso8601WeekNumberingYear(dt_ctx.current_time));
+ }
+ break;
+ }
+ case ISO8601_WEEK_OF_YEAR: {
+ if (!ParseAndValidate(current_pos, token_len, 1, 53,
+ &iso8601_week_based_date_values.week_of_year)) {
+ return false;
+ }
+ break;
+ }
+ case ISO8601_DAY_OF_WEEK: {
+ if (!ParseAndValidate(current_pos, token_len, 1, 7,
+ &iso8601_week_based_date_values.day_of_week)) {
+ return false;
+ }
+ break;
+ }
default: {
return false;
}
@@ -213,6 +255,10 @@
}
}
+ if (iso8601_week_based_date_values.IsSet()) {
+ if (!iso8601_week_based_date_values.ExtractYearMonthDay(result)) return false;
+ }
+
return true;
}
@@ -299,6 +345,10 @@
if (input_len < MAX_MONTH_NAME_LENGTH) return nullptr;
return input_str + MAX_MONTH_NAME_LENGTH;
}
+ if (tok.type == DAY_NAME && fx_provided && !tok.fm_modifier) {
+ if (input_len < MAX_DAY_NAME_LENGTH) return nullptr;
+ return input_str + MAX_DAY_NAME_LENGTH;
+ }
int max_tok_len = min(input_len, tok.len);
const char* start_of_token = input_str;
@@ -335,25 +385,40 @@
return nullptr;
}
-void IsoSqlFormatParser::PrefixYearFromCurrentYear(int actual_token_len,
- const TimestampValue* now, DateTimeParseResult* result) {
- DCHECK(actual_token_len > 0 && actual_token_len < 4);
- DCHECK(now != nullptr);
- DCHECK(result != nullptr);
- int adjust_factor = pow(10, actual_token_len);
- int adjustment = (now->date().year() / adjust_factor) * adjust_factor;
- result->year += adjustment;
+int IsoSqlFormatParser::PrefixYearFromCurrentYear(int year, int year_token_len,
+ int current_year) {
+ DCHECK(year >= 0 && year < 1000);
+ DCHECK(year_token_len > 0 && year_token_len < 4);
+ int adjust_factor = pow(10, year_token_len);
+ int adjustment = (current_year / adjust_factor) * adjust_factor;
+ return year + adjustment;
}
-void IsoSqlFormatParser::GetRoundYear(const TimestampValue* now,
- DateTimeParseResult* result) {
+int IsoSqlFormatParser::RoundYearFromCurrentYear(int year, int current_year) {
+ DCHECK(year >= 0 && year < 100);
+ int postfix_of_curr_year = current_year % 100;
+ if (year < 50 && postfix_of_curr_year > 49) year += 100;
+ if (year > 49 && postfix_of_curr_year < 50) year -= 100;
+ return year + (current_year / 100) * 100;
+}
+
+int IsoSqlFormatParser::GetIso8601WeekNumberingYear(const TimestampValue* now) {
DCHECK(now != nullptr);
+ DCHECK(now->HasDate());
+
+ static const boost::gregorian::date EPOCH(1970, 1, 1);
+ DateValue dv = DateValue((now->date() - EPOCH).days());
+ DCHECK(dv.IsValid());
+ return dv.Iso8601WeekNumberingYear();
+}
+
+bool IsoSqlFormatParser::Iso8601WeekBasedDateParseResult::ExtractYearMonthDay(
+ DateTimeParseResult* result) const {
DCHECK(result != nullptr);
- DCHECK(result->year >= 0 && result->year < 100);
- int postfix_of_curr_year = now->date().year() % 100;
- if (result->year < 50 && postfix_of_curr_year > 49) result->year += 100;
- if (result->year > 49 && postfix_of_curr_year < 50) result->year -= 100;
- result->year += (now->date().year() / 100) * 100;
+ DCHECK(IsSet());
+ DateValue dv = DateValue::CreateFromIso8601WeekBasedDateVals(year, week_of_year,
+ day_of_week);
+ return dv.ToYearMonthDay(&result->year, &result->month, &result->day);
}
}
diff --git a/be/src/runtime/datetime-iso-sql-format-parser.h b/be/src/runtime/datetime-iso-sql-format-parser.h
index 5491abf..3247ab0 100644
--- a/be/src/runtime/datetime-iso-sql-format-parser.h
+++ b/be/src/runtime/datetime-iso-sql-format-parser.h
@@ -40,6 +40,24 @@
WARN_UNUSED_RESULT;
private:
+ // Struct used for capturing ISO 8601 week-based tokens during parsing.
+ struct Iso8601WeekBasedDateParseResult {
+ int year = -1;
+ int week_of_year = -1;
+ int day_of_week = -1;
+
+ bool IsSet() const {
+ return (year != -1 || week_of_year != -1 || day_of_week != -1);
+ }
+
+ /// Calcuates year/month/day date values corresponding to the ISO 8601 week-based
+ /// date values stored in this instance.
+ /// If the ISO 8601 week-based date values are valid, the resulting year/month/day
+ /// values are copied to 'result' and true is returned.
+ /// Otherwise, 'result' is left intact and false is returned.
+ bool ExtractYearMonthDay(DateTimeParseResult* result) const WARN_UNUSED_RESULT;
+ };
+
/// 'input_str' points to a location in the input string where the parsing stands now.
/// Given 'tok' as the next token in the list of tokens created by the tokenizer this
/// function finds the end of the next token.
@@ -60,15 +78,16 @@
static const char* ParseMeridiemIndicatorFromInput(const char* input_str,
int input_len);
- /// If the year part of the input is shorter than 4 digits then prefixes the year with
- /// digits from the current year. Puts the result into 'result->year'.
- static void PrefixYearFromCurrentYear(int actual_token_len, const TimestampValue* now,
- DateTimeParseResult* result);
+ /// Takes 'year' that is shorter than 4 digits and prefixes it with digits from
+ /// 'current_year'. Returns the resulting 4-digit year.
+ static int PrefixYearFromCurrentYear(int year, int year_token_len, int current_year);
- /// Uses 'result->year' as an input. Can call this function for 2-digit years and it
- /// constructs a 4-digit year based on the year provided and the current date. Puts the
- /// result back to 'result->year'.
- static void GetRoundYear(const TimestampValue* now, DateTimeParseResult* result);
+ /// This function takes a 2-digit 'year' and constructs a 4-digit year from it based on
+ /// the 'current_year'. Returns the resulting year.
+ static int RoundYearFromCurrentYear(int year, int current_year);
+
+ /// Returns the ISO 8601 week numbering year corresponding to 'now' date.
+ static int GetIso8601WeekNumberingYear(const TimestampValue* now);
/// Gets a pointer to the current char in the input string and an index to the current
/// token being processed within 'dt_ctx->toks'. Advances these pointers to the end of
diff --git a/be/src/runtime/datetime-iso-sql-format-tokenizer.cc b/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
index 124f03a..b0f4912 100644
--- a/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
+++ b/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
@@ -70,7 +70,13 @@
{"D", IsoSqlFormatTokenizer::TokenItem(DAY_OF_WEEK, true, false)},
{"Q", IsoSqlFormatTokenizer::TokenItem(QUARTER_OF_YEAR, true, false)},
{"WW", IsoSqlFormatTokenizer::TokenItem(WEEK_OF_YEAR, true, false)},
- {"W", IsoSqlFormatTokenizer::TokenItem(WEEK_OF_MONTH, true, false)}
+ {"W", IsoSqlFormatTokenizer::TokenItem(WEEK_OF_MONTH, true, false)},
+ {"IYYY", IsoSqlFormatTokenizer::TokenItem(ISO8601_WEEK_NUMBERING_YEAR, true, false)},
+ {"IYY", IsoSqlFormatTokenizer::TokenItem(ISO8601_WEEK_NUMBERING_YEAR, true, false)},
+ {"IY", IsoSqlFormatTokenizer::TokenItem(ISO8601_WEEK_NUMBERING_YEAR, true, false)},
+ {"I", IsoSqlFormatTokenizer::TokenItem(ISO8601_WEEK_NUMBERING_YEAR, true, false)},
+ {"IW", IsoSqlFormatTokenizer::TokenItem(ISO8601_WEEK_OF_YEAR, true, false)},
+ {"ID", IsoSqlFormatTokenizer::TokenItem(ISO8601_DAY_OF_WEEK, true, false)}
});
const unordered_map<string, int> IsoSqlFormatTokenizer::SPECIAL_LENGTHS({
@@ -187,7 +193,11 @@
short provided_year_count = IsUsedToken("YYYY") + IsUsedToken("YYY") +
IsUsedToken("YY") + IsUsedToken("Y");
short provided_round_year_count = IsUsedToken("RRRR") + IsUsedToken("RR");
- if (provided_year_count > 1 || provided_round_year_count > 1) {
+ short provided_iso8601_week_numbering_year_count = IsUsedToken("IYYY") +
+ IsUsedToken("IYY") + IsUsedToken("IY") + IsUsedToken("I");
+
+ if (provided_year_count > 1 || provided_round_year_count > 1 ||
+ provided_iso8601_week_numbering_year_count > 1) {
return CONFLICTING_YEAR_TOKENS_ERROR;
}
@@ -197,20 +207,20 @@
if (IsUsedToken("Q")) return QUARTER_NOT_ALLOWED_FOR_PARSING;
- short provided_month_tokens = IsUsedToken("MM") + IsUsedToken("MONTH") +
- IsUsedToken("MON");
- if (provided_month_tokens > 1) return CONFLICTING_MONTH_TOKENS_ERROR;
-
if (IsUsedToken("WW") || IsUsedToken("W")) return WEEK_NUMBER_NOT_ALLOWED_FOR_PARSING;
- if (IsUsedToken("DDD") && (IsUsedToken("DD") || provided_month_tokens == 1)) {
+ short provided_month_count = IsUsedToken("MM") + IsUsedToken("MONTH")
+ + IsUsedToken("MON");
+ if (provided_month_count > 1) return CONFLICTING_MONTH_TOKENS_ERROR;
+
+ bool is_used_day_of_year = IsUsedToken("DDD");
+ bool is_used_day_of_month = IsUsedToken("DD");
+ if (is_used_day_of_year && (is_used_day_of_month || provided_month_count == 1)) {
return DAY_OF_YEAR_TOKEN_CONFLICT;
}
if (IsUsedToken("D")) return DAY_OF_WEEK_NOT_ALLOWED_FOR_PARSING;
- if (IsUsedToken("DAY") || IsUsedToken("DY")) return DAY_NAME_NOT_ALLOWED_FOR_PARSING;
-
short provided_hour_tokens = IsUsedToken("HH") + IsUsedToken("HH12") +
IsUsedToken("HH24");
if (provided_hour_tokens > 1) {
@@ -243,6 +253,27 @@
if (provided_fractional_second_count > 1) {
return CONFLICTING_FRACTIONAL_SECOND_TOKENS_ERROR;
}
+
+ short provided_day_of_week_count = IsUsedToken("ID") + IsUsedToken("DAY") +
+ IsUsedToken("DY");
+ if (provided_day_of_week_count > 1) return CONFLICTING_DAY_OF_WEEK_TOKENS_ERROR;
+
+ bool is_used_iso8601_week_of_year = IsUsedToken("IW");
+ if (provided_iso8601_week_numbering_year_count == 1 || is_used_iso8601_week_of_year ||
+ provided_day_of_week_count == 1) {
+ if (provided_year_count == 1 || provided_round_year_count == 1 ||
+ provided_month_count == 1 || is_used_day_of_year || is_used_day_of_month) {
+ if (IsUsedToken("DAY") || IsUsedToken("DY")) {
+ return DAY_NAME_NOT_ALLOWED_FOR_PARSING;
+ }
+ return CONFLICTING_DATE_TOKENS_ERROR;
+ }
+ if (provided_iso8601_week_numbering_year_count == 0 ||
+ !is_used_iso8601_week_of_year || provided_day_of_week_count == 0) {
+ return MISSING_ISO8601_WEEK_BASED_TOKEN_ERROR;
+ }
+ }
+
return SUCCESS;
}
diff --git a/be/src/runtime/datetime-parser-common.cc b/be/src/runtime/datetime-parser-common.cc
index 5ff3430..1cf1474 100644
--- a/be/src/runtime/datetime-parser-common.cc
+++ b/be/src/runtime/datetime-parser-common.cc
@@ -28,6 +28,7 @@
using boost::algorithm::is_any_of;
using std::string;
+using std::unordered_map;
using std::unordered_set;
namespace impala {
@@ -115,7 +116,7 @@
ss << "PARSE ERROR: Time tokens provided with date type.";
break;
case CONFLICTING_FRACTIONAL_SECOND_TOKENS_ERROR:
- ss << "PARSE ERROR: Multiple fractional second token provided.";
+ ss << "PARSE ERROR: Multiple fractional second tokens provided.";
break;
case TEXT_TOKEN_NOT_CLOSED:
ss << "PARSE ERROR: Missing closing quotation mark.";
@@ -125,6 +126,16 @@
break;
case MISPLACED_FX_MODIFIER_ERROR:
ss << "PARSE ERROR: FX modifier should be at the beginning of the format string.";
+ case CONFLICTING_DAY_OF_WEEK_TOKENS_ERROR:
+ ss << "PARSE ERROR: Multiple day of week tokens provided.";
+ break;
+ case MISSING_ISO8601_WEEK_BASED_TOKEN_ERROR:
+ ss << "PARSE ERROR: One or more required ISO 8601 week-based date tokens "
+ "(i.e. IYYY, IW, ID) are missing.";
+ break;
+ case CONFLICTING_DATE_TOKENS_ERROR:
+ ss << "PARSE ERROR: ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) are not "
+ << "allowed to be used with regular date tokens.";
break;
case QUARTER_NOT_ALLOWED_FOR_PARSING:
ss << "PARSE_ERROR: Quarter token is not allowed in a string to datetime "
@@ -136,7 +147,7 @@
break;
case DAY_NAME_NOT_ALLOWED_FOR_PARSING:
ss << "PARSE_ERROR: Day name token is not allowed in a string to datetime "
- "conversion";
+ "conversion except with IYYY|IYY|IY|I and IW tokens";
break;
case WEEK_NUMBER_NOT_ALLOWED_FOR_PARSING:
ss << "PARSE_ERROR: Week number token is not allowed in a string to datetime "
@@ -189,6 +200,67 @@
return (month - 1) / 3 + 1;
}
+namespace {
+// Helper function used by ParseMonthNameToken() and ParseDayNameToken().
+// Parses 'token_start' string and returns true if it finds a valid short or normal name.
+// The valid name prefixes are stored as keys in 'prefix_to_suffix' map. The valid name
+// prefixes are expected to be equal to the corresponding valid short names.
+// The valid name suffixes and the corresponding int IDs are stored as values in
+// 'prefix_to_suffix' map.
+// Returns true iff parsing was successful.
+// If the parsed name in 'token_start' is not followed by a separator then the end of the
+// name is found using 'prefix_to_suffix'. If parsing is successful, '*token_end' is
+// adjusted to point to the character right after the end of the name.
+bool ParseNameTokenHelper(
+ bool is_short_name, int short_name_len, int max_name_len,
+ const char* token_start, const char** token_end,
+ bool is_strict_matching,
+ const unordered_map<string, pair<string, int>>& prefix_to_suffix,
+ int* ret_val) {
+ DCHECK(token_start != nullptr);
+ DCHECK(ret_val != nullptr);
+ DCHECK(token_end != nullptr && *token_end != nullptr);
+ DCHECK(token_start <= *token_end);
+ int token_len = *token_end - token_start;
+ if (token_len < short_name_len) return false;
+
+ string buff(token_start, token_len);
+ boost::to_lower(buff);
+ const string& prefix = buff.substr(0, short_name_len);
+ const auto it = prefix_to_suffix.find(prefix);
+ if (UNLIKELY(it == prefix_to_suffix.end())) return false;
+ if (is_short_name) {
+ DCHECK(token_len == short_name_len);
+ *ret_val = it->second.second;
+ return true;
+ }
+
+ if (is_strict_matching) {
+ DCHECK(buff.length() == max_name_len);
+ trim_right_if(buff, is_any_of(" "));
+ }
+
+ // Check if the remaining characters match.
+ const string& expected_suffix = it->second.first;
+ if (buff.length() - short_name_len < expected_suffix.length()) return false;
+ const char* actual_suffix = buff.c_str() + short_name_len;
+ if (strncmp(actual_suffix, expected_suffix.c_str(), expected_suffix.length()) != 0) {
+ return false;
+ }
+ *ret_val = it->second.second;
+
+ // If the end of the name token wasn't identified because it wasn't followed by a
+ // separator then the end of the name token has to be adjusted.
+ if (prefix.length() + expected_suffix.length() < buff.length()) {
+ if (is_strict_matching) return false;
+ *token_end = token_start + prefix.length() + expected_suffix.length();
+ }
+
+ return true;
+}
+
+}
+
bool ParseMonthNameToken(const DateTimeFormatToken& tok, const char* token_start,
const char** token_end, bool fx_modifier, int* month) {
DCHECK(token_start != nullptr);
@@ -196,42 +268,31 @@
DCHECK(month != nullptr);
DCHECK(token_end != nullptr && *token_end != nullptr);
DCHECK(token_start <= *token_end);
- int token_len = *token_end - token_start;
- if (token_len < SHORT_MONTH_NAME_LENGTH) return false;
- string buff(token_start, token_len);
- boost::to_lower(buff);
- const string& prefix = buff.substr(0, SHORT_MONTH_NAME_LENGTH);
- const auto month_iter = MONTH_PREFIX_TO_SUFFIX.find(prefix);
- if (UNLIKELY(month_iter == MONTH_PREFIX_TO_SUFFIX.end())) return false;
- if (tok.type == MONTH_NAME_SHORT) {
- *month = month_iter->second.second;
- return true;
- }
+ return ParseNameTokenHelper(
+ tok.type == MONTH_NAME_SHORT,
+ SHORT_MONTH_NAME_LENGTH, MAX_MONTH_NAME_LENGTH,
+ token_start, token_end,
+ fx_modifier && !tok.fm_modifier,
+ MONTH_PREFIX_TO_SUFFIX,
+ month);
+}
- DCHECK(tok.type == MONTH_NAME);
- if (fx_modifier && !tok.fm_modifier) {
- DCHECK(buff.length() == MAX_MONTH_NAME_LENGTH);
- trim_right_if(buff, is_any_of(" "));
- }
+bool ParseDayNameToken(const DateTimeFormatToken& tok, const char* token_start,
+ const char** token_end, bool fx_modifier, int* day) {
+ DCHECK(token_start != nullptr);
+ DCHECK(tok.type == DAY_NAME || tok.type == DAY_NAME_SHORT);
+ DCHECK(day != nullptr);
+ DCHECK(token_end != nullptr && *token_end != nullptr);
+ DCHECK(token_start <= *token_end);
- // Check if the remaining characters match.
- const string& expected_suffix = month_iter->second.first;
- if (buff.length() - SHORT_MONTH_NAME_LENGTH < expected_suffix.length()) return false;
- const char* actual_suffix = buff.c_str() + SHORT_MONTH_NAME_LENGTH;
- if (strncmp(actual_suffix, expected_suffix.c_str(), expected_suffix.length()) != 0) {
- return false;
- }
- *month = month_iter->second.second;
-
- // If the end of the month token wasn't identified because it wasn't followed by a
- // separator then the end of the month token has to be adjusted.
- if (prefix.length() + expected_suffix.length() < buff.length()) {
- if (fx_modifier && !tok.fm_modifier) return false;
- *token_end = token_start + prefix.length() + expected_suffix.length();
- }
-
- return true;
+ return ParseNameTokenHelper(
+ tok.type == DAY_NAME_SHORT,
+ SHORT_DAY_NAME_LENGTH, MAX_DAY_NAME_LENGTH,
+ token_start, token_end,
+ fx_modifier && !tok.fm_modifier,
+ DAY_PREFIX_TO_SUFFIX,
+ day);
}
int GetDayInYear(int year, int month, int day_in_month) {
@@ -343,5 +404,13 @@
return (day - 1) / 7 + 1;
}
+int AdjustYearToLength(int year, int len) {
+ if (len < 4) {
+ int adjust_factor = std::pow(10, len);
+ return year % adjust_factor;
+ }
+ return year;
+}
+
}
}
diff --git a/be/src/runtime/datetime-parser-common.h b/be/src/runtime/datetime-parser-common.h
index 7ab1a2b..9cbc58f 100644
--- a/be/src/runtime/datetime-parser-common.h
+++ b/be/src/runtime/datetime-parser-common.h
@@ -73,6 +73,20 @@
{"dec", {"ember", 12}}
};
+/// Similar to 'MONTH_PREFIX_TO_SUFFIX' but maps the 3-letter prefix of a day name to the
+/// suffix of the day name and the ordinal number of the day (1 means Monday and 7 means
+/// Sunday).
+const std::unordered_map<std::string, std::pair<std::string, int>>
+ DAY_PREFIX_TO_SUFFIX = {
+ {"mon", {"day", 1}},
+ {"tue", {"sday", 2}},
+ {"wed", {"nesday", 3}},
+ {"thu", {"rsday", 4}},
+ {"fri", {"day", 5}},
+ {"sat", {"urday", 6}},
+ {"sun", {"day", 7}}
+};
+
/// Length of short month names like 'JAN', 'FEB', etc.
const int SHORT_MONTH_NAME_LENGTH = 3;
@@ -111,7 +125,10 @@
QUARTER_NOT_ALLOWED_FOR_PARSING,
DAY_OF_WEEK_NOT_ALLOWED_FOR_PARSING,
DAY_NAME_NOT_ALLOWED_FOR_PARSING,
- WEEK_NUMBER_NOT_ALLOWED_FOR_PARSING
+ WEEK_NUMBER_NOT_ALLOWED_FOR_PARSING,
+ CONFLICTING_DAY_OF_WEEK_TOKENS_ERROR,
+ MISSING_ISO8601_WEEK_BASED_TOKEN_ERROR,
+ CONFLICTING_DATE_TOKENS_ERROR
};
/// Holds all the token types that serve as building blocks for datetime format patterns.
@@ -145,7 +162,10 @@
DAY_OF_WEEK,
QUARTER_OF_YEAR,
WEEK_OF_YEAR,
- WEEK_OF_MONTH
+ WEEK_OF_MONTH,
+ ISO8601_WEEK_NUMBERING_YEAR,
+ ISO8601_WEEK_OF_YEAR,
+ ISO8601_DAY_OF_WEEK
};
/// Indicates whether the cast is a 'datetime to string' or a 'string to datetime' cast.
@@ -290,6 +310,20 @@
const char** token_end, bool fx_modifier, int* month)
WARN_UNUSED_RESULT;
+/// Gets a day name token (either full or short name) and converts it to the ordinal
+/// number of day between 1 and 7. Make sure 'tok.type' is either DAY_NAME or
+/// DAY_NAME_SHORT.
+/// Result is stored in 'day'. Returns false if the given day name is invalid.
+/// 'fx_modifier' indicates if there is an active FX modifier on the whole format.
+/// If the day part of the input is not followed by a separator then the end of the day
+/// part is found using DAY_PREFIX_TO_SUFFIX. First, the 3 letter prefix of the day name
+/// identifies a particular day and then checks if the rest of the day name matches. If it
+/// does then '*token_end' is adjusted to point to the character right after the end of
+/// the day part.
+bool ParseDayNameToken(const DateTimeFormatToken& tok, const char* token_start,
+ const char** token_end, bool fx_modifier, int* day)
+ WARN_UNUSED_RESULT;
+
inline bool IsLeapYear(int year) {
return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
}
@@ -331,6 +365,9 @@
/// month starts from the first day of the month.
int GetWeekOfMonth(int day);
+/// Returns the year adjusted to 'len' digits.
+/// E.g. AdjustYearToLength(1789, 3) returns 789.
+int AdjustYearToLength(int year, int len);
}
}
diff --git a/be/src/runtime/timestamp-parse-util.cc b/be/src/runtime/timestamp-parse-util.cc
index 93a6039..9c6bac0 100644
--- a/be/src/runtime/timestamp-parse-util.cc
+++ b/be/src/runtime/timestamp-parse-util.cc
@@ -20,9 +20,9 @@
#include "common/names.h"
#include "runtime/datetime-iso-sql-format-parser.h"
#include "runtime/datetime-simple-date-format-parser.h"
+#include "runtime/date-value.h"
#include "runtime/runtime-state.h"
#include "runtime/string-value.inline.h"
-#include "runtime/timestamp-value.h"
#include "udf/udf-internal.h"
#include "util/string-parser.h"
@@ -227,11 +227,7 @@
switch (tok.type) {
case YEAR:
case ROUND_YEAR: {
- num_val = d.year();
- if (tok.len < 4) {
- int adjust_factor = std::pow(10, tok.len);
- num_val %= adjust_factor;
- }
+ num_val = AdjustYearToLength(d.year(), tok.len);
break;
}
case QUARTER_OF_YEAR: {
@@ -307,6 +303,21 @@
result.append(FormatTextToken(tok));
break;
}
+ case ISO8601_WEEK_NUMBERING_YEAR: {
+ num_val = AdjustYearToLength(GetIso8601WeekNumberingYear(d), tok.len);
+ break;
+ }
+ case ISO8601_WEEK_OF_YEAR: {
+ num_val = d.week_number();
+ break;
+ }
+ case ISO8601_DAY_OF_WEEK: {
+ // day_of_week() returns 0 for Sunday, 1 for Monday and 6 for Saturday.
+ num_val = d.day_of_week();
+ // We need to output 1 for Monday and 7 for Sunday.
+ if (num_val == 0) num_val = 7;
+ break;
+ }
default: DCHECK(false) << "Unknown date/time format token";
}
if (num_val > -1) {
@@ -320,4 +331,19 @@
return result;
}
+int TimestampParser::GetIso8601WeekNumberingYear(const boost::gregorian::date& d) {
+ DCHECK(!d.is_special());
+ DCHECK(1400 <= d.year() && d.year() <= 9999);
+
+ static const boost::gregorian::date epoch(1970, 1, 1);
+ DateValue dv((d - epoch).days());
+ DCHECK(dv.IsValid());
+
+ int week_numbering_year = dv.Iso8601WeekNumberingYear();
+ // 1400.01.01 is Wednesday. 9999.12.31 is Friday.
+ // This means that week_numbering_year must fall in the [1400, 9999] range.
+ DCHECK(1400 <= week_numbering_year && week_numbering_year <= 9999);
+ return week_numbering_year;
+}
+
}
diff --git a/be/src/runtime/timestamp-parse-util.h b/be/src/runtime/timestamp-parse-util.h
index e9b00a4..ca71836 100644
--- a/be/src/runtime/timestamp-parse-util.h
+++ b/be/src/runtime/timestamp-parse-util.h
@@ -102,6 +102,14 @@
/// to 24 or zero otherwise.
static int AdjustWithTimezone(boost::posix_time::time_duration* t,
const boost::posix_time::time_duration& tz_offset);
+
+ /// Returns the year that the current week belongs to. Returned value is in the
+ /// [d.year() - 1, d.year() + 1] range.
+ /// Weeks start with Monday. Each week's year is the Gregorian year in which the
+ /// Thursday falls.
+ /// 'd' date is expected to fall in the [1400, 9999] year range. The returned week
+ /// numbering year must also fall in the [1400, 9999] range.
+ static int GetIso8601WeekNumberingYear(const boost::gregorian::date& d);
};
}
diff --git a/tests/query_test/test_cast_with_format.py b/tests/query_test/test_cast_with_format.py
index 7cbae34..f584bf6 100644
--- a/tests/query_test/test_cast_with_format.py
+++ b/tests/query_test/test_cast_with_format.py
@@ -154,7 +154,7 @@
"timestamp FORMAT 'YYYY-MM-DDTHH24:MI:SSZ')")
assert result.data == ["2018-11-10 15:11:04"]
- # ISO8601 format elements are case-insensitive
+ # ISO 8601 format elements are case-insensitive
result = self.client.execute("select cast('2018-11-09t15:11:04Z' as "
"timestamp FORMAT 'YYYY-MM-DDTHH24:MI:SSz')")
assert result.data == ["2018-11-09 15:11:04"]
@@ -803,7 +803,70 @@
assert result.data == ["123"]
def test_day_name(self):
- # Different lowercase and uppercase scenarios.
+ # String to datetime: Test different lowercase vs uppercase scenarios.
+ result = self.execute_query(
+ "select cast('2010-08-Tuesday' as timestamp FORMAT 'IYYY-IW-DAY'), "
+ " cast('2010-monday-08' as timestamp FORMAT 'IYYY-DAY-IW'), "
+ " cast('2010-Wednesday-08' as date FORMAT 'IYYY-DAY-IW'), "
+ " cast('2010 08 THURSDAY' as timestamp FORMAT 'IYYY IW DAY'), "
+ " cast('2010 08 Friday' as date FORMAT 'IYYY IW DAY'), "
+ " cast('2010 08 saturday' as timestamp FORMAT 'IYYY IW DAY'), "
+ " cast('sUnDay 2010 08' as date FORMAT 'DAY IYYY IW'), "
+ " cast('Monday 2010 09' as date FORMAT 'DAY IYYY IW')")
+ assert result.data == [
+ "2010-02-23 00:00:00\t2010-02-22 00:00:00\t2010-02-24\t"
+ "2010-02-25 00:00:00\t2010-02-26\t2010-02-27 00:00:00\t"
+ "2010-02-28\t2010-03-01"]
+ # And now with short day names.
+ result = self.execute_query(
+ "select cast('2010-08-Tue' as timestamp FORMAT 'IYYY-IW-DY'), "
+ " cast('2010-mon-08' as timestamp FORMAT 'IYYY-DY-IW'), "
+ " cast('2010-Wed-08' as date FORMAT 'IYYY-DY-IW'), "
+ " cast('2010 08 THU' as timestamp FORMAT 'IYYY IW DY'), "
+ " cast('2010 08 Fri' as date FORMAT 'IYYY IW DY'), "
+ " cast('2010 08 sat' as timestamp FORMAT 'IYYY IW DY'), "
+ " cast('sUn 2010 08' as date FORMAT 'DY IYYY IW'), "
+ " cast('Mon 2010 09' as date FORMAT 'DY IYYY IW')")
+ assert result.data == [
+ "2010-02-23 00:00:00\t2010-02-22 00:00:00\t2010-02-24\t"
+ "2010-02-25 00:00:00\t2010-02-26\t2010-02-27 00:00:00\t"
+ "2010-02-28\t2010-03-01"]
+
+ # String to datetime: Incorrect day name.
+ result = self.execute_query("select cast('2010 09 Mondau' as timestamp FORMAT "
+ "'IYYY IW DAY')")
+ assert result.data == ["NULL"]
+
+ # String to datetime: DAY token without surrounding separators.
+ result = self.execute_query(
+ "select cast('2010MONDAY09' as date FORMAT 'IYYYDAYIW'), "
+ " cast('2010WEDNESDAY9' as timestamp FORMAT 'IYYYDAYIW')")
+ assert result.data == ["2010-03-01\t2010-03-03 00:00:00"]
+ # And now with short day names.
+ result = self.execute_query(
+ "select cast('2010MON09' as date FORMAT 'IYYYDYIW'), "
+ " cast('2010WED9' as timestamp FORMAT 'IYYYDYIW')")
+ assert result.data == ["2010-03-01\t2010-03-03 00:00:00"]
+
+ # String to datetime: FX and FM modifiers.
+ result = self.execute_query(
+ "select cast('2010-Monday-09' as timestamp FORMAT 'FXIYYY-DAY-IW'), "
+ " cast('2010-Monday X-09' as timestamp FORMAT 'FXIYYY-DAY-IW')")
+ assert result.data == ["NULL\tNULL"]
+
+ result = self.execute_query(
+ "select cast('2010-Monday -09' as timestamp FORMAT 'FXIYYY-DAY-IW'), "
+ " cast('2010-Monday 09' as date FORMAT 'FXIYYY-DAYIW')")
+ assert result.data == ["2010-03-01 00:00:00\t2010-03-01"]
+
+ result = self.execute_query(
+ "select cast('2010-Monday-09' as timestamp FORMAT 'FXIYYY-FMDAY-IW'), "
+ " cast('2010-Monday09' as timestamp FORMAT 'FXIYYY-FMDAYIW'), "
+ " cast('2010Monday09' as date FORMAT 'FXIYYYFMDAYIW')")
+ assert result.data == ["2010-03-01 00:00:00\t2010-03-01 00:00:00\t"
+ "2010-03-01"]
+
+ # Datetime to string: Different lowercase and uppercase scenarios.
result = self.execute_query("select cast(date'2019-11-13' as string "
"format 'DAY Day day DY Dy dy')")
assert result.data == ["WEDNESDAY Wednesday wednesday WED Wed wed"]
@@ -832,7 +895,7 @@
"format 'DAY Day day DY Dy dy')")
assert result.data == ["TUESDAY Tuesday tuesday TUE Tue tue"]
- # Different lowercase and uppercase scenarios when FM is provided.
+ # Datetime to string: Different lowercase and uppercase scenarios when FM is provided.
result = self.execute_query("select cast(cast('2019-11-13' as timestamp) as string "
"format 'FMDAY FMDay FMday FMDY FMDy FMdy')")
assert result.data == ["WEDNESDAY Wednesday wednesday WED Wed wed"]
@@ -861,12 +924,12 @@
"format 'FMDAY FMDay FMday FMDY FMDy FMdy')")
assert result.data == ["TUESDAY Tuesday tuesday TUE Tue tue"]
- # Test odd casing of day token.
+ # Datetime to string: Test odd casing of day token.
result = self.execute_query("select cast(date'2010-01-20' as string FORMAT "
"'DAy dAY daY dY')")
assert result.data == ["WEDNESDAY wednesday wednesday wed"]
- # Day token without surrounding separators
+ # Datetime to string: Day token without surrounding separators.
result = self.execute_query("select cast(date'2019-11-11' as string "
"format 'YYYYDayMonth')")
assert result.data == ["2019Monday November "]
@@ -883,7 +946,7 @@
"format 'YYYYDYDD')")
assert result.data == ["2019TUE12"]
- # Day token with FM and FX modifiers.
+ # Datetime to string: Day token with FM and FX modifiers.
result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
"format 'FXYYYY DAY DD')")
assert result.data == ["2019 TUESDAY 01"]
@@ -1388,6 +1451,143 @@
r''' 'FXYYYY-"\\\'"-MM-DD') ''')
assert result.data == ["2010-02-01"]
+ def test_iso8601_week_based_date_tokens(self):
+ # Format 0001-01-01 and 9999-12-31 dates.
+ # 0001-01-01 is Monday, belongs to the 1st week of year 1.
+ # 9999-12-31 is Friday, belongs to the 52nd week of year 9999.
+ result = self.client.execute(
+ "select cast(date'0001-01-01' as string format 'IYYY/IW/ID'), "
+ " cast(date'9999-12-31' as string format 'IYYY/IW/ID')")
+ assert result.data == ["0001/01/01\t9999/52/05"]
+
+ # Parse 0001-01-01 and 9999-12-31 dates.
+ result = self.client.execute(
+ "select cast('0001/01/01' as date format 'IYYY/IW/ID'), "
+ " cast('9999/52/05' as date format 'IYYY/IW/ID')")
+ assert result.data == ["0001-01-01\t9999-12-31"]
+
+ # Parse out-of-range dates.
+ # Year 9999 has 52 weeks. 9999-12-31 is Friday.
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('9999/52/06' as date format 'IYYY/IW/ID')")
+ assert 'String to Date parse failed. Invalid string val: "9999/52/06"' in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('9999/53/01' as date format 'IYYY/IW/ID')")
+ assert 'String to Date parse failed. Invalid string val: "9999/53/01"' in str(err)
+
+ # Format 1400-01-01 and 9999-12-31 timestamps.
+ # 1400-01-01 is Wednesday, belongs to the 1st week of year 1400.
+ # 9999-12-31 is Friday, belongs to the 52nd week of year 9999.
+ result = self.client.execute(
+ "select cast(cast('1400-01-01' as timestamp) as string format 'IYYY/IW/ID'), "
+ " cast(cast('9999-12-31' as timestamp) as string format 'IYYY/IW/ID')")
+ assert result.data == ["1400/01/03\t9999/52/05"]
+
+ # Parse 1400-01-01 and 9999-12-31 timestamps.
+ result = self.client.execute(
+ "select cast('1400/01/03' as timestamp format 'IYYY/IW/ID'), "
+ " cast('9999/52/05' as timestamp format 'IYYY/IW/ID')")
+ assert result.data == ["1400-01-01 00:00:00\t9999-12-31 00:00:00"]
+
+ # Parse out-of-range timestamps.
+ # - Tuesday of the 1st week of year 1400 is 1399-12-31, which is out of the valid
+ # timestamp range.
+ # - Year 9999 has 52 weeks. 9999-12-31 is Friday.
+ result = self.client.execute(
+ "select cast('1400/01/02' as timestamp format 'IYYY/IW/ID'), "
+ " cast('9999/52/06' as timestamp format 'IYYY/IW/ID'), "
+ " cast('9999/53/01' as timestamp format 'IYYY/IW/ID')")
+ assert result.data == ["NULL\tNULL\tNULL"]
+
+ # Formatting dates arond Dec 31.
+ # 2019-12-31 is Tuesday, belongs to 1st week of year 2020.
+ # 2020-12-31 is Thursday, belongs to 53rd week of year 2020.
+ result = self.client.execute(
+ "select cast(date'2019-12-29' as string format 'IYYY/IW/ID'), "
+ " cast(date'2019-12-30' as string format 'IYYY/IW/ID'), "
+ " cast(date'2019-12-31' as string format 'IYYY/IW/ID'), "
+ " cast(date'2020-01-01' as string format 'IYYY/IW/ID'), "
+ " cast(date'2020-12-31' as string format 'IYYY/IW/ID'), "
+ " cast(date'2021-01-01' as string format 'IYYY/IW/ID')")
+ assert result.data == [
+ "2019/52/07\t2020/01/01\t2020/01/02\t2020/01/03\t2020/53/04\t2020/53/05"]
+
+ # Parsing dates around Dec 31.
+ result = self.client.execute(
+ "select cast('2019/52/07' as date format 'IYYY/IW/ID'), "
+ " cast('2020/01/01' as date format 'IYYY/IW/ID'), "
+ " cast('2020/01/02' as date format 'IYYY/IW/ID'), "
+ " cast('2020/01/03' as date format 'IYYY/IW/ID'), "
+ " cast('2020/53/04' as date format 'IYYY/IW/ID'), "
+ " cast('2020/53/05' as date format 'IYYY/IW/ID')")
+ assert result.data == [
+ "2019-12-29\t2019-12-30\t2019-12-31\t2020-01-01\t2020-12-31\t2021-01-01"]
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2019/53/01' as date format 'IYYY/IW/ID')")
+ assert 'String to Date parse failed. Invalid string val: "2019/53/01"' in str(err)
+
+ # Format 4, 3, 2, 1-digit week numbering year.
+ # 2020-01-01 is Wednesday, belongs to week 1 of year 2020.
+ query_options = dict({'now_string': '2019-01-01 11:11:11'})
+ result = self.execute_query(
+ "select cast(date'2020-01-01' as string format 'IYYY/IW/ID'), "
+ " cast(date'2020-01-01' as string format 'IYY/IW/ID'), "
+ " cast(date'2020-01-01' as string format 'IY/IW/ID'), "
+ " cast(date'2020-01-01' as string format 'I/IW/ID')", query_options)
+ assert result.data == ["2020/01/03\t020/01/03\t20/01/03\t0/01/03"]
+
+ # Parse 4, 3, 2, 1-digit week numbering year.
+ result = self.execute_query(
+ "select cast('2020/01/03' as date format 'IYYY/IW/ID'), "
+ " cast('020/01/03' as date format 'IYYY/IW/ID'), "
+ " cast('20/01/03' as date format 'IYYY/IW/ID'), "
+ " cast('0/01/03' as date format 'IYYY/IW/ID'), "
+ " cast('020/01/03' as date format 'IYY/IW/ID'), "
+ " cast('20/01/03' as date format 'IYY/IW/ID'), "
+ " cast('0/01/03' as date format 'IYY/IW/ID'), "
+ " cast('20/01/03' as date format 'IY/IW/ID'), "
+ " cast('0/01/03' as date format 'IY/IW/ID'), "
+ " cast('0/01/03' as date format 'I/IW/ID')", query_options)
+ assert result.data == ['2020-01-01\t2020-01-01\t2020-01-01\t2010-01-06\t'
+ '2020-01-01\t2020-01-01\t2010-01-06\t'
+ '2020-01-01\t2010-01-06\t'
+ '2010-01-06']
+
+ # 2000-01-01 is Saturday, so it belongs to the 1999 ISO 8601 week-numbering year.
+ # Test that 1999 is used for prefixing 3, 2, 1-digit week numbering year.
+ query_options = dict({'now_string': '2000-01-01 11:11:11'})
+ result = self.execute_query(
+ "select cast('2005/01/01' as date format 'IYYY/IW/ID'), "
+ " cast('005/01/01' as date format 'IYYY/IW/ID'), "
+ " cast('05/01/01' as date format 'IYYY/IW/ID'), "
+ " cast('5/01/01' as date format 'IYYY/IW/ID'), "
+ " cast('05/01/01' as date format 'IY/IW/ID'), "
+ " cast('5/01/01' as date format 'IY/IW/ID'), "
+ " cast('5/01/01' as date format 'I/IW/ID')", query_options)
+ assert result.data == ['2005-01-03\t1004-12-31\t1905-01-02\t1995-01-02\t'
+ '1905-01-02\t1995-01-02\t'
+ '1995-01-02']
+
+ # Parse 1-digit week of year and 1-digit week day.
+ result = self.client.execute(
+ "select cast('2020/53/4' as date format 'IYYY/IW/ID'), "
+ " cast('2020/1/3' as date format 'IYYY/IW/ID')")
+ assert result.data == ["2020-12-31\t2020-01-01"]
+
+ # Parse dayname with week-based tokens
+ result = self.client.execute(
+ "select cast('2020/wed/1' as date format 'IYYY/DY/IW'), "
+ " cast('2020/wed1' as date format 'iyyy/dyiw'), "
+ " cast('2020wed1' as date format 'IYYYDYIW'), "
+ " cast('2020WEd1' as date format 'iyyydyiw'), "
+ " cast('2020/wednesday/1' as date format 'IYYY/DAY/IW'), "
+ " cast('2020/wednesday1' as date format 'iyyy/dayiw'), "
+ " cast('2020wednesday1' as date format 'IYYYDAYIW'), "
+ " cast('2020wEdnESday1' as date format 'iyyydayiw')")
+ assert result.data == ["2020-01-01\t2020-01-01\t2020-01-01\t2020-01-01\t"
+ "2020-01-01\t2020-01-01\t2020-01-01\t2020-01-01"]
+
def test_fm_fx_modifiers(self):
# Exact mathcing for the whole format.
result = self.client.execute("select cast('2001-03-01 03:10:15.123456 -01:30' as "
@@ -1456,6 +1656,32 @@
"format 'FXYYYY-MM-DD HH12:MI:SS.FF')")
assert result.data == ["NULL"]
+ # Strict week-based token length matching.
+ result = self.client.execute(
+ "select cast('2015/3/05' as timestamp format 'FXIYYY/IW/ID'), "
+ " cast('2015/03/5' as timestamp format 'FXIYYY/IW/ID'), "
+ " cast('015/03/05' as timestamp format 'FXIYYY/IW/ID'), "
+ " cast('15/03/05' as timestamp format 'FXIYYY/IW/ID'), "
+ " cast('5/03/05' as timestamp format 'FXIYYY/IW/ID')")
+ assert result.data == ["NULL\tNULL\tNULL\tNULL\tNULL"]
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2015/3/05' as date format 'FXIYYY/IW/ID')")
+ assert 'String to Date parse failed. Invalid string val: "2015/3/05"' in str(err)
+
+ query_options = dict({'now_string': '2019-01-01 11:11:11'})
+ result = self.execute_query(
+ "select cast('2015/03/05' as timestamp format 'FXIYYY/IW/ID'), "
+ " cast('015/03/05' as timestamp format 'FXIYY/IW/ID'), "
+ " cast('15/03/05' as timestamp format 'FXIY/IW/ID'), "
+ " cast('5/03/05' as timestamp format 'FXI/IW/ID'), "
+ " cast('2015/03/05' as date format 'FXIYYY/IW/ID'), "
+ " cast('015/03/05' as date format 'FXIYY/IW/ID'), "
+ " cast('15/03/05' as date format 'FXIY/IW/ID'), "
+ " cast('5/03/05' as date format 'FXI/IW/ID')", query_options)
+ assert result.data == ["2015-01-16 00:00:00\t2015-01-16 00:00:00\t"
+ "2015-01-16 00:00:00\t2015-01-16 00:00:00\t"
+ "2015-01-16\t2015-01-16\t2015-01-16\t2015-01-16"]
+
# Strict token length matching with text token containing escaped double quote.
result = self.client.execute(r'''select cast('2001-03-09 some "text03:25:00' '''
r'''as timestamp format "FXYYYY-MM-DD \"some \\\"text\"HH12:MI:SS")''')
@@ -1514,6 +1740,11 @@
"timestamp) as string format 'FXYYYY-MM-DD HH24:MI:SS.FF7')")
assert result.data == ["2001-03-05 03:10:15.1234560"]
+ result = self.client.execute(
+ "select cast(date'0001-01-10' as string format 'FXIYYY-IW-ID'), "
+ " cast(date'0001-10-10' as string format 'FXIYYY-IW-ID')")
+ assert result.data == ["0001-02-03\t0001-41-03"]
+
# Datetime to string path: Tokens with FM modifier don't pad output to a given
# length.
result = self.client.execute("select cast(cast('2001-03-14 03:06:08' as timestamp) "
@@ -1528,6 +1759,11 @@
"'FMYY-FMMM-FMDD')")
assert result.data == ["1-3-10"]
+ result = self.client.execute(
+ "select cast(date'0001-01-10' as string format 'FMIYYY-FMIW-FMID'), "
+ " cast(date'0001-10-10' as string format 'FMIYYY-FMIW-FMID')")
+ assert result.data == ["1-2-3\t1-41-3"]
+
# Datetime to string path: FM modifier is effective even if FX modifier is also
# given.
result = self.client.execute("select cast(cast('2001-03-15 03:06:08' as "
@@ -1542,11 +1778,17 @@
"as string format 'FXFMYYYY-FMMM-FMDD')")
assert result.data == ["1-4-10"]
+ result = self.client.execute(
+ "select cast(date'0001-01-10' as string format 'FXFMIYYY-FMIW-FMID'), "
+ " cast(date'0001-10-10' as string format 'FXFMIYYY-FMIW-FMID')")
+ assert result.data == ["1-2-3\t1-41-3"]
+
# FX and FM modifiers are case-insensitive.
result = self.client.execute("select cast('2019-5-10' as date format "
"'fxYYYY-fmMM-DD')")
assert result.data == ["2019-05-10"]
+
def test_quarter(self):
result = self.client.execute("select cast(date'2001-01-01' as string "
"FORMAT 'YYYY Q MM')")
@@ -1636,6 +1878,19 @@
"select cast('2017-05-01' as timestamp format 'YYYY-MONTH-DD-MON')")
assert "Multiple month tokens provided" in str(err)
+ # Conflict between DAY, DY and ID tokens.
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01-Monday' as timestamp format 'IYYY-IW-ID-DAY')")
+ assert "Multiple day of week tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01-Mon' as timestamp format 'IYYY-IW-ID-DY')")
+ assert "Multiple day of week tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-Monday-Mon' as timestamp format 'IYYY-IW-DAY-DY')")
+ assert "Multiple day of week tokens provided" in str(err)
+
# Week of year token not allowed in a string to datetime conversion.
err = self.execute_query_expect_failure(self.client,
"select cast('2017-1-01' as timestamp format 'YYYY-WW-DD')")
@@ -1674,8 +1929,8 @@
# Day name token not allowed in a string to datetime conversion.
err = self.execute_query_expect_failure(self.client,
"select cast('2017-1-02 Monday' as timestamp format 'YYYY-DD-MM DAY')")
- assert "Day name token is not allowed in a string to datetime conversion" in \
- str(err)
+ assert "Day name token is not allowed in a string to datetime conversion except " \
+ "with IYYY|IYY|IY|I and IW tokens" in str(err)
# Conflict between hour tokens
err = self.execute_query_expect_failure(self.client,
@@ -1764,23 +2019,77 @@
# Multiple fraction second token conflict
err = self.execute_query_expect_failure(self.client,
"select cast('2018-10-10' as timestamp format 'FF FF1')")
- assert "Multiple fractional second token provided." in str(err)
+ assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
"select cast('2018-10-10' as timestamp format 'FF2 FF3')")
- assert "Multiple fractional second token provided." in str(err)
+ assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
"select cast('2018-10-10' as timestamp format 'FF4 FF5')")
- assert "Multiple fractional second token provided." in str(err)
+ assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
"select cast('2018-10-10' as timestamp format 'FF6 FF7')")
- assert "Multiple fractional second token provided." in str(err)
+ assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
"select cast('2018-10-10' as timestamp format 'FF8 FF9')")
- assert "Multiple fractional second token provided." in str(err)
+ assert "Multiple fractional second tokens provided." in str(err)
+
+ # ISO 8601 Week-based and normal date pattern tokens must not be mixed.
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-01' as date format 'IYYY-MM-ID')")
+ assert "ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) are not allowed to be " \
+ "used with regular date tokens." in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-01 01:00' as timestamp format 'IYYY-MM-ID HH24:MI')")
+ assert "ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) are not allowed to be " \
+ "used with regular date tokens." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-01' as date format 'YYYY-IW-DD')")
+ assert "ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) are not allowed to be " \
+ "used with regular date tokens." in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-01' as timestamp format 'IYYY-IW-DD')")
+ assert "ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) are not allowed to be " \
+ "used with regular date tokens." in str(err)
+
+ # Missing ISO 8601 week-based pattern tokens.
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10' as date format 'IYYY-IW')")
+ assert "One or more required ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) " \
+ "are missing." in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10 01:00' as timestamp format 'IYYY-IW HH24:MI')")
+ assert "One or more required ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) " \
+ "are missing." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('18-07' as date format 'IY-ID')")
+ assert "One or more required ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) " \
+ "are missing." in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('18-07 01:00' as timestamp format 'IY-ID HH24:MI')")
+ assert "One or more required ISO 8601 week-based date tokens (i.e. IYYY, IW, ID) " \
+ "are missing." in str(err)
+
+ # ISO 8601 Week numbering year conflict
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-018-10-01' as date format 'IYYY-IYY-IW-DD')")
+ assert "Multiple year tokens provided" in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-018-10-01 01:00' as timestamp format "
+ "'IYYY-IYY-IW-DD HH24:MI')")
+ assert "Multiple year tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('018-8-10-01' as date format 'IYY-I-IW-DD')")
+ assert "Multiple year tokens provided" in str(err)
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('018-8-10-01 01:00' as timestamp format 'IYY-I-IW-DD HH24:MI')")
+ assert "Multiple year tokens provided" in str(err)
# Verify that conflict check is not skipped when format ends with separators.
err = self.execute_query_expect_failure(self.client,
