| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package array_test |
| |
| import ( |
| "strings" |
| "testing" |
| "time" |
| |
| "github.com/apache/arrow-go/v18/arrow" |
| "github.com/apache/arrow-go/v18/arrow/array" |
| "github.com/apache/arrow-go/v18/arrow/memory" |
| "github.com/apache/arrow-go/v18/internal/json" |
| "github.com/stretchr/testify/assert" |
| "github.com/stretchr/testify/require" |
| ) |
| |
| const ( |
| // We test against the deprecated layout for backwards compatibility |
| // See https://github.com/apache/arrow-go/pull/450 |
| deprecatedLayout = "2006-01-02 15:04:05.999999999Z0700" |
| ) |
| |
| func TestTimestampStringRoundTrip(t *testing.T) { |
| // 1. create array |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dt := &arrow.TimestampType{Unit: arrow.Second} |
| b := array.NewTimestampBuilder(mem, dt) |
| defer b.Release() |
| |
| b.Append(1) |
| b.Append(2) |
| b.Append(3) |
| b.AppendNull() |
| b.Append(5) |
| b.Append(6) |
| b.AppendNull() |
| b.Append(8) |
| b.Append(9) |
| b.Append(10) |
| |
| arr := b.NewArray().(*array.Timestamp) |
| defer arr.Release() |
| |
| // 2. create array via AppendValueFromString |
| b1 := array.NewTimestampBuilder(mem, dt) |
| defer b1.Release() |
| |
| for i := 0; i < arr.Len(); i++ { |
| assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i))) |
| } |
| |
| arr1 := b1.NewArray().(*array.Timestamp) |
| defer arr1.Release() |
| |
| assert.True(t, array.Equal(arr, arr1)) |
| } |
| |
| func TestNewTimestampBuilder(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| timestamp := time.Now() |
| dtype := &arrow.TimestampType{Unit: arrow.Second} |
| ab := array.NewTimestampBuilder(mem, dtype) |
| defer ab.Release() |
| |
| ab.Retain() |
| ab.Release() |
| |
| ab.Append(1) |
| ab.Append(2) |
| ab.Append(3) |
| ab.AppendNull() |
| ab.Append(5) |
| ab.Append(6) |
| ab.AppendNull() |
| ab.Append(8) |
| ab.Append(9) |
| ab.Append(10) |
| ab.AppendTime(timestamp) |
| |
| // check state of builder before NewTimestampArray |
| assert.Equal(t, 11, ab.Len(), "unexpected Len()") |
| assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") |
| |
| a := ab.NewTimestampArray() |
| |
| // check state of builder after NewTimestampArray |
| assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewTimestampArray did not reset state") |
| assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewTimestampArray did not reset state") |
| assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewTimestampArray did not reset state") |
| |
| // check state of array |
| assert.Equal(t, 2, a.NullN(), "unexpected null count") |
| assert.Equal(t, []arrow.Timestamp{1, 2, 3, 0, 5, 6, 0, 8, 9, 10, arrow.Timestamp(timestamp.Unix())}, a.TimestampValues(), "unexpected TimestampValues") |
| assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity |
| assert.Len(t, a.TimestampValues(), 11, "unexpected length of TimestampValues") |
| |
| a.Release() |
| |
| ab.Append(7) |
| ab.Append(8) |
| |
| a = ab.NewTimestampArray() |
| |
| assert.Equal(t, 0, a.NullN()) |
| assert.Equal(t, []arrow.Timestamp{7, 8}, a.TimestampValues()) |
| assert.Len(t, a.TimestampValues(), 2) |
| |
| a.Release() |
| |
| var ( |
| want = []arrow.Timestamp{1, 2, 3, 4} |
| valids = []bool{true, true, false, true} |
| ) |
| |
| ab.AppendValues(want, valids) |
| a = ab.NewTimestampArray() |
| |
| sub := array.MakeFromData(a.Data()) |
| defer sub.Release() |
| |
| if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { |
| t.Fatalf("invalid type: got=%q, want=%q", got, want) |
| } |
| |
| if _, ok := sub.(*array.Timestamp); !ok { |
| t.Fatalf("could not type-assert to array.Timestamp") |
| } |
| |
| if got, want := a.String(), `[1 2 (null) 4]`; got != want { |
| t.Fatalf("got=%q, want=%q", got, want) |
| } |
| |
| slice := array.NewSliceData(a.Data(), 2, 4) |
| defer slice.Release() |
| |
| sub1 := array.MakeFromData(slice) |
| defer sub1.Release() |
| |
| v, ok := sub1.(*array.Timestamp) |
| if !ok { |
| t.Fatalf("could not type-assert to array.Timestamp") |
| } |
| |
| if got, want := v.String(), `[(null) 4]`; got != want { |
| t.Fatalf("got=%q, want=%q", got, want) |
| } |
| |
| a.Release() |
| } |
| |
| func TestTimestampBuilder_AppendValues(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dtype := &arrow.TimestampType{Unit: arrow.Second} |
| ab := array.NewTimestampBuilder(mem, dtype) |
| defer ab.Release() |
| |
| exp := []arrow.Timestamp{0, 1, 2, 3} |
| ab.AppendValues(exp, nil) |
| a := ab.NewTimestampArray() |
| assert.Equal(t, exp, a.TimestampValues()) |
| |
| a.Release() |
| } |
| |
| func TestTimestampBuilder_Empty(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dtype := &arrow.TimestampType{Unit: arrow.Second} |
| ab := array.NewTimestampBuilder(mem, dtype) |
| defer ab.Release() |
| |
| exp := []arrow.Timestamp{0, 1, 2, 3} |
| |
| ab.AppendValues([]arrow.Timestamp{}, nil) |
| a := ab.NewTimestampArray() |
| assert.Zero(t, a.Len()) |
| a.Release() |
| |
| ab.AppendValues(nil, nil) |
| a = ab.NewTimestampArray() |
| assert.Zero(t, a.Len()) |
| a.Release() |
| |
| ab.AppendValues([]arrow.Timestamp{}, nil) |
| ab.AppendValues(exp, nil) |
| a = ab.NewTimestampArray() |
| assert.Equal(t, exp, a.TimestampValues()) |
| a.Release() |
| |
| ab.AppendValues(exp, nil) |
| ab.AppendValues([]arrow.Timestamp{}, nil) |
| a = ab.NewTimestampArray() |
| assert.Equal(t, exp, a.TimestampValues()) |
| a.Release() |
| } |
| |
| func TestTimestampBuilder_Resize(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dtype := &arrow.TimestampType{Unit: arrow.Second} |
| ab := array.NewTimestampBuilder(mem, dtype) |
| defer ab.Release() |
| |
| assert.Equal(t, 0, ab.Cap()) |
| assert.Equal(t, 0, ab.Len()) |
| |
| ab.Reserve(63) |
| assert.Equal(t, 64, ab.Cap()) |
| assert.Equal(t, 0, ab.Len()) |
| |
| for i := 0; i < 63; i++ { |
| ab.Append(0) |
| } |
| assert.Equal(t, 64, ab.Cap()) |
| assert.Equal(t, 63, ab.Len()) |
| |
| ab.Resize(5) |
| assert.Equal(t, 5, ab.Len()) |
| |
| ab.Resize(32) |
| assert.Equal(t, 5, ab.Len()) |
| } |
| |
| func TestTimestampValueStr(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: "America/Phoenix"} |
| b := array.NewTimestampBuilder(mem, dt) |
| defer b.Release() |
| |
| b.Append(-34226955) |
| b.Append(1456767743) |
| |
| arr := b.NewArray() |
| defer arr.Release() |
| |
| assert.Equal(t, "1968-11-30T13:30:45-07:00", arr.ValueStr(0)) |
| assert.Equal(t, "2016-02-29T10:42:23-07:00", arr.ValueStr(1)) |
| } |
| |
| func TestTimestampBuilderUnmarshalRejectsInvalidTimezone(t *testing.T) { |
| for _, tc := range []struct { |
| name string |
| value string |
| useNumber bool |
| }{ |
| {name: "timestamp string", value: `"2024-01-01T00:00:00"`}, |
| {name: "integer string", value: `"0"`}, |
| {name: "json number", value: `0`, useNumber: true}, |
| {name: "float64", value: `0`}, |
| } { |
| t.Run(tc.name, func(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: "not/a_timezone"} |
| b := array.NewTimestampBuilder(mem, dt) |
| defer b.Release() |
| |
| dec := json.NewDecoder(strings.NewReader(tc.value)) |
| if tc.useNumber { |
| dec.UseNumber() |
| } |
| err := b.UnmarshalOne(dec) |
| assert.ErrorContains(t, err, "could not find timezone location") |
| assert.Zero(t, b.Len()) |
| }) |
| } |
| } |
| |
| func TestTimestampValueStrWithDeprecatedLayout(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: "America/Phoenix"} |
| b := array.NewTimestampBuilderWithValueStrLayout(mem, dt, deprecatedLayout) |
| defer b.Release() |
| |
| b.Append(-34226955) |
| b.Append(1456767743) |
| |
| arr := b.NewArray() |
| defer arr.Release() |
| |
| assert.Equal(t, "1968-11-30 13:30:45-0700", arr.ValueStr(0)) |
| assert.Equal(t, "2016-02-29 10:42:23-0700", arr.ValueStr(1)) |
| } |
| |
| func TestTimestampEquality(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| tsDatatypes := []*arrow.TimestampType{ |
| {Unit: arrow.Second}, |
| {Unit: arrow.Second, TimeZone: "UTC"}, |
| {Unit: arrow.Second, TimeZone: "America/Phoenix"}, |
| } |
| |
| arrs := make([]*array.Timestamp, 0, len(tsDatatypes)) |
| for _, dt := range tsDatatypes { |
| bldr := array.NewTimestampBuilder(mem, dt) |
| defer bldr.Release() |
| |
| bldr.Append(-34226955) |
| bldr.Append(1456767743) |
| |
| arr := bldr.NewTimestampArray() |
| defer arr.Release() |
| |
| arrs = append(arrs, arr) |
| } |
| |
| // No timezone, "wall clock" semantics |
| // These timestamps have no actual timezone, but we still represent as UTC per Go conventions |
| assert.Equal(t, "1968-11-30T20:30:45Z", arrs[0].ValueStr(0)) |
| assert.Equal(t, "2016-02-29T17:42:23Z", arrs[0].ValueStr(1)) |
| |
| // UTC timezone, "instant" semantics |
| assert.Equal(t, "1968-11-30T20:30:45Z", arrs[1].ValueStr(0)) |
| assert.Equal(t, "2016-02-29T17:42:23Z", arrs[1].ValueStr(1)) |
| |
| // America/Phoenix timezone, "instant" semantics |
| assert.Equal(t, "1968-11-30T13:30:45-07:00", arrs[2].ValueStr(0)) |
| assert.Equal(t, "2016-02-29T10:42:23-07:00", arrs[2].ValueStr(1)) |
| |
| // Despite timezone and semantics, the physical values are equivalent |
| assert.Equal(t, arrs[0].Value(0), arrs[1].Value(0)) |
| assert.Equal(t, arrs[0].Value(0), arrs[2].Value(0)) |
| assert.Equal(t, arrs[1].Value(0), arrs[2].Value(0)) |
| |
| assert.Equal(t, arrs[0].Value(1), arrs[1].Value(1)) |
| assert.Equal(t, arrs[0].Value(1), arrs[2].Value(1)) |
| assert.Equal(t, arrs[1].Value(1), arrs[2].Value(1)) |
| } |
| |
| func TestTimestampArrayJSONRoundTrip(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| tz, _ := time.LoadLocation("America/Phoenix") |
| dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: tz.String()} |
| b := array.NewTimestampBuilder(mem, dt) |
| defer b.Release() |
| |
| b.Append(-34226955) |
| b.Append(1456767743) |
| |
| arr := b.NewArray() |
| defer arr.Release() |
| |
| assert.Equal(t, "1968-11-30T13:30:45-07:00", arr.ValueStr(0)) |
| assert.Equal(t, "2016-02-29T10:42:23-07:00", arr.ValueStr(1)) |
| assert.Equal(t, 2, arr.Len()) |
| assert.Equal(t, 0, arr.NullN()) |
| |
| json_bytes, err := arr.MarshalJSON() |
| require.NoError(t, err) |
| |
| expectedJSON := `["1968-11-30T13:30:45-07:00","2016-02-29T10:42:23-07:00"]` |
| require.Equal(t, expectedJSON, string(json_bytes)) |
| |
| var timestamps []time.Time |
| err = json.Unmarshal(json_bytes, ×tamps) |
| require.NoError(t, err) |
| require.Len(t, timestamps, 2) |
| |
| expectedTime1 := time.Date(1968, time.November, 30, 13, 30, 45, 0, tz) |
| expectedTime2 := time.Date(2016, time.February, 29, 10, 42, 23, 0, tz) |
| |
| assert.Equal(t, expectedTime1.Unix(), timestamps[0].Unix()) |
| assert.Equal(t, expectedTime2.Unix(), timestamps[1].Unix()) |
| } |
| |
| func TestTimestampArrayJSONRoundTripWithDeprecatedLayout(t *testing.T) { |
| mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) |
| defer mem.AssertSize(t, 0) |
| |
| tz, _ := time.LoadLocation("America/Phoenix") |
| dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: tz.String()} |
| b := array.NewTimestampBuilderWithValueStrLayout(mem, dt, deprecatedLayout) |
| defer b.Release() |
| |
| b.Append(-34226955) |
| b.Append(1456767743) |
| |
| arr := b.NewArray() |
| defer arr.Release() |
| |
| json_bytes, err := arr.MarshalJSON() |
| require.NoError(t, err) |
| |
| expectedJSON := `["1968-11-30 13:30:45-0700","2016-02-29 10:42:23-0700"]` |
| require.Equal(t, expectedJSON, string(json_bytes)) |
| |
| } |
