arrow/array/map_test.go

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package array_test

import (
	"strconv"
	"testing"

	"github.com/apache/arrow-go/v18/arrow"
	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/memory"
	"github.com/stretchr/testify/assert"
)

func TestMapArray(t *testing.T) {
	pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
	defer pool.AssertSize(t, 0)

	var (
		arr, equalArr, unequalArr *array.Map

		equalValid     = []bool{true, true, true, true, true, true, true}
		equalOffsets   = []int32{0, 1, 2, 5, 6, 7, 8, 10}
		equalKeys      = []string{"a", "a", "a", "b", "c", "a", "a", "a", "a", "b"}
		equalValues    = []int32{1, 2, 3, 4, 5, 2, 2, 2, 5, 6}
		unequalValid   = []bool{true, true, true}
		unequalOffsets = []int32{0, 1, 4, 7}
		unequalKeys    = []string{"a", "a", "b", "c", "a", "b", "c"}
		unequalValues  = []int32{1, 2, 2, 2, 3, 4, 5}
	)

	bldr := array.NewMapBuilder(pool, arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int32, false)
	defer bldr.Release()

	kb := bldr.KeyBuilder().(*array.StringBuilder)
	ib := bldr.ItemBuilder().(*array.Int32Builder)

	bldr.AppendValues(equalOffsets, equalValid)
	for _, k := range equalKeys {
		kb.Append(k)
	}
	ib.AppendValues(equalValues, nil)

	assert.Equal(t, len(equalValid), bldr.Len())
	assert.Zero(t, bldr.NullN())

	arr = bldr.NewMapArray()
	defer arr.Release()

	bldr.AppendValues(equalOffsets, equalValid)
	for _, k := range equalKeys {
		kb.Append(k)
	}
	ib.AppendValues(equalValues, nil)

	equalArr = bldr.NewMapArray()
	defer equalArr.Release()

	bldr.AppendValues(unequalOffsets, unequalValid)
	for _, k := range unequalKeys {
		kb.Append(k)
	}
	ib.AppendValues(unequalValues, nil)

	unequalArr = bldr.NewMapArray()
	defer unequalArr.Release()

	assert.True(t, array.Equal(arr, arr))
	assert.True(t, array.Equal(arr, equalArr))
	assert.True(t, array.Equal(equalArr, arr))
	assert.False(t, array.Equal(equalArr, unequalArr))
	assert.False(t, array.Equal(unequalArr, equalArr))

	assert.True(t, array.SliceEqual(arr, 0, 1, unequalArr, 0, 1))
	assert.False(t, array.SliceEqual(arr, 0, 2, unequalArr, 0, 2))
	assert.False(t, array.SliceEqual(arr, 1, 2, unequalArr, 1, 2))
	assert.True(t, array.SliceEqual(arr, 2, 3, unequalArr, 2, 3))

	t.Run("items non nullable", func(t *testing.T) {
		mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
		defer mem.AssertSize(t, 0)

		dt := arrow.MapOf(arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Int16)
		dt.KeysSorted = true
		dt.SetItemNullable(false)

		bldr := array.NewBuilder(pool, dt).(*array.MapBuilder)
		defer bldr.Release()

		kb := bldr.KeyBuilder().(*array.Int16Builder)
		ib := bldr.ItemBuilder().(*array.Int16Builder)

		bldr.Append(true)
		kb.Append(1)
		ib.AppendNull()

		assert.Panics(t, func() {
			_ = bldr.NewArray()
		})
	})
}

func TestMapArrayBuildIntToInt(t *testing.T) {
	pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
	defer pool.AssertSize(t, 0)

	var (
		dtype      = arrow.MapOf(arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Int16)
		keys       = []int16{0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5}
		items      = []int16{1, 1, 2, 3, 5, 8, -1, -1, 0, 1, -1, 2}
		validItems = []bool{true, true, true, true, true, true, false, false, true, true, false, true}
		offsets    = []int32{0, 6, 6, 12, 12}
		validMaps  = []bool{true, false, true, true}
	)

	bldr := array.NewBuilder(pool, dtype).(*array.MapBuilder)
	defer bldr.Release()

	bldr.Reserve(len(validMaps))

	kb := bldr.KeyBuilder().(*array.Int16Builder)
	ib := bldr.ItemBuilder().(*array.Int16Builder)

	bldr.Append(true)
	kb.AppendValues(keys[:6], nil)
	ib.AppendValues(items[:6], nil)

	bldr.AppendNull()
	bldr.Append(true)
	kb.AppendValues(keys[6:], nil)
	ib.AppendValues(items[6:], []bool{false, false, true, true, false, true})

	bldr.Append(true)
	arr := bldr.NewArray().(*array.Map)
	defer arr.Release()

	assert.Equal(t, arrow.MAP, arr.DataType().ID())
	assert.EqualValues(t, len(validMaps), arr.Len())

	for i, ex := range validMaps {
		assert.Equal(t, ex, arr.IsValid(i))
		assert.Equal(t, !ex, arr.IsNull(i))
	}

	assert.Equal(t, offsets, arr.Offsets())
	assert.Equal(t, keys, arr.Keys().(*array.Int16).Int16Values())

	itemArr := arr.Items().(*array.Int16)
	for i, ex := range validItems {
		if ex {
			assert.True(t, itemArr.IsValid(i))
			assert.False(t, itemArr.IsNull(i))
			assert.Equal(t, items[i], itemArr.Value(i))
		} else {
			assert.False(t, itemArr.IsValid(i))
			assert.True(t, itemArr.IsNull(i))
		}
	}

	assert.Equal(t, "[{[0 1 2 3 4 5] [1 1 2 3 5 8]} (null) {[0 1 2 3 4 5] [(null) (null) 0 1 (null) 2]} {[] []}]", arr.String())
}

func TestMapBuilderRejectsInvalidEntryLengths(t *testing.T) {
	tests := []struct {
		name      string
		build     func(*array.MapBuilder)
		panicText string
	}{
		{
			name: "more keys than items",
			build: func(b *array.MapBuilder) {
				b.Append(true)
				b.KeyBuilder().(*array.Int32Builder).Append(1)
			},
			panicText: "invalid: arrow/array: map key and item builders must have equal lengths (keys=1, items=0)",
		},
		{
			name: "more items than keys",
			build: func(b *array.MapBuilder) {
				b.Append(true)
				b.ItemBuilder().(*array.Int32Builder).Append(1)
			},
			panicText: "invalid: arrow/array: map key and item builders must have equal lengths (keys=0, items=1)",
		},
		{
			name: "struct longer than entries",
			build: func(b *array.MapBuilder) {
				b.Append(true)
				b.ValueBuilder().(*array.StructBuilder).Append(true)
			},
			panicText: "invalid: arrow/array: map struct builder length exceeds key and item length (struct=1, entries=0)",
		},
		{
			name: "too few offsets",
			build: func(b *array.MapBuilder) {
				b.AppendValues(nil, []bool{true})
			},
			panicText: "invalid: arrow/array: map offset count must equal map length or map length plus one (offsets=0, maps=1)",
		},
		{
			name: "too many offsets",
			build: func(b *array.MapBuilder) {
				b.AppendValues([]int32{0, 0, 0}, []bool{true})
			},
			panicText: "invalid: arrow/array: map offset count must equal map length or map length plus one (offsets=3, maps=1)",
		},
		{
			name: "final offset exceeds entries",
			build: func(b *array.MapBuilder) {
				b.AppendValues([]int32{0, 2}, []bool{true})
				b.KeyBuilder().(*array.Int32Builder).Append(1)
				b.ItemBuilder().(*array.Int32Builder).Append(2)
			},
			panicText: "invalid: arrow/array: map final offset must match key and item length (offset=2, entries=1)",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
			defer mem.AssertSize(t, 0)

			b := array.NewMapBuilder(mem, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32, false)
			defer b.Release()
			tt.build(b)

			assert.PanicsWithError(t, tt.panicText, func() {
				b.NewMapArray()
			})
		})
	}
}

func TestMapStringRoundTrip(t *testing.T) {
	// 1. create array
	dt := arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int32)

	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
	defer mem.AssertSize(t, 0)

	b := array.NewMapBuilderWithType(mem, dt)
	defer b.Release()

	kb := b.KeyBuilder().(*array.StringBuilder)
	ib := b.ItemBuilder().(*array.Int32Builder)

	for n := 0; n < 10; n++ {
		b.AppendNull()
		b.Append(true)

		for r := 'a'; r <= 'z'; r++ {
			kb.Append(string(r) + strconv.Itoa(n))
			if (n+int(r))%2 == 0 {
				ib.AppendNull()
			} else {
				ib.Append(int32(n + int(r)))
			}
		}
	}

	arr := b.NewArray().(*array.Map)
	defer arr.Release()

	// 2. create array via AppendValueFromString
	b1 := array.NewMapBuilderWithType(mem, dt)
	defer b1.Release()

	for i := 0; i < arr.Len(); i++ {
		assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
	}

	arr1 := b1.NewArray().(*array.Map)
	defer arr1.Release()

	assert.True(t, array.Equal(arr, arr1))
}

func TestMapBuilder_SetNull(t *testing.T) {
	pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
	defer pool.AssertSize(t, 0)

	var (
		arr          *array.Map
		equalValid   = []bool{true, true, true, true, true, true, true}
		equalOffsets = []int32{0, 1, 2, 5, 6, 7, 8, 10}
		equalKeys    = []string{"a", "a", "a", "b", "c", "a", "a", "a", "a", "b"}
		equalValues  = []int32{1, 2, 3, 4, 5, 2, 2, 2, 5, 6}
	)

	bldr := array.NewMapBuilder(pool, arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int32, false)
	defer bldr.Release()

	kb := bldr.KeyBuilder().(*array.StringBuilder)
	ib := bldr.ItemBuilder().(*array.Int32Builder)

	bldr.AppendValues(equalOffsets, equalValid)
	for _, k := range equalKeys {
		kb.Append(k)
	}
	ib.AppendValues(equalValues, nil)

	bldr.SetNull(0)
	bldr.SetNull(3)
	assert.EqualValues(t, 2, bldr.NullN())

	bldr.SetNull(3) // idempotent
	assert.EqualValues(t, 2, bldr.NullN())

	arr = bldr.NewMapArray()
	defer arr.Release()

	assert.True(t, arr.IsNull(0))
	assert.True(t, arr.IsValid(1))
	assert.True(t, arr.IsNull(3))
}