blob: d8199e57e17c471b8e44c0606bc1dab480f6cd72 [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import {
Bool, DateDay, DateMillisecond, Dictionary, Float64, Int32, List, makeVector, Struct, Utf8, LargeUtf8, Utf8View, BinaryView, util, Vector, vectorFromArray, makeData, FixedSizeList, Field,
} from 'apache-arrow';
describe(`makeVectorFromArray`, () => {
describe(`works with null values`, () => {
const values = [1, 2, 3, 4, null, 5];
const vector = vectorFromArray(values);
basicVectorTests(vector, values, []);
test(`toArray returns typed array for numbers`, () => {
expect(vector.toArray()).toEqual(Float64Array.from(values.map(n => n === null ? 0 : n)));
});
test(`toJSON retains null`, () => {
expect(vector.toJSON()).toEqual(values);
});
});
});
describe(`basic vector methods`, () => {
test(`not nullable`, () => {
const vector = makeVector([makeData({ data: new Int32Array([1, 2, 3]), nullCount: -1, type: new Int32() })]);
expect(vector.nullable).toBe(false);
expect(vector.nullCount).toBe(0);
});
test(`nullable`, () => {
const vector = makeVector([makeData({ data: new Int32Array([1, 2, 3]), nullCount: 0, type: new Int32() })]);
expect(vector.nullable).toBe(true);
expect(vector.nullCount).toBe(0);
expect(vector.isValid(0)).toBe(true);
// set a value to null
vector.set(0, null);
expect(vector.nullable).toBe(true);
expect(vector.nullCount).toBe(1);
expect(vector.isValid(0)).toBe(false);
// set the same value to null which should not change anything
vector.set(0, null);
expect(vector.nullable).toBe(true);
expect(vector.nullCount).toBe(1);
// set a different value to null
vector.set(1, null);
expect(vector.nullable).toBe(true);
expect(vector.nullCount).toBe(2);
// set first value to non-null
vector.set(0, 1);
expect(vector.nullable).toBe(true);
expect(vector.nullCount).toBe(1);
// set last null to non-null
vector.set(1, 2);
expect(vector.nullable).toBe(true);
expect(vector.nullCount).toBe(0);
});
});
describe(`StructVector`, () => {
test(`makeVectorFromArray`, () => {
const values: { a?: number; b?: string | null; c?: boolean | null }[] = [
{ a: 1, b: null },
{ a: 4, b: 'foo', c: null },
{ a: 7, b: 'bar', c: true },
{ a: 10, b: 'baz', c: true },
];
const vector = vectorFromArray(values);
expect(vector.numChildren).toBe(3);
expect(vector).toHaveLength(4);
expect(vector.type.children[0].type).toBeInstanceOf(Float64);
expect(vector.type.children[1].type).toBeInstanceOf(Dictionary);
expect(vector.type.children[2].type).toBeInstanceOf(Bool);
});
const values: { a?: number; b?: string; c?: boolean }[] = [
{ a: 1, b: 'foo', c: true },
{ a: 4, b: 'foo', c: false },
{ a: 7, b: 'bar', c: true },
{ a: 10, b: 'baz', c: true },
];
const vector = vectorFromArray(values);
test(`has list struct`, () => {
expect(vector.type).toBeInstanceOf(Struct);
expect(vector.type.children[0].type).toBeInstanceOf(Float64);
expect(vector.type.children[1].type).toBeInstanceOf(Dictionary);
expect(vector.type.children[2].type).toBeInstanceOf(Bool);
expect(vector.type.children[0].nullable).toBeTruthy();
expect(vector.type.children[1].nullable).toBeTruthy();
expect(vector.type.children[2].nullable).toBeTruthy();
});
test(`get value`, () => {
for (const [i, value] of values.entries()) {
expect(vector.get(i)!.toJSON()).toEqual(value);
}
});
});
describe(`DateVector`, () => {
const extras = [
new Date(2000, 0, 1),
new Date(1991, 5, 28, 12, 11, 10)
];
describe(`unit = MILLISECOND`, () => {
const values = [
new Date(1989, 5, 22, 1, 2, 3),
new Date(1988, 3, 25, 4, 5, 6),
new Date(1987, 2, 24, 7, 8, 9),
new Date(2018, 4, 12, 17, 30, 0)
].map(v => v.getTime());
const vector = vectorFromArray(values, new DateMillisecond);
basicVectorTests(vector, values, extras);
});
describe(`unit = DAY`, () => {
// Use UTC to ensure that dates are always at midnight
const values = [
new Date(Date.UTC(1989, 5, 22)),
new Date(Date.UTC(1988, 3, 25)),
new Date(Date.UTC(1987, 2, 24)),
new Date(Date.UTC(2018, 4, 12))
].map(v => v.getTime());
const vector = vectorFromArray(values, new DateDay);
basicVectorTests(vector, values, extras);
});
});
describe(`DictionaryVector`, () => {
const dictionary = ['foo', 'bar', 'baz'];
const extras = ['abc', '123']; // values to search for that should NOT be found
const dictionary_vec = vectorFromArray(dictionary, new Utf8).memoize();
const indices = Array.from({ length: 50 }, () => Math.trunc(Math.random() * 3));
const validity = Array.from({ length: indices.length }, () => Math.random() > 0.2);
describe(`index with nullCount == 0`, () => {
const values = indices.map((d) => dictionary[d]);
const vector = makeVector({
data: indices,
dictionary: dictionary_vec,
type: new Dictionary(dictionary_vec.type, new Int32)
});
basicVectorTests(vector, values, extras);
describe(`sliced`, () => {
basicVectorTests(vector.slice(10, 20), values.slice(10, 20), extras);
});
});
describe(`index with nullCount > 0`, () => {
const nullBitmap = util.packBools(validity);
const nullCount = validity.reduce((acc, d) => acc + (d ? 0 : 1), 0);
const values = indices.map((d, i) => validity[i] ? dictionary[d] : null);
const vector = makeVector({
data: indices,
nullCount,
nullBitmap,
dictionary: dictionary_vec,
type: new Dictionary(dictionary_vec.type, new Int32)
});
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(10, 20), values.slice(10, 20), extras);
});
});
describe(`vectorFromArray`, () => {
const values = ['foo', 'bar', 'baz', 'foo', 'bar'];
const vector = vectorFromArray(values);
test(`has dictionary type`, () => {
expect(vector.type).toBeInstanceOf(Dictionary);
});
test(`has memoized dictionary`, () => {
expect(vector.isMemoized).toBe(true);
const unmemoized = vector.unmemoize();
expect(unmemoized.isMemoized).toBe(false);
});
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(1, 3), values.slice(1, 3), ['foo', 'abc']);
});
});
});
describe(`Utf8Vector`, () => {
const values = ['foo', 'bar', 'baz', 'foo bar', 'bar'];
const vector = vectorFromArray(values, new Utf8);
test(`has utf8 type`, () => {
expect(vector.type).toBeInstanceOf(Utf8);
});
test(`is not memoized`, () => {
expect(vector.isMemoized).toBe(false);
const memoizedVector = vector.memoize();
expect(memoizedVector.isMemoized).toBe(true);
const unMemoizedVector = vector.unmemoize();
expect(unMemoizedVector.isMemoized).toBe(false);
});
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(1, 3), values.slice(1, 3), ['foo', 'abc']);
});
});
describe(`LargeUtf8Vector`, () => {
const values = ['foo', 'bar', 'baz', 'foo bar', 'bar'];
const vector = vectorFromArray(values, new LargeUtf8);
test(`has largeUtf8 type`, () => {
expect(vector.type).toBeInstanceOf(LargeUtf8);
});
test(`is not memoized`, () => {
expect(vector.isMemoized).toBe(false);
const memoizedVector = vector.memoize();
expect(memoizedVector.isMemoized).toBe(true);
const unMemoizedVector = vector.unmemoize();
expect(unMemoizedVector.isMemoized).toBe(false);
});
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(1, 3), values.slice(1, 3), ['foo', 'abc']);
});
});
describe(`Utf8ViewVector`, () => {
const values = ['foo', 'bar', 'baz', 'foo bar', 'bar'];
const vector = vectorFromArray(values, new Utf8View);
test(`has utf8View type`, () => {
expect(vector.type).toBeInstanceOf(Utf8View);
});
test(`is not memoized`, () => {
expect(vector.isMemoized).toBe(false);
const memoizedVector = vector.memoize();
expect(memoizedVector.isMemoized).toBe(true);
const unMemoizedVector = vector.unmemoize();
expect(unMemoizedVector.isMemoized).toBe(false);
});
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {
basicVectorTests(vector.slice(1, 3), values.slice(1, 3), ['foo', 'abc']);
});
});
describe(`BinaryViewVector`, () => {
const values = [new Uint8Array([1, 2, 3]), new Uint8Array([4, 5]), new Uint8Array([6, 7, 8, 9])];
const vector = vectorFromArray(values, new BinaryView);
test(`has binaryView type`, () => {
expect(vector.type).toBeInstanceOf(BinaryView);
});
test(`is not memoized`, () => {
expect(vector.isMemoized).toBe(false);
const memoizedVector = vector.memoize();
expect(memoizedVector.isMemoized).toBe(true);
const unMemoizedVector = vector.unmemoize();
expect(unMemoizedVector.isMemoized).toBe(false);
});
basicVectorTests(vector, values, [new Uint8Array([10, 11]), new Uint8Array([12])]);
describe(`sliced`, () => {
basicVectorTests(vector.slice(1, 3), values.slice(1, 3), [new Uint8Array([1, 2]), new Uint8Array([3, 4])]);
});
});
describe(`ListVector`, () => {
const values = [[1, 2], [1, 2, 3]];
const vector = vectorFromArray(values);
test(`has list type`, () => {
expect(vector.type).toBeInstanceOf(List);
});
test(`get value`, () => {
for (let i = 0; i < values.length; i++) {
expect(vector.get(i)!.toJSON()).toEqual(values[i]);
expect(vector.at(i)!.toJSON()).toEqual(values.at(i));
expect(vector.at(-i)!.toJSON()).toEqual(values.at(-i));
}
});
});
describe(`toArray()`, () => {
test(`when some data blobs have been padded`, () => {
const d1 = vectorFromArray([...new Array(16).keys()]);
const d2 = vectorFromArray([...new Array(10).keys()]);
// Padding has been added
expect(d2.length).toBeLessThan(d2.data[0].buffers[1].length);
const vector = new Vector([d1, d2]);
// This used to crash with "RangeError: offset is out of bounds"
// https://issues.apache.org/jira/browse/ARROW-18247
const array = vector.toArray();
expect(array).toHaveLength(26);
});
});
// Creates some basic tests for the given vector.
// Verifies that:
// - `get` and the native iterator return the same data as `values`
// - `indexOf` returns the same indices as `values`
function basicVectorTests(vector: Vector, values: any[], extras: any[]) {
const n = values.length;
test(`gets expected values`, () => {
for (let i = 0; i < values.length; i++) {
expect(vector.get(i)).toEqual(values[i]);
expect(vector.at(i)).toEqual(values.at(i));
expect(vector.at(-i)).toEqual(values.at(-i));
}
});
test(`iterates expected values`, () => {
expect.hasAssertions();
let i = -1;
for (const v of vector) {
expect(++i).toBeLessThan(n);
expect(v).toEqual(values[i]);
}
});
test(`indexOf returns expected values`, () => {
const testValues = values.concat(extras);
for (const value of testValues) {
const actual = vector.indexOf(value);
const expected = values.indexOf(value);
expect(actual).toEqual(expected);
}
});
}
// GH-45862: Make sure vectorFromArray produces the correct result for
// FixedSizeList with null slots
describe(`vecorFromArray() with FixedSizeList<T> and null slots`, () => {
test(`correct child length with null slot first`, () => {
const vector = vectorFromArray(
[null, [1, 2, 3]],
new FixedSizeList(3, new Field('item', new Int32())),
);
const child = vector.getChildAt(0);
expect(child).toHaveLength(6);
expect(child?.nullCount).toBe(3);
});
test(`correct child length with null slot last`, () => {
const vector = vectorFromArray(
[[1, 2, 3], null],
new FixedSizeList(3, new Field('item', new Int32())),
);
const child = vector.getChildAt(0);
expect(child).toHaveLength(6);
expect(child?.nullCount).toBe(3);
});
test(`correct child length with null in the middle`, () => {
const vector = vectorFromArray(
[[1, 2, 3], null, [7, 8, 9]],
new FixedSizeList(3, new Field('item', new Int32())),
);
const child = vector.getChildAt(0);
expect(child).toHaveLength(9);
expect(child?.nullCount).toBe(3);
});
});