c_glib/test/test-table.rb

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

class TestTable < Test::Unit::TestCase
  include Helper::Buildable
  include Helper::Omittable

  sub_test_case(".new") do
    def setup
      @fields = [
        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
      ]
      @schema = Arrow::Schema.new(@fields)
    end

    def dump_table(table)
      table.n_columns.times.collect do |i|
        field = table.schema.get_field(i)
        chunked_array = table.get_column_data(i)
        values = []
        chunked_array.chunks.each do |chunk|
          chunk.length.times do |j|
            values << chunk.get_value(j)
          end
        end
        [
          field.name,
          values,
        ]
      end
    end

    def test_arrays
      require_gi_bindings(3, 3, 1)
      arrays = [
        build_boolean_array([true]),
        build_boolean_array([false]),
      ]
      table = Arrow::Table.new(@schema, arrays)
      assert_equal([
                     ["visible", [true]],
                     ["valid", [false]],
                   ],
                   dump_table(table))
    end

    def test_chunked_arrays
      require_gi_bindings(3, 3, 1)
      arrays = [
        Arrow::ChunkedArray.new([build_boolean_array([true]),
                                 build_boolean_array([false])]),
        Arrow::ChunkedArray.new([build_boolean_array([false]),
                                 build_boolean_array([true])]),
      ]
      table = Arrow::Table.new(@schema, arrays)
      assert_equal([
                     ["visible", [true, false]],
                     ["valid", [false, true]],
                   ],
                   dump_table(table))
    end

    def test_record_batches
      require_gi_bindings(3, 3, 1)
      record_batches = [
        build_record_batch({
                             "visible" => build_boolean_array([true]),
                             "valid" => build_boolean_array([false])
                           }),
        build_record_batch({
                             "visible" => build_boolean_array([false]),
                             "valid" => build_boolean_array([true])
                           }),
      ]
      table = Arrow::Table.new(@schema, record_batches)

      assert_equal([
                     ["visible", [true, false]],
                     ["valid", [false, true]],
                   ],
                   dump_table(table))
    end
  end

  sub_test_case("instance methods") do
    def setup
      @fields = [
        Arrow::Field.new("visible", Arrow::BooleanDataType.new),
        Arrow::Field.new("valid", Arrow::BooleanDataType.new),
      ]
      @schema = Arrow::Schema.new(@fields)
      @columns = [
        build_boolean_array([true]),
        build_boolean_array([false]),
      ]
      @table = Arrow::Table.new(@schema, @columns)
    end

    def test_equal
      other_table = Arrow::Table.new(@schema, @columns)
      assert_equal(@table, other_table)
    end

    def test_equal_metadata
      other_table = Arrow::Table.new(@schema, @columns)
      assert do
        @table.equal_metadata(other_table, true)
      end
    end

    def test_schema
      assert_equal(["visible", "valid"],
                   @table.schema.fields.collect(&:name))
    end

    def test_column_data
      assert_equal([
                     Arrow::ChunkedArray.new([build_boolean_array([true])]),
                     Arrow::ChunkedArray.new([build_boolean_array([false])]),
                   ],
                   [
                     @table.get_column_data(0),
                     @table.get_column_data(-1),
                   ])
    end

    def test_n_columns
      assert_equal(2, @table.n_columns)
    end

    def test_n_rows
      assert_equal(1, @table.n_rows)
    end

    def test_add_column
      field = Arrow::Field.new("added", Arrow::BooleanDataType.new)
      chunked_array = Arrow::ChunkedArray.new([build_boolean_array([true])])
      new_table = @table.add_column(1, field, chunked_array)
      assert_equal(["visible", "added", "valid"],
                   new_table.schema.fields.collect(&:name))
    end

    def test_remove_column
      new_table = @table.remove_column(0)
      assert_equal(["valid"],
                   new_table.schema.fields.collect(&:name))
    end

    def test_replace_column
      field = Arrow::Field.new("added", Arrow::BooleanDataType.new)
      chunked_array = Arrow::ChunkedArray.new([build_boolean_array([true])])
      new_table = @table.replace_column(0, field, chunked_array)
      assert_equal(["added", "valid"],
                   new_table.schema.fields.collect(&:name))
    end

    def test_to_s
      table = build_table("valid" => build_boolean_array([true, false, true]))
      assert_equal(<<-TABLE, table.to_s)
valid: bool
----
valid:
  [
    [
      true,
      false,
      true
    ]
  ]
      TABLE
    end

    sub_test_case("#concatenate") do
      def test_without_options
        table = build_table("visible" =>
                            build_boolean_array([true, false, true, false]))
        table1 = build_table("visible" => build_boolean_array([true]))
        table2 = build_table("visible" => build_boolean_array([false, true]))
        table3 = build_table("visible" => build_boolean_array([false]))
        assert_equal(table, table1.concatenate([table2, table3]))
      end

      def test_with_options
        options = Arrow::TableConcatenateOptions.new
        options.unify_schemas = true
        table = build_table("a" => build_int32_array([1, nil, 3]),
                            "b" => build_int32_array([10, nil, 30]),
                            "c" => build_int32_array([nil, 200, nil]))
        table1 = build_table("a" => build_int32_array([1]),
                             "b" => build_int32_array([10]))
        table2 = build_table("c" => build_int32_array([200]))
        table3 = build_table("a" => build_int32_array([3]),
                             "b" => build_int32_array([30]))
        assert_equal(table, table1.concatenate([table2, table3], options))
      end
    end

    sub_test_case("#slice") do
      test("offset: positive") do
        visibles = [true, false, true]
        table = build_table("visible" => build_boolean_array(visibles))
        assert_equal(build_table("visible" => build_boolean_array([false, true])),
                     table.slice(1, 2))
      end

      test("offset: negative") do
        visibles = [true, false, true]
        table = build_table("visible" => build_boolean_array(visibles))
        assert_equal(build_table("visible" => build_boolean_array([false, true])),
                     table.slice(-2, 2))
      end
    end

    def test_combine_chunks
      table = build_table(
        "visible" => Arrow::ChunkedArray::new([build_boolean_array([true, false, true]),
                                               build_boolean_array([false, true]),
                                               build_boolean_array([false])])
      )
      combined_table = table.combine_chunks
      all_values = combined_table.n_columns.times.collect do |i|
        column = combined_table.get_column_data(i)
        column.n_chunks.times.collect do |j|
          column.get_chunk(j).values
        end
      end
      assert_equal([[[true, false, true, false, true, false]]],
                   all_values)
    end

    sub_test_case("#validate") do
      def setup
        @id_field = Arrow::Field.new("id", Arrow::UInt8DataType.new)
        @name_field = Arrow::Field.new("name", Arrow::StringDataType.new)
        @schema = Arrow::Schema.new([@id_field, @name_field])

        @id_array = build_uint_array([1])
        @name_array = build_string_array(["abc"])
        @arrays = [@id_array, @name_array]
      end

      def test_valid
        table = Arrow::Table.new(@schema, @arrays)

        assert do
          table.validate
        end
      end

      def test_invalid
        message = "[table][validate]: Invalid: " +
          "Column 1 named name expected length 1 but got length 2"

        invalid_values = [@id_array, build_string_array(["abc", "def"])]
        table = Arrow::Table.new(@schema, invalid_values)
        error = assert_raise(Arrow::Error::Invalid) do
          table.validate
        end
        assert_equal(message,
                     error.message.lines.first.chomp)
      end
    end

    sub_test_case("#validate_full") do
      def setup
        @id_field = Arrow::Field.new("uint8", Arrow::UInt8DataType.new)
        @name_field = Arrow::Field.new("string", Arrow::StringDataType.new)
        @schema = Arrow::Schema.new([@id_field, @name_field])

        @id_values = build_uint_array([1])
        @valid_name_values = build_string_array(["abc"])

        # U+3042 HIRAGANA LETTER A, U+3044 HIRAGANA LETTER I
        data = "\u3042\u3044".b[0..-2]
        value_offsets = Arrow::Buffer.new([0, data.size].pack("l*"))
        @invalid_name_values = Arrow::StringArray.new(1,
                                                      value_offsets,
                                                      Arrow::Buffer.new(data),
                                                      nil,
                                                      -1)
      end

      def test_valid
        columns = [@id_values, @valid_name_values]
        table = Arrow::Table.new(@schema, columns)

        assert do
          table.validate_full
        end
      end

      def test_invalid
        message = "[table][validate-full]: Invalid: " +
          "Column 1: In chunk 0: Invalid: Invalid UTF8 sequence at string index 0"
        columns = [@id_values, @invalid_name_values]
        table = Arrow::Table.new(@schema, columns)

        error = assert_raise(Arrow::Error::Invalid) do
          table.validate_full
        end
        assert_equal(message,
                     error.message.lines.first.chomp)
      end
    end

    sub_test_case("#write_as_feather") do
      def setup
        super
        @tempfile = Tempfile.open("arrow-table-write-as-feather")
        begin
          yield
        ensure
          @tempfile.close!
        end
      end

      def read_feather
        input = Arrow::MemoryMappedInputStream.new(@tempfile.path)
        reader = Arrow::FeatherFileReader.new(input)
        begin
          yield(reader.read)
        ensure
          input.close
        end
      end

      test("default") do
        output = Arrow::FileOutputStream.new(@tempfile.path, false)
        @table.write_as_feather(output)
        output.close

        read_feather do |read_table|
          assert_equal(@table, read_table)
        end
      end

      test("compression") do
        output = Arrow::FileOutputStream.new(@tempfile.path, false)
        properties = Arrow::FeatherWriteProperties.new
        properties.compression = :zstd
        @table.write_as_feather(output, properties)
        output.close

        read_feather do |read_table|
          assert_equal(@table, read_table)
        end
      end
    end
  end
end