| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # pylint:disable=redefined-outer-name |
| from typing import Any |
| |
| import pytest |
| |
| from pyiceberg.conversions import to_bytes |
| from pyiceberg.expressions import ( |
| And, |
| EqualTo, |
| GreaterThan, |
| GreaterThanOrEqual, |
| In, |
| IsNaN, |
| IsNull, |
| LessThan, |
| LessThanOrEqual, |
| Not, |
| NotEqualTo, |
| NotIn, |
| NotNaN, |
| NotNull, |
| NotStartsWith, |
| Or, |
| StartsWith, |
| ) |
| from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator |
| from pyiceberg.manifest import DataFile, FileFormat |
| from pyiceberg.schema import Schema |
| from pyiceberg.types import ( |
| DoubleType, |
| FloatType, |
| IcebergType, |
| IntegerType, |
| NestedField, |
| PrimitiveType, |
| StringType, |
| ) |
| |
| INT_MIN_VALUE = 30 |
| INT_MAX_VALUE = 79 |
| |
| |
| def _to_byte_buffer(field_type: IcebergType, val: Any) -> bytes: |
| if not isinstance(field_type, PrimitiveType): |
| raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") |
| return to_bytes(field_type, val) |
| |
| |
| INT_MIN = _to_byte_buffer(IntegerType(), INT_MIN_VALUE) |
| INT_MAX = _to_byte_buffer(IntegerType(), INT_MAX_VALUE) |
| |
| STRING_MIN = _to_byte_buffer(StringType(), "a") |
| STRING_MAX = _to_byte_buffer(StringType(), "z") |
| |
| |
| @pytest.fixture |
| def schema_data_file() -> Schema: |
| return Schema( |
| NestedField(1, "id", IntegerType(), required=True), |
| NestedField(2, "no_stats", IntegerType(), required=False), |
| NestedField(3, "required", StringType(), required=True), |
| NestedField(4, "all_nulls", StringType(), required=False), |
| NestedField(5, "some_nulls", StringType(), required=False), |
| NestedField(6, "no_nulls", StringType(), required=False), |
| NestedField(7, "all_nans", DoubleType(), required=False), |
| NestedField(8, "some_nans", FloatType(), required=False), |
| NestedField(9, "no_nans", FloatType(), required=False), |
| NestedField(10, "all_nulls_double", DoubleType(), required=False), |
| NestedField(11, "all_nans_v1_stats", FloatType(), required=False), |
| NestedField(12, "nan_and_null_only", DoubleType(), required=False), |
| NestedField(13, "no_nan_stats", DoubleType(), required=False), |
| NestedField(14, "some_empty", StringType(), required=False), |
| ) |
| |
| |
| @pytest.fixture |
| def data_file() -> DataFile: |
| return DataFile( |
| file_path="file_1.parquet", |
| file_format=FileFormat.PARQUET, |
| partition={}, |
| record_count=50, |
| file_size_in_bytes=3, |
| value_counts={ |
| 4: 50, |
| 5: 50, |
| 6: 50, |
| 7: 50, |
| 8: 50, |
| 9: 50, |
| 10: 50, |
| 11: 50, |
| 12: 50, |
| 13: 50, |
| 14: 50, |
| }, |
| null_value_counts={4: 50, 5: 10, 6: 0, 10: 50, 11: 0, 12: 1, 14: 8}, |
| nan_value_counts={ |
| 7: 50, |
| 8: 10, |
| 9: 0, |
| }, |
| lower_bounds={ |
| 1: to_bytes(IntegerType(), INT_MIN_VALUE), |
| 11: to_bytes(FloatType(), float("nan")), |
| 12: to_bytes(DoubleType(), float("nan")), |
| 14: to_bytes(StringType(), ""), |
| }, |
| upper_bounds={ |
| 1: to_bytes(IntegerType(), INT_MAX_VALUE), |
| 11: to_bytes(FloatType(), float("nan")), |
| 12: to_bytes(DoubleType(), float("nan")), |
| 14: to_bytes(StringType(), "房东整租霍营小区二层两居室"), |
| }, |
| ) |
| |
| |
| @pytest.fixture |
| def data_file_2() -> DataFile: |
| return DataFile( |
| file_path="file_2.parquet", |
| file_format=FileFormat.PARQUET, |
| partition={}, |
| record_count=50, |
| file_size_in_bytes=3, |
| value_counts={3: 20}, |
| null_value_counts={3: 2}, |
| nan_value_counts=None, |
| lower_bounds={3: to_bytes(StringType(), "aa")}, |
| upper_bounds={3: to_bytes(StringType(), "dC")}, |
| ) |
| |
| |
| @pytest.fixture |
| def data_file_3() -> DataFile: |
| return DataFile( |
| file_path="file_3.parquet", |
| file_format=FileFormat.PARQUET, |
| partition={}, |
| record_count=50, |
| file_size_in_bytes=3, |
| value_counts={3: 20}, |
| null_value_counts={3: 2}, |
| nan_value_counts=None, |
| lower_bounds={3: to_bytes(StringType(), "1str1")}, |
| upper_bounds={3: to_bytes(StringType(), "3str3")}, |
| ) |
| |
| |
| @pytest.fixture |
| def data_file_4() -> DataFile: |
| return DataFile( |
| file_path="file_4.parquet", |
| file_format=FileFormat.PARQUET, |
| partition={}, |
| record_count=50, |
| file_size_in_bytes=3, |
| value_counts={3: 20}, |
| null_value_counts={3: 2}, |
| nan_value_counts=None, |
| lower_bounds={3: to_bytes(StringType(), "abc")}, |
| upper_bounds={3: to_bytes(StringType(), "イロハニホヘト")}, |
| ) |
| |
| |
| def test_all_null(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("all_nulls")).eval(data_file) |
| assert not should_read, "Should skip: no non-null value in all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("all_nulls", "a")).eval(data_file) |
| assert not should_read, "Should skip: lessThan on all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("all_nulls", "a")).eval(data_file) |
| assert not should_read, "Should skip: lessThanOrEqual on all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("all_nulls", "a")).eval(data_file) |
| assert not should_read, "Should skip: greaterThan on all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("all_nulls", "a")).eval(data_file) |
| assert not should_read, "Should skip: greaterThanOrEqual on all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("all_nulls", "a")).eval(data_file) |
| assert not should_read, "Should skip: equal on all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("some_nulls")).eval(data_file) |
| assert should_read, "Should read: column with some nulls contains a non-null value" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("no_nulls")).eval(data_file) |
| assert should_read, "Should read: non-null column contains a non-null value" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("all_nulls", "asad")).eval(data_file) |
| assert not should_read, "Should skip: startsWith on all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("all_nulls", "asad")).eval(data_file) |
| assert should_read, "Should read: notStartsWith on all null column" |
| |
| |
| def test_no_nulls(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("all_nulls")).eval(data_file) |
| assert should_read, "Should read: at least one null value in all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("some_nulls")).eval(data_file) |
| assert should_read, "Should read: column with some nulls contains a null value" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("no_nulls")).eval(data_file) |
| assert not should_read, "Should skip: non-null column contains no null values" |
| |
| |
| def test_is_nan(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("all_nans")).eval(data_file) |
| assert should_read, "Should read: at least one nan value in all nan column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("some_nans")).eval(data_file) |
| assert should_read, "Should read: at least one nan value in some nan column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("no_nans")).eval(data_file) |
| assert not should_read, "Should skip: no-nans column contains no nan values" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("all_nulls_double")).eval(data_file) |
| assert not should_read, "Should skip: all-null column doesn't contain nan value" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("no_nan_stats")).eval(data_file) |
| assert should_read, "Should read: no guarantee on if contains nan value without nan stats" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("all_nans_v1_stats")).eval(data_file) |
| assert should_read, "Should read: at least one nan value in all nan column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("nan_and_null_only")).eval(data_file) |
| assert should_read, "Should read: at least one nan value in nan and nulls only column" |
| |
| |
| def test_not_nan(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("all_nans")).eval(data_file) |
| assert not should_read, "Should skip: column with all nans will not contain non-nan" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("some_nans")).eval(data_file) |
| assert should_read, "Should read: at least one non-nan value in some nan column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("no_nans")).eval(data_file) |
| assert should_read, "Should read: at least one non-nan value in no nan column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("all_nulls_double")).eval(data_file) |
| assert should_read, "Should read: at least one non-nan value in all null column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("no_nan_stats")).eval(data_file) |
| assert should_read, "Should read: no guarantee on if contains nan value without nan stats" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("all_nans_v1_stats")).eval(data_file) |
| assert should_read, "Should read: no guarantee on if contains nan value without nan stats" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("nan_and_null_only")).eval(data_file) |
| assert should_read, "Should read: at least one null value in nan and nulls only column" |
| |
| |
| def test_required_column(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("required")).eval(data_file) |
| assert should_read, "Should read: required columns are always non-null" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("required")).eval(data_file) |
| assert not should_read, "Should skip: required columns are always non-null" |
| |
| |
| def test_missing_column(schema_data_file: Schema, data_file: DataFile) -> None: |
| with pytest.raises(ValueError) as exc_info: |
| _ = _InclusiveMetricsEvaluator(schema_data_file, LessThan("missing", 22)).eval(data_file) |
| |
| assert str(exc_info.value) == "Could not find field with name missing, case_sensitive=True" |
| |
| |
| def test_missing_stats() -> None: |
| no_stats_schema = Schema( |
| NestedField(2, "no_stats", DoubleType(), required=False), |
| ) |
| |
| no_stats_file = DataFile( |
| file_path="file_1.parquet", |
| file_format=FileFormat.PARQUET, |
| partition={}, |
| record_count=50, |
| value_counts=None, |
| null_value_counts=None, |
| nan_value_counts=None, |
| lower_bounds=None, |
| upper_bounds=None, |
| ) |
| |
| expressions = [ |
| LessThan("no_stats", 5), |
| LessThanOrEqual("no_stats", 30), |
| EqualTo("no_stats", 70), |
| GreaterThan("no_stats", 78), |
| GreaterThanOrEqual("no_stats", 90), |
| NotEqualTo("no_stats", 101), |
| IsNull("no_stats"), |
| NotNull("no_stats"), |
| IsNaN("no_stats"), |
| NotNaN("no_stats"), |
| ] |
| |
| for expression in expressions: |
| should_read = _InclusiveMetricsEvaluator(no_stats_schema, expression).eval(no_stats_file) |
| assert should_read, f"Should read when stats are missing for: {expression}" |
| |
| |
| def test_zero_record_file_stats(schema_data_file: Schema) -> None: |
| zero_record_data_file = DataFile(file_path="file_1.parquet", file_format=FileFormat.PARQUET, partition={}, record_count=0) |
| |
| expressions = [ |
| LessThan("no_stats", 5), |
| LessThanOrEqual("no_stats", 30), |
| EqualTo("no_stats", 70), |
| GreaterThan("no_stats", 78), |
| GreaterThanOrEqual("no_stats", 90), |
| NotEqualTo("no_stats", 101), |
| IsNull("no_stats"), |
| NotNull("no_stats"), |
| IsNaN("no_stats"), |
| NotNaN("no_stats"), |
| ] |
| |
| for expression in expressions: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, expression).eval(zero_record_data_file) |
| assert not should_read, f"Should skip a datafile without records: {expression}" |
| |
| |
| def test_not(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(LessThan("id", INT_MIN_VALUE - 25))).eval(data_file) |
| assert should_read, "Should read: not(false)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(GreaterThan("id", INT_MIN_VALUE - 25))).eval(data_file) |
| assert not should_read, "Should skip: not(true)" |
| |
| |
| def test_and(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator( |
| schema_data_file, And(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MIN_VALUE - 30)) |
| ).eval(data_file) |
| assert not should_read, "Should skip: and(false, true)" |
| |
| should_read = _InclusiveMetricsEvaluator( |
| schema_data_file, And(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MIN_VALUE + 1)) |
| ).eval(data_file) |
| assert not should_read, "Should skip: and(false, false)" |
| |
| should_read = _InclusiveMetricsEvaluator( |
| schema_data_file, And(GreaterThan("id", INT_MIN_VALUE - 25), LessThanOrEqual("id", INT_MIN_VALUE)) |
| ).eval(data_file) |
| assert should_read, "Should read: and(true, true)" |
| |
| |
| def test_or(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator( |
| schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MAX_VALUE + 1)) |
| ).eval(data_file) |
| assert not should_read, "Should skip: or(false, false)" |
| |
| should_read = _InclusiveMetricsEvaluator( |
| schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MAX_VALUE - 19)) |
| ).eval(data_file) |
| assert should_read, "Should read: or(false, true)" |
| |
| |
| def test_integer_lt(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MIN_VALUE - 25)).eval(data_file) |
| assert not should_read, "Should not read: id range below lower bound (5 < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MIN_VALUE)).eval(data_file) |
| assert not should_read, "Should not read: id range below lower bound (30 is not < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MIN_VALUE + 1)).eval(data_file) |
| assert should_read, "Should read: one possible id" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MAX_VALUE)).eval(data_file) |
| assert should_read, "Should read: may possible ids" |
| |
| |
| def test_integer_lt_eq(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MIN_VALUE - 25)).eval(data_file) |
| assert not should_read, "Should not read: id range below lower bound (5 < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MIN_VALUE - 1)).eval(data_file) |
| assert not should_read, "Should not read: id range below lower bound (30 is not < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MIN_VALUE)).eval(data_file) |
| assert should_read, "Should read: one possible id" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MAX_VALUE)).eval(data_file) |
| assert should_read, "Should read: may possible ids" |
| |
| |
| def test_integer_gt(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MAX_VALUE + 6)).eval(data_file) |
| assert not should_read, "Should not read: id range above upper bound (85 < 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MAX_VALUE)).eval(data_file) |
| assert not should_read, "Should not read: id range above upper bound (79 is not > 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MIN_VALUE - 1)).eval(data_file) |
| assert should_read, "Should read: one possible id" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MAX_VALUE - 4)).eval(data_file) |
| assert should_read, "Should read: may possible ids" |
| |
| |
| def test_integer_gt_eq(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE + 6)).eval(data_file) |
| assert not should_read, "Should not read: id range above upper bound (85 < 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE + 1)).eval(data_file) |
| assert not should_read, "Should not read: id range above upper bound (80 > 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE)).eval(data_file) |
| assert should_read, "Should read: one possible id" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE - 4)).eval(data_file) |
| assert should_read, "Should read: may possible ids" |
| |
| |
| def test_integer_eq(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MIN_VALUE - 25)).eval(data_file) |
| assert not should_read, "Should not read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MIN_VALUE - 1)).eval(data_file) |
| assert not should_read, "Should not read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MIN_VALUE)).eval(data_file) |
| assert should_read, "Should read: id equal to lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE - 4)).eval(data_file) |
| assert should_read, "Should read: id between lower and upper bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE)).eval(data_file) |
| assert should_read, "Should read: id equal to upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE + 1)).eval(data_file) |
| assert not should_read, "Should not read: id above upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE + 6)).eval(data_file) |
| assert not should_read, "Should not read: id above upper bound" |
| |
| |
| def test_integer_not_eq(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MIN_VALUE - 25)).eval(data_file) |
| assert should_read, "Should read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MIN_VALUE - 1)).eval(data_file) |
| assert should_read, "Should read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MIN_VALUE)).eval(data_file) |
| assert should_read, "Should read: id equal to lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE - 4)).eval(data_file) |
| assert should_read, "Should read: id between lower and upper bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE)).eval(data_file) |
| assert should_read, "Should read: id equal to upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE + 1)).eval(data_file) |
| assert should_read, "Should read: id above upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE + 6)).eval(data_file) |
| assert should_read, "Should read: id above upper bound" |
| |
| |
| def test_integer_not_eq_rewritten(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MIN_VALUE - 25))).eval(data_file) |
| assert should_read, "Should read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MIN_VALUE - 1))).eval(data_file) |
| assert should_read, "Should read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MIN_VALUE))).eval(data_file) |
| assert should_read, "Should read: id equal to lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE - 4))).eval(data_file) |
| assert should_read, "Should read: id between lower and upper bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE))).eval(data_file) |
| assert should_read, "Should read: id equal to upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE + 1))).eval(data_file) |
| assert should_read, "Should read: id above upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE + 6))).eval(data_file) |
| assert should_read, "Should read: id above upper bound" |
| |
| |
| def test_integer_case_insensitive_not_eq_rewritten(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MIN_VALUE - 25)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MIN_VALUE - 1)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id below lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MIN_VALUE)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id equal to lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE - 4)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id between lower and upper bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id equal to upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE + 1)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id above upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE + 6)), case_sensitive=False).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id above upper bound" |
| |
| |
| def test_missing_column_case_sensitive(schema_data_file: Schema, data_file: DataFile) -> None: |
| with pytest.raises(ValueError) as exc_info: |
| _ = _InclusiveMetricsEvaluator(schema_data_file, LessThan("ID", 22), case_sensitive=True).eval(data_file) |
| |
| assert str(exc_info.value) == "Could not find field with name ID, case_sensitive=True" |
| |
| |
| def test_integer_in(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval(data_file) |
| assert not should_read, "Should not read: id below lower bound (5 < 30, 6 < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MIN_VALUE - 2, INT_MIN_VALUE - 1})).eval(data_file) |
| assert not should_read, "Should not read: id below lower bound (28 < 30, 29 < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval(data_file) |
| assert should_read, "Should read: id equal to lower bound (30 == 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE - 4, INT_MAX_VALUE - 3})).eval(data_file) |
| assert should_read, "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE, INT_MAX_VALUE + 1})).eval(data_file) |
| assert should_read, "Should read: id equal to upper bound (79 == 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE + 1, INT_MAX_VALUE + 2})).eval(data_file) |
| assert not should_read, "Should not read: id above upper bound (80 > 79, 81 > 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE + 6, INT_MAX_VALUE + 7})).eval(data_file) |
| assert not should_read, "Should not read: id above upper bound (85 > 79, 86 > 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("all_nulls", {"abc", "def"})).eval(data_file) |
| assert not should_read, "Should skip: in on all nulls column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("some_nulls", {"abc", "def"})).eval(data_file) |
| assert should_read, "Should read: in on some nulls column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("no_nulls", {"abc", "def"})).eval(data_file) |
| assert should_read, "Should read: in on no nulls column" |
| |
| ids = list(range(400)) |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", ids)).eval(data_file) |
| assert should_read, "Should read: large in expression" |
| |
| |
| def test_integer_not_in(schema_data_file: Schema, data_file: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id below lower bound (5 < 30, 6 < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MIN_VALUE - 2, INT_MIN_VALUE - 1})).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id below lower bound (28 < 30, 29 < 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval(data_file) |
| assert should_read, "Should read: id equal to lower bound (30 == 30)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE - 4, INT_MAX_VALUE - 3})).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE, INT_MAX_VALUE + 1})).eval(data_file) |
| assert should_read, "Should read: id equal to upper bound (79 == 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE + 1, INT_MAX_VALUE + 2})).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id above upper bound (80 > 79, 81 > 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE + 6, INT_MAX_VALUE + 7})).eval( |
| data_file |
| ) |
| assert should_read, "Should read: id above upper bound (85 > 79, 86 > 79)" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("all_nulls", {"abc", "def"})).eval(data_file) |
| assert should_read, "Should read: notIn on all nulls column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("some_nulls", {"abc", "def"})).eval(data_file) |
| assert should_read, "Should read: in on some nulls column" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("no_nulls", {"abc", "def"})).eval(data_file) |
| assert should_read, "Should read: in on no nulls column" |
| |
| |
| @pytest.fixture |
| def schema_data_file_nan() -> Schema: |
| return Schema( |
| NestedField(1, "all_nan", DoubleType(), required=True), |
| NestedField(2, "max_nan", DoubleType(), required=True), |
| NestedField(3, "min_max_nan", FloatType(), required=False), |
| NestedField(4, "all_nan_null_bounds", DoubleType(), required=True), |
| NestedField(5, "some_nan_correct_bounds", FloatType(), required=False), |
| ) |
| |
| |
| @pytest.fixture |
| def data_file_nan() -> DataFile: |
| return DataFile( |
| file_path="file.avro", |
| file_format=FileFormat.PARQUET, |
| partition={}, |
| record_count=50, |
| file_size_in_bytes=3, |
| column_sizes={ |
| 1: 10, |
| 2: 10, |
| 3: 10, |
| 4: 10, |
| 5: 10, |
| }, |
| value_counts={ |
| 1: 10, |
| 2: 10, |
| 3: 10, |
| 4: 10, |
| 5: 10, |
| }, |
| null_value_counts={ |
| 1: 0, |
| 2: 0, |
| 3: 0, |
| 4: 0, |
| 5: 0, |
| }, |
| nan_value_counts={1: 10, 4: 10, 5: 5}, |
| lower_bounds={ |
| 1: to_bytes(DoubleType(), float("nan")), |
| 2: to_bytes(DoubleType(), 7), |
| 3: to_bytes(FloatType(), float("nan")), |
| 5: to_bytes(FloatType(), 7), |
| }, |
| upper_bounds={ |
| 1: to_bytes(DoubleType(), float("nan")), |
| 2: to_bytes(DoubleType(), float("nan")), |
| 3: to_bytes(FloatType(), float("nan")), |
| 5: to_bytes(FloatType(), 22), |
| }, |
| ) |
| |
| |
| def test_inclusive_metrics_evaluator_less_than_and_less_than_equal(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: |
| for operator in [LessThan, LessThanOrEqual]: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: 1 is smaller than lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 10)).eval(data_file_nan) |
| assert should_read, "Should match: 10 is larger than lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("min_max_nan", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan_null_bounds", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: 1 is smaller than lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 10)).eval( |
| data_file_nan |
| ) |
| assert should_read, "Should match: 10 larger than lower bound" |
| |
| |
| def test_inclusive_metrics_evaluator_greater_than_and_greater_than_equal( |
| schema_data_file_nan: Schema, data_file_nan: DataFile |
| ) -> None: |
| for operator in [GreaterThan, GreaterThanOrEqual]: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 1)).eval(data_file_nan) |
| assert should_read, "Should match: upper bound is larger than 1" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 10)).eval(data_file_nan) |
| assert should_read, "Should match: upper bound is larger than 10" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("min_max_nan", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan_null_bounds", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 1)).eval(data_file_nan) |
| assert should_read, "Should match: 1 is smaller than upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 10)).eval( |
| data_file_nan |
| ) |
| assert should_read, "Should match: 10 is smaller than upper bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan", 30)).eval(data_file_nan) |
| assert not should_read, "Should not match: 30 is greater than upper bound" |
| |
| |
| def test_inclusive_metrics_evaluator_equals(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("all_nan", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("max_nan", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: 1 is smaller than lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("max_nan", 10)).eval(data_file_nan) |
| assert should_read, "Should match: 10 is within bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("min_max_nan", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("all_nan_null_bounds", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("some_nan_correct_bounds", 1)).eval(data_file_nan) |
| assert not should_read, "Should not match: 1 is smaller than lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("some_nan_correct_bounds", 10)).eval(data_file_nan) |
| assert should_read, "Should match: 10 is within bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("all_nan", 30)).eval(data_file_nan) |
| assert not should_read, "Should not match: 30 is greater than upper bound" |
| |
| |
| def test_inclusive_metrics_evaluator_not_equals(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("all_nan", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("max_nan", 10)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("max_nan", 10)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("min_max_nan", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("all_nan_null_bounds", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("some_nan_correct_bounds", 1)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("some_nan_correct_bounds", 10)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("some_nan_correct_bounds", 30)).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| |
| def test_inclusive_metrics_evaluator_in(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("all_nan", (1, 10, 30))).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("max_nan", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: 10 and 30 are greater than lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("min_max_nan", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("all_nan_null_bounds", (1, 10, 30))).eval(data_file_nan) |
| assert not should_read, "Should not match: all nan column doesn't contain number" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: 10 within bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) |
| assert not should_read, "Should not match: 1 and 30 not within bounds" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (5, 7))).eval(data_file_nan) |
| assert should_read, "Should match: overlap with lower bound" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (22, 25))).eval(data_file_nan) |
| assert should_read, "Should match: overlap with upper bounds" |
| |
| |
| def test_inclusive_metrics_evaluator_not_in(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("all_nan", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("max_nan", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("max_nan", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("min_max_nan", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("all_nan_null_bounds", (1, 10, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) |
| assert should_read, "Should match: no visibility" |
| |
| |
| def test_string_starts_with( |
| schema_data_file: Schema, data_file: DataFile, data_file_2: DataFile, data_file_3: DataFile, data_file_4: DataFile |
| ) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "a")).eval(data_file) |
| assert should_read, "Should read: no stats" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "a")).eval(data_file_2) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "aa")).eval(data_file_2) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "aaa")).eval(data_file_2) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "1s")).eval(data_file_3) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "1str1x")).eval(data_file_3) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "ff")).eval(data_file_4) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "aB")).eval(data_file_2) |
| assert not should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "dWX")).eval(data_file_2) |
| assert not should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "5")).eval(data_file_3) |
| assert not should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "3str3x")).eval(data_file_3) |
| assert not should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("some_empty", "房东整租霍")).eval(data_file) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("all_nulls", "")).eval(data_file) |
| assert not should_read, "Should not read: range doesn't match" |
| |
| # above_max = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); |
| |
| # should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", above_max)).eval(data_file_4) |
| # assert not should_read, "Should not read: range doesn't match" |
| |
| |
| def test_string_not_starts_with( |
| schema_data_file: Schema, data_file: DataFile, data_file_2: DataFile, data_file_3: DataFile, data_file_4: DataFile |
| ) -> None: |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "a")).eval(data_file) |
| assert should_read, "Should read: no stats" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "a")).eval(data_file_2) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "aa")).eval(data_file_2) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "aaa")).eval(data_file_2) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "1s")).eval(data_file_3) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "1str1x")).eval(data_file_3) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "ff")).eval(data_file_4) |
| assert should_read, "Should read: range matches" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "aB")).eval(data_file_2) |
| assert should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "dWX")).eval(data_file_2) |
| assert should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "5")).eval(data_file_3) |
| assert should_read, "Should not read: range doesn't match" |
| |
| should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "3str3x")).eval(data_file_3) |
| assert should_read, "Should not read: range doesn't match" |
| |
| # above_max = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); |
| |
| # should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", above_max)).eval(data_file_4) |
| # assert should_read, "Should not read: range doesn't match" |
