tests/function_modifiers/test_validation.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import inspect

import numpy as np
import pandas as pd
import pytest

from hamilton import node
from hamilton.data_quality.base import DataValidationError, ValidationResult
from hamilton.function_modifiers import (
    DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG,
    IS_DATA_VALIDATOR_TAG,
    check_output,
    check_output_custom,
)
from hamilton.node import DependencyType

from tests.resources.dq_dummy_examples import (
    DUMMY_VALIDATORS_FOR_TESTING,
    AsyncSampleDataValidator,
    SampleDataValidator1,
    SampleDataValidator2,
    SampleDataValidator3,
)


def test_check_output_node_transform():
    decorator = check_output(
        importance="warn",
        default_validator_candidates=DUMMY_VALIDATORS_FOR_TESTING,
        dataset_length=1,
        dtype=np.int64,
    )

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    assert 4 == len(subdag)
    subdag_as_dict = {node_.name: node_ for node_ in subdag}
    assert sorted(subdag_as_dict.keys()) == [
        "fn",
        "fn_dummy_data_validator_2",
        "fn_dummy_data_validator_3",
        "fn_raw",
    ]
    # TODO -- change when we change the naming scheme
    assert subdag_as_dict["fn_raw"].input_types["input"][1] == DependencyType.REQUIRED
    assert 3 == len(
        subdag_as_dict["fn"].input_types
    )  # Three dependencies -- the two with DQ + the original
    # The final function should take in everything but only use the raw results
    assert (
        subdag_as_dict["fn"].callable(
            fn_raw="test",
            fn_dummy_data_validator_2=ValidationResult(True, "", {}),
            fn_dummy_data_validator_3=ValidationResult(True, "", {}),
        )
        == "test"
    )


def test_check_output_custom_node_transform():
    decorator = check_output_custom(
        SampleDataValidator2(dataset_length=1, importance="warn"),
        SampleDataValidator3(dtype=np.int64, importance="warn"),
    )

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    assert 4 == len(subdag)
    subdag_as_dict = {node_.name: node_ for node_ in subdag}
    assert sorted(subdag_as_dict.keys()) == [
        "fn",
        "fn_dummy_data_validator_2",
        "fn_dummy_data_validator_3",
        "fn_raw",
    ]
    # TODO -- change when we change the naming scheme
    assert subdag_as_dict["fn_raw"].input_types["input"][1] == DependencyType.REQUIRED
    assert 3 == len(
        subdag_as_dict["fn"].input_types
    )  # Three dependencies -- the two with DQ + the original
    data_validators = [
        value
        for value in subdag_as_dict.values()
        if value.tags.get("hamilton.data_quality.contains_dq_results", False)
    ]
    assert len(data_validators) == 2  # One for each validator
    first_validator, _ = data_validators
    assert (
        IS_DATA_VALIDATOR_TAG in first_validator.tags
        and first_validator.tags[IS_DATA_VALIDATOR_TAG] is True
    )  # Validates that all the required tags are included
    assert (
        DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG in first_validator.tags
        and first_validator.tags[DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG] == "fn"
    )

    # The final function should take in everything but only use the raw results
    assert (
        subdag_as_dict["fn"].callable(
            fn_raw="test",
            fn_dummy_data_validator_2=ValidationResult(True, "", {}),
            fn_dummy_data_validator_3=ValidationResult(True, "", {}),
        )
        == "test"
    )


def test_check_output_custom_only_uses_applicable_validators():
    applicable_validator = SampleDataValidator1(equal_to=10, importance="warn")
    incompatible_validator = SampleDataValidator2(dataset_length=1, importance="warn")
    decorator = check_output_custom(applicable_validator, incompatible_validator)

    def fn(input: int) -> int:
        return input

    validators = decorator.get_validators(node.Node.from_fn(fn))

    assert validators == [applicable_validator]


def test_check_output_custom_uses_no_incompatible_validators():
    decorator = check_output_custom(
        SampleDataValidator2(dataset_length=1, importance="warn"),
        SampleDataValidator3(dtype=np.int64, importance="warn"),
    )

    def fn(input: int) -> int:
        return input

    assert decorator.get_validators(node.Node.from_fn(fn)) == []


def test_check_output_custom_node_transform_duplicate():
    """You should be able to pass in the same validator twice; IRL it would be different args."""
    decorator = check_output_custom(
        SampleDataValidator2(dataset_length=1, importance="warn"),
        SampleDataValidator2(dataset_length=1, importance="warn"),
    )

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    assert 4 == len(subdag)
    subdag_as_dict = {node_.name: node_ for node_ in subdag}
    assert sorted(subdag_as_dict.keys()) == [
        "fn",
        "fn_dummy_data_validator_2",
        "fn_dummy_data_validator_2_1",
        "fn_raw",
    ]
    # TODO -- change when we change the naming scheme
    assert subdag_as_dict["fn_raw"].input_types["input"][1] == DependencyType.REQUIRED
    assert 3 == len(
        subdag_as_dict["fn"].input_types
    )  # Three dependencies -- the two with DQ + the original
    data_validators = [
        value
        for value in subdag_as_dict.values()
        if value.tags.get("hamilton.data_quality.contains_dq_results", False)
    ]
    assert len(data_validators) == 2  # One for each validator
    first_validator, _ = data_validators
    assert (
        IS_DATA_VALIDATOR_TAG in first_validator.tags
        and first_validator.tags[IS_DATA_VALIDATOR_TAG] is True
    )  # Validates that all the required tags are included
    assert (
        DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG in first_validator.tags
        and first_validator.tags[DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG] == "fn"
    )

    # The final function should take in everything but only use the raw results
    assert (
        subdag_as_dict["fn"].callable(
            fn_raw="test",
            fn_dummy_data_validator_2=ValidationResult(True, "", {}),
            fn_dummy_data_validator_2_1=ValidationResult(True, "", {}),
        )
        == "test"
    )


def test_check_output_custom_node_transform_raises_exception_with_failure():
    decorator = check_output_custom(
        SampleDataValidator2(dataset_length=1, importance="fail"),
        SampleDataValidator3(dtype=np.int64, importance="fail"),
    )

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    assert 4 == len(subdag)
    subdag_as_dict = {node_.name: node_ for node_ in subdag}

    with pytest.raises(DataValidationError):
        subdag_as_dict["fn"].callable(
            fn_raw=pd.Series([1.0, 2.0, 3.0]),
            fn_dummy_data_validator_2=ValidationResult(False, "", {}),
            fn_dummy_data_validator_3=ValidationResult(False, "", {}),
        )


def test_check_output_custom_node_transform_layered():
    decorator_1 = check_output_custom(
        SampleDataValidator2(dataset_length=1, importance="warn"),
    )

    decorator_2 = check_output_custom(SampleDataValidator3(dtype=np.int64, importance="warn"))

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag_first_transformation = decorator_1.transform_dag([node_], config={}, fn=fn)
    subdag_second_transformation = decorator_2.transform_dag(
        subdag_first_transformation, config={}, fn=fn
    )
    # One node for each dummy validator
    # One final node
    # One intermediate node for each of the functions (E.G. raw)
    # TODO -- ensure that the intermediate nodes don't share names
    assert 5 == len(subdag_second_transformation)


def test_data_quality_constants_for_api_consistency():
    # simple tests to test data quality constants remain the same
    assert IS_DATA_VALIDATOR_TAG == "hamilton.data_quality.contains_dq_results"
    assert DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG == "hamilton.data_quality.source_node"


def test_check_output_validation_error():
    """Tests that we wrap an error raised appropriately."""
    decorator = check_output(
        importance="warn",
        dtype=np.int64,
    )

    def fn(input: pd.Series) -> pd.DataFrame:
        return pd.DataFrame({"a": input})

    node_ = node.Node.from_fn(fn)
    with pytest.raises(ValueError) as e:
        decorator.transform_node(node_, config={}, fn=fn)
        assert "Could not resolve validators for @check_output for function [fn]" in str(e)


def test_check_output_custom_async_validator_creates_async_wrapper():
    """Async validators should produce async validation wrapper functions."""
    async_validator = AsyncSampleDataValidator(equal_to=10, importance="warn")
    decorator = check_output_custom(async_validator)

    def fn(input: int) -> int:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    subdag_as_dict = {n.name: n for n in subdag}

    validator_node = subdag_as_dict["fn_async_dummy_data_validator"]
    assert inspect.iscoroutinefunction(validator_node.callable)

    # final_node_callable should remain sync
    final_node = subdag_as_dict["fn"]
    assert not inspect.iscoroutinefunction(final_node.callable)


def test_check_output_custom_mixed_sync_async_validators():
    """Mix of sync and async validators should create correct wrapper types."""
    async_validator = AsyncSampleDataValidator(equal_to=10, importance="warn")
    sync_validator = SampleDataValidator1(equal_to=10, importance="warn")
    decorator = check_output_custom(async_validator, sync_validator)

    def fn(input: int) -> int:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    subdag_as_dict = {n.name: n for n in subdag}

    async_node = subdag_as_dict["fn_async_dummy_data_validator"]
    assert inspect.iscoroutinefunction(async_node.callable)

    sync_node = subdag_as_dict["fn_dummy_data_validator_1"]
    assert not inspect.iscoroutinefunction(sync_node.callable)

    # final_node_callable should remain sync
    final_node = subdag_as_dict["fn"]
    assert not inspect.iscoroutinefunction(final_node.callable)


@pytest.mark.asyncio
async def test_async_validator_wrapper_returns_validation_result():
    """Async validation wrapper should return a ValidationResult when awaited."""
    async_validator = AsyncSampleDataValidator(equal_to=10, importance="warn")
    decorator = check_output_custom(async_validator)

    def fn(input: int) -> int:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    subdag_as_dict = {n.name: n for n in subdag}

    validator_node = subdag_as_dict["fn_async_dummy_data_validator"]
    result = await validator_node.callable(fn_raw=10)
    assert isinstance(result, ValidationResult)
    assert result.passes is True

    result_fail = await validator_node.callable(fn_raw=5)
    assert isinstance(result_fail, ValidationResult)
    assert result_fail.passes is False


def test_check_output_disabled_via_config_returns_original_node():
    """With hamilton.data_quality.disable_checks=True, transform_node returns the original node unchanged."""
    decorator = check_output_custom(
        SampleDataValidator2(dataset_length=1, importance="warn"),
        SampleDataValidator3(dtype=np.int64, importance="warn"),
    )

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(
        node_, config={"hamilton.data_quality.disable_checks": True}, fn=fn
    )
    assert len(subdag) == 1
    assert subdag[0] is node_


def test_check_output_builtin_disabled_via_config_returns_original_node():
    """check_output (not custom) also respects hamilton.data_quality.disable_checks."""
    decorator = check_output(
        importance="warn",
        default_validator_candidates=DUMMY_VALIDATORS_FOR_TESTING,
        dataset_length=1,
        dtype=np.int64,
    )

    def fn(input: pd.Series) -> pd.Series:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(
        node_, config={"hamilton.data_quality.disable_checks": True}, fn=fn
    )
    assert len(subdag) == 1
    assert subdag[0] is node_


def test_sync_wrapper_guards_against_unawaited_coroutine():
    """Sync wrapper should raise TypeError if validator accidentally returns a coroutine."""

    class _SneakyAsyncValidator(AsyncSampleDataValidator):
        """Validator that is async but we'll test the guard by calling it synchronously."""

        pass

    # Manually construct a sync wrapper that calls an async validator
    # to test the guard path
    validator = _SneakyAsyncValidator(equal_to=10, importance="warn")

    # The sync wrapper from validation.py includes a guard
    # We test by directly calling the sync path with a validator that returns a coroutine

    decorator = check_output_custom(validator)

    def fn(input: int) -> int:
        return input

    node_ = node.Node.from_fn(fn)
    subdag = decorator.transform_node(node_, config={}, fn=fn)
    subdag_as_dict = {n.name: n for n in subdag}

    # The async validator should get an async wrapper, so guard won't trigger here.
    # Instead, test the guard directly by simulating a sync call to an async validate.
    validator_node = subdag_as_dict["fn_async_dummy_data_validator"]
    assert inspect.iscoroutinefunction(validator_node.callable)