| ################################################################################ | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| ################################################################################ | |
| import json | |
| import unittest | |
| from types import MappingProxyType | |
| from typing import get_type_hints | |
| import pyflink.dataframe as pf | |
| from pyflink.dataframe import llm | |
| from pyflink.table import TableEnvironment | |
| class ModelProviderTests(unittest.TestCase): | |
| def test_triton_options_follow_community_option_names(self): | |
| self.assertEqual(pf.TritonProvider("http://localhost:8000").to_options(), | |
| {"endpoint": "http://localhost:8000"}) | |
| headers = {"X-Request": "comma, colon: quote'", "X-Empty": ""} | |
| provider = pf.TritonProvider( | |
| "http://localhost:8000", model_name="model", model_version="2", timeout="10 s", | |
| flatten_batch_dim=False, priority=0, sequence_id="18446744073709551615", | |
| sequence_start=False, sequence_end=True, compression="gzip", auth_token="token", | |
| custom_headers=headers, max_retries=0, retry_initial_backoff="100 ms", | |
| retry_max_backoff="1 s", default_value="null", health_check_enabled=True, | |
| health_check_interval="5 s", circuit_breaker_enabled=False, | |
| circuit_breaker_failure_threshold=0.5, circuit_breaker_timeout="30 s", | |
| circuit_breaker_half_open_requests=2, | |
| ) | |
| self.assertEqual(provider.provider_identifier(), "triton") | |
| self.assertEqual(provider.model_option_key(), "model-name") | |
| options = provider.to_options() | |
| self.assertEqual(json.loads(options.pop("custom-headers")), headers) | |
| expected = { | |
| "endpoint": "http://localhost:8000", "model-name": "model", "model-version": "2", | |
| "timeout": "10 s", "flatten-batch-dim": "false", "priority": "0", | |
| "sequence-id": "18446744073709551615", "sequence-start": "false", | |
| "sequence-end": "true", "compression": "gzip", "auth-token": "token", | |
| "max-retries": "0", "retry-initial-backoff": "100 ms", "retry-max-backoff": "1 s", | |
| "default-value": "null", "health-check-enabled": "true", "health-check-interval": "5 s", | |
| "circuit-breaker-enabled": "false", "circuit-breaker-failure-threshold": "0.5", | |
| "circuit-breaker-timeout": "30 s", "circuit-breaker-half-open-requests": "2", | |
| } | |
| self.assertEqual(options, expected) | |
| headers["X-Request"] = "changed" | |
| self.assertEqual(json.loads(provider.to_options()["custom-headers"]), | |
| {"X-Request": "comma, colon: quote'", "X-Empty": ""}) | |
| provider.to_options().clear() | |
| self.assertEqual(set(provider.to_options()), set(expected) | {"custom-headers"}) | |
| def test_triton_rejects_invalid_typed_options(self): | |
| for options, error in [ | |
| ({"model_name": 1}, TypeError), | |
| ({"timeout": 10}, TypeError), | |
| ({"flatten_batch_dim": "false"}, TypeError), | |
| ({"sequence_start": 1}, TypeError), | |
| ({"priority": True}, TypeError), | |
| ({"priority": 0.5}, TypeError), | |
| ({"priority": -1}, ValueError), | |
| ({"priority": 256}, ValueError), | |
| ({"max_retries": -1}, ValueError), | |
| ({"compression": "deflate"}, ValueError), | |
| ({"default_value": [0.0]}, TypeError), | |
| ({"custom_headers": [("X-Header", "value")]}, TypeError), | |
| ({"custom_headers": {1: "value"}}, TypeError), | |
| ({"custom_headers": {"X-Header": 1}}, TypeError), | |
| ({"health_check_enabled": 1}, TypeError), | |
| ({"circuit_breaker_enabled": "false"}, TypeError), | |
| ({"circuit_breaker_failure_threshold": 0.0}, ValueError), | |
| ({"circuit_breaker_failure_threshold": 1.1}, ValueError), | |
| ({"circuit_breaker_failure_threshold": True}, TypeError), | |
| ({"circuit_breaker_failure_threshold": float("inf")}, ValueError), | |
| ({"circuit_breaker_half_open_requests": 0}, ValueError), | |
| ({"retry_initial_backoff": 100}, TypeError), | |
| ({"extra_body": "{}"}, TypeError), | |
| ]: | |
| with self.subTest(options=options): | |
| with self.assertRaises(error): | |
| pf.TritonProvider("http://localhost:8000", **options) | |
| def test_openai_connection_options_allow_deferred_model_selection(self): | |
| endpoint = "https://example.test/v1/embeddings" | |
| provider = pf.OpenAIProvider(endpoint=endpoint, api_key="key") | |
| self.assertEqual(provider.provider_identifier(), "openai") | |
| self.assertEqual(provider.model_option_key(), "model") | |
| self.assertEqual(provider.to_options(), {"endpoint": endpoint, "api-key": "key"}) | |
| self.assertEqual( | |
| pf.OpenAIProvider(endpoint, "key", model="embed-model").to_options(), | |
| {"endpoint": endpoint, "api-key": "key", "model": "embed-model"}, | |
| ) | |
| def test_openai_options_follow_community_option_names(self): | |
| provider = pf.OpenAIProvider( | |
| "https://example.test/v1/chat/completions", "key", model="chat-model", | |
| system_prompt="", temperature=0.0, top_p=1.0, max_tokens=100, | |
| stop="END,STOP", presence_penalty=-1.0, n=2, seed=0, | |
| response_format="json_object", dimension=3, max_context_size=1000, | |
| context_overflow_action="truncated-head-log", error_handling_strategy="IGNORE", | |
| retry_num=0, retry_fallback_strategy="IGNORE", | |
| ) | |
| expected = { | |
| "endpoint": "https://example.test/v1/chat/completions", "api-key": "key", | |
| "model": "chat-model", "system-prompt": "", "temperature": "0.0", | |
| "top-p": "1.0", "max-tokens": "100", "stop": "END,STOP", | |
| "presence-penalty": "-1.0", "n": "2", "seed": "0", | |
| "response-format": "json_object", "dimension": "3", "max-context-size": "1000", | |
| "context-overflow-action": "truncated-head-log", "error-handling-strategy": "IGNORE", | |
| "retry-num": "0", "retry-fallback-strategy": "IGNORE", | |
| } | |
| self.assertEqual(provider.to_options(), expected) | |
| provider.to_options().clear() | |
| self.assertEqual(provider.to_options(), expected) | |
| def test_openai_rejects_invalid_typed_options(self): | |
| for overrides, error in [ | |
| ({"endpoint": None}, TypeError), | |
| ({"api_key": ""}, ValueError), | |
| ({"model": 1}, TypeError), | |
| ({"system_prompt": 1}, TypeError), | |
| ({"max_tokens": True}, TypeError), | |
| ({"dimension": "3"}, TypeError), | |
| ({"temperature": True}, TypeError), | |
| ({"temperature": float("nan")}, ValueError), | |
| ({"temperature": float("-inf")}, ValueError), | |
| ({"seed": True}, TypeError), | |
| ({"top_p": -0.1}, ValueError), | |
| ({"top_p": 1.1}, ValueError), | |
| ({"presence_penalty": -3.0}, ValueError), | |
| ({"presence_penalty": 3.0}, ValueError), | |
| ({"retry_num": -1}, ValueError), | |
| ({"n": 0}, ValueError), | |
| ({"max_context_size": 0}, ValueError), | |
| ({"stop": ["STOP"]}, TypeError), | |
| ({"response_format": "json_schema"}, ValueError), | |
| ({"context_overflow_action": "unknown"}, ValueError), | |
| ({"error_handling_strategy": "unknown"}, ValueError), | |
| ({"retry_fallback_strategy": "RETRY"}, ValueError), | |
| ({"task": "chat/completions"}, TypeError), | |
| ]: | |
| with self.subTest(overrides=overrides): | |
| arguments = {"endpoint": "https://example.test/chat/completions", "api_key": "key"} | |
| arguments.update(overrides) | |
| with self.assertRaises(error): | |
| pf.OpenAIProvider(**arguments) | |
| def test_numeric_options_accept_boundaries_and_unbounded_values(self): | |
| for top_p, presence_penalty in [(0, -2), (1, 2)]: | |
| with self.subTest(top_p=top_p, presence_penalty=presence_penalty): | |
| provider = pf.OpenAIProvider( | |
| "https://example.test/chat/completions", "key", | |
| temperature=-0.5, seed=-100, top_p=top_p, presence_penalty=presence_penalty, | |
| max_tokens=1, n=1, dimension=1, max_context_size=1, retry_num=0, | |
| ) | |
| self.assertEqual(provider.to_options(), { | |
| "endpoint": "https://example.test/chat/completions", "api-key": "key", | |
| "temperature": "-0.5", "seed": "-100", "top-p": str(top_p), | |
| "presence-penalty": str(presence_penalty), "max-tokens": "1", "n": "1", | |
| "dimension": "1", "max-context-size": "1", "retry-num": "0", | |
| }) | |
| provider = pf.TritonProvider( | |
| "http://localhost:8000", priority=255, max_retries=0, | |
| circuit_breaker_failure_threshold=1, circuit_breaker_half_open_requests=1, | |
| ) | |
| self.assertEqual(provider.to_options(), { | |
| "endpoint": "http://localhost:8000", "priority": "255", "max-retries": "0", | |
| "circuit-breaker-failure-threshold": "1", "circuit-breaker-half-open-requests": "1", | |
| }) | |
| def test_generic_provider_preserves_raw_options(self): | |
| expected = {"endpoint": "HTTPS://example.test/v1", "api-key": "key", | |
| "custom_option": "", "enabled": "false", "identifier": "raw", | |
| "self": "raw", "options": "raw"} | |
| for form in ("keywords", "mapping", "mixed", "none"): | |
| with self.subTest(form=form): | |
| options = dict(expected) | |
| if form == "keywords": | |
| provider = pf.GenericProvider("custom", **options) | |
| elif form == "mapping": | |
| provider = pf.GenericProvider("custom", MappingProxyType(options)) | |
| elif form == "mixed": | |
| provider = pf.GenericProvider( | |
| "custom", {"endpoint": options.pop("endpoint")}, **options) | |
| else: | |
| provider = pf.GenericProvider("custom", None, **options) | |
| options.clear() | |
| self.assertIsInstance(provider, pf.ModelProvider) | |
| self.assertEqual(provider.provider_identifier(), "custom") | |
| self.assertEqual(provider.model_option_key(), "model") | |
| self.assertEqual(provider.to_options(), expected) | |
| provider.to_options().clear() | |
| self.assertEqual(provider.to_options(), expected) | |
| def test_typed_providers_accept_raw_options(self): | |
| for provider_type, connection, raw, typed, expected in [ | |
| (pf.OpenAIProvider, | |
| {"endpoint": "https://example.test/chat/completions", "api_key": "key"}, | |
| {"model": "chat-model", "max-tokens": "100", "temperature": "0.0", | |
| "system-prompt": "", "future-option": "Mixed CASE"}, | |
| {"max_tokens": 100, "temperature": 0.0, "system_prompt": ""}, | |
| {"endpoint": "https://example.test/chat/completions", "api-key": "key", | |
| "model": "chat-model", "max-tokens": "100", "temperature": "0.0", | |
| "system-prompt": "", "future-option": "Mixed CASE"}), | |
| (pf.TritonProvider, | |
| {"endpoint": "http://localhost:8000"}, | |
| {"model-name": "classifier", "max-retries": "0", "flatten-batch-dim": "false", | |
| "future_option": ""}, | |
| {"max_retries": 0, "flatten_batch_dim": False}, | |
| {"endpoint": "http://localhost:8000", "model-name": "classifier", | |
| "max-retries": "0", "flatten-batch-dim": "false", "future_option": ""}), | |
| ]: | |
| for parameters in ({}, typed): | |
| with self.subTest(provider=provider_type.__name__, parameters=parameters): | |
| options = dict(raw) | |
| provider = provider_type( | |
| **connection, **parameters, options=MappingProxyType(options)) | |
| options.clear() | |
| self.assertEqual(provider.to_options(), expected) | |
| provider.to_options().clear() | |
| self.assertEqual(provider.to_options(), expected) | |
| def test_public_contracts_have_resolvable_type_hints(self): | |
| for api in (pf.set_model_provider, pf.set_default_model_provider, pf.list_model_providers, | |
| pf.ModelProvider.provider_identifier, pf.ModelProvider.to_options, | |
| pf.ModelProvider.model_option_key, pf.OpenAIProvider.__init__, | |
| pf.TritonProvider.__init__, pf.GenericProvider.__init__): | |
| with self.subTest(api=api.__qualname__): | |
| self.assertTrue(get_type_hints(api)) | |
| def test_providers_reject_conflicting_options(self): | |
| for construct, key, value in [ | |
| (lambda options: pf.OpenAIProvider( | |
| "https://example.test/chat/completions", "key", max_tokens=10, options=options), | |
| "max-tokens", "10"), | |
| (lambda options: pf.OpenAIProvider( | |
| "https://example.test/chat/completions", "key", options=options), | |
| "api-key", "key"), | |
| (lambda options: pf.OpenAIProvider( | |
| "https://example.test/chat/completions", "key", system_prompt="", options=options), | |
| "system-prompt", ""), | |
| (lambda options: pf.TritonProvider( | |
| "http://localhost:8000", flatten_batch_dim=False, options=options), | |
| "flatten-batch-dim", "false"), | |
| (lambda options: pf.TritonProvider( | |
| "http://localhost:8000", max_retries=0, options=options), | |
| "max-retries", "0"), | |
| (lambda options: pf.GenericProvider("custom", options, timeout="10 s"), | |
| "timeout", "10 s"), | |
| ]: | |
| with self.subTest(key=key): | |
| options = {key: value} | |
| self.assertEqual(construct(options).to_options()[key], value) | |
| options[key] = "different" | |
| with self.assertRaisesRegex(ValueError, "conflicting values for option"): | |
| construct(options) | |
| self.assertEqual(options, {key: "different"}) | |
| def test_generic_provider_rejects_invalid_configuration(self): | |
| for identifier, options, error in [ | |
| (1, {}, TypeError), | |
| ("", {}, ValueError), | |
| (" ", {}, ValueError), | |
| ("custom", {"enabled": True}, TypeError), | |
| ("custom", {"retries": 1}, TypeError), | |
| ("custom", {"endpoint": None}, TypeError), | |
| ]: | |
| with self.subTest(identifier=identifier, options=options): | |
| with self.assertRaises(error): | |
| pf.GenericProvider(identifier, **options) | |
| def test_providers_reject_invalid_raw_options(self): | |
| for name, construct, expected in ( | |
| ("openai", lambda options: pf.OpenAIProvider( | |
| "https://example.test/chat/completions", "key", options=options), | |
| {"endpoint": "https://example.test/chat/completions", "api-key": "key"}), | |
| ("triton", lambda options: pf.TritonProvider("http://localhost:8000", options=options), | |
| {"endpoint": "http://localhost:8000"}), | |
| ("generic", lambda options: pf.GenericProvider("custom", options), {}), | |
| ): | |
| for options in ([], "options", False, {1: "value"}, {"option": None}, | |
| {"option": False}, {"option": 1}): | |
| with self.subTest(provider=name, options=options): | |
| with self.assertRaises(TypeError): | |
| construct(options) | |
| self.assertEqual(construct({}).to_options(), expected) | |
| with self.assertRaises(TypeError): | |
| pf.OpenAIProvider("https://example.test/chat/completions", "key", | |
| max_tokens=True, options={"max-tokens": "1"}) | |
| with self.assertRaises(ValueError): | |
| pf.TritonProvider("http://localhost:8000", priority=256, options={"priority": "1"}) | |
| class ModelProviderRegistryTests(unittest.TestCase): | |
| def setUp(self): | |
| previous = dict(llm._provider_registry) | |
| self.addCleanup(llm._provider_registry.update, previous) | |
| self.addCleanup(llm._provider_registry.clear) | |
| llm._provider_registry.clear() | |
| previous_default = llm._default_provider | |
| self.addCleanup(setattr, llm, "_default_provider", previous_default) | |
| llm._default_provider = None | |
| def test_registration_is_lazy_and_names_are_ordered(self): | |
| class UnserializedProvider(pf.ModelProvider): | |
| def provider_identifier(self): | |
| raise AssertionError("Registration must not inspect the provider") | |
| def to_options(self): | |
| raise AssertionError("Registration must not serialize the provider") | |
| pf.set_model_provider(name="chat", provider=UnserializedProvider()) | |
| pf.set_model_provider("embed", provider=pf.GenericProvider("custom")) | |
| names = pf.list_model_providers() | |
| self.assertEqual(names, ["chat", "embed"]) | |
| names.clear() | |
| self.assertEqual(pf.list_model_providers(), ["chat", "embed"]) | |
| def test_single_provider_registration_derives_names_without_serializing(self): | |
| class UnserializedProvider(pf.ModelProvider): | |
| def provider_identifier(self): | |
| return "custom" | |
| def to_options(self): | |
| raise AssertionError("Registration must not serialize the provider") | |
| def model_option_key(self): | |
| raise AssertionError("Registration must not inspect model options") | |
| provider = UnserializedProvider() | |
| pf.set_model_provider(provider) | |
| self.assertEqual(pf.list_model_providers(), ["custom"]) | |
| self.assertIs(llm._resolve_provider(), provider) | |
| pf.set_model_provider(provider=pf.OpenAIProvider("https://example.test/embeddings", "key")) | |
| with self.assertRaisesRegex(ValueError, "Multiple model providers"): | |
| llm._resolve_provider() | |
| pf.set_default_model_provider("custom") | |
| pf.set_model_provider(pf.TritonProvider("http://localhost:8000")) | |
| self.assertEqual(pf.list_model_providers(), ["custom", "openai", "triton"]) | |
| self.assertIs(llm._resolve_provider(), provider) | |
| def test_invalid_registration_preserves_registered_names(self): | |
| class InvalidIdentifierProvider(pf.ModelProvider): | |
| def __init__(self, identifier): | |
| self.identifier = identifier | |
| def provider_identifier(self): | |
| if isinstance(self.identifier, RuntimeError): | |
| raise self.identifier | |
| return self.identifier | |
| def to_options(self): | |
| raise AssertionError("Registration must not serialize the provider") | |
| provider = pf.GenericProvider("custom") | |
| other = pf.GenericProvider("other") | |
| pf.set_model_provider("chat", provider) | |
| pf.set_model_provider("embed", other) | |
| pf.set_default_model_provider("chat") | |
| invalid_calls = [ | |
| (("", provider), {}, ValueError), | |
| ((" ", provider), {}, ValueError), | |
| ((1, provider), {}, TypeError), | |
| ((None, provider), {}, TypeError), | |
| (("chat", None), {}, TypeError), | |
| (("embed", "custom"), {}, TypeError), | |
| ((), {}, TypeError), | |
| (("chat",), {}, TypeError), | |
| ((), {"provider": None}, TypeError), | |
| ((), {"name": provider}, TypeError), | |
| ((provider, provider), {}, TypeError), | |
| ((provider, None), {}, TypeError), | |
| (("chat", provider, provider), {}, TypeError), | |
| (("chat",), {"name": "embed", "provider": provider}, TypeError), | |
| (("chat", provider), {"provider": other}, TypeError), | |
| (("chat", provider), {"endpoint": "https://example.test"}, TypeError), | |
| ] | |
| for identifier, error in [ | |
| ("", ValueError), (" ", ValueError), (1, TypeError), (None, TypeError), | |
| (RuntimeError("Identifier unavailable"), RuntimeError), | |
| ]: | |
| invalid_calls.append(((), {"provider": InvalidIdentifierProvider(identifier)}, error)) | |
| for args, kwargs, error in invalid_calls: | |
| with self.subTest(args=args, kwargs=kwargs): | |
| with self.assertRaises(error): | |
| pf.set_model_provider(*args, **kwargs) | |
| self.assertEqual(pf.list_model_providers(), ["chat", "embed"]) | |
| self.assertIs(llm._resolve_provider(), provider) | |
| self.assertIs(llm._resolve_provider("chat"), provider) | |
| self.assertIs(llm._resolve_provider("embed"), other) | |
| def test_replacement_preserves_order_and_defaults_bound_to_names(self): | |
| original = pf.GenericProvider("custom", token="old") | |
| pf.set_model_provider(original) | |
| replacement = pf.GenericProvider("custom", token="new") | |
| pf.set_model_provider(replacement) | |
| self.assertEqual(pf.list_model_providers(), ["custom"]) | |
| self.assertIs(llm._resolve_provider(), replacement) | |
| pf.set_model_provider("other", pf.GenericProvider("other")) | |
| pf.set_default_model_provider("custom") | |
| named = pf.GenericProvider("different-factory") | |
| pf.set_model_provider(name="custom", provider=named) | |
| self.assertEqual(pf.list_model_providers(), ["custom", "other"]) | |
| self.assertIs(llm._resolve_provider(), named) | |
| pf.set_model_provider(provider=original) | |
| other = pf.GenericProvider("another-factory") | |
| pf.set_model_provider("other", other) | |
| self.assertEqual(pf.list_model_providers(), ["custom", "other"]) | |
| self.assertIs(llm._resolve_provider(), original) | |
| self.assertIs(llm._resolve_provider("other"), other) | |
| def test_selection_requires_choice_only_with_multiple_providers(self): | |
| chat = pf.GenericProvider("custom", task="chat") | |
| embed = pf.GenericProvider("custom", task="embed") | |
| with self.assertRaisesRegex(ValueError, "No model provider"): | |
| llm._resolve_provider() | |
| pf.set_model_provider("chat", chat) | |
| self.assertIs(llm._resolve_provider(), chat) | |
| pf.set_model_provider("embed", embed) | |
| with self.assertRaisesRegex(ValueError, "Multiple model providers"): | |
| llm._resolve_provider() | |
| pf.set_default_model_provider("chat") | |
| pf.set_model_provider("other", pf.GenericProvider("other")) | |
| self.assertIs(llm._resolve_provider(), chat) | |
| self.assertIs(llm._resolve_provider("embed"), embed) | |
| def test_invalid_selection_preserves_default(self): | |
| provider = pf.GenericProvider("custom") | |
| pf.set_model_provider("chat", provider) | |
| pf.set_default_model_provider("chat") | |
| for name, error in [("unknown", ValueError), ("", ValueError), | |
| (" ", ValueError), (1, TypeError), (None, TypeError)]: | |
| with self.subTest(name=name): | |
| with self.assertRaises(error): | |
| pf.set_default_model_provider(name) | |
| self.assertIs(llm._resolve_provider(), provider) | |
| if name is not None: | |
| with self.assertRaises(error): | |
| llm._resolve_provider(name) | |
| def test_registry_and_default_survive_environment_replacement(self): | |
| previous_environment = pf.get_table_environment() | |
| buffered = dict(pf.config._buffered) | |
| self.addCleanup(pf.config._buffered.update, buffered) | |
| self.addCleanup(pf.config._buffered.clear) | |
| self.addCleanup(pf.set_table_environment, previous_environment) | |
| pf.config._buffered.clear() | |
| provider = pf.GenericProvider("custom") | |
| pf.set_model_provider("chat", provider) | |
| pf.set_model_provider("embed", pf.GenericProvider("custom")) | |
| pf.set_default_model_provider("chat") | |
| for environment in (object.__new__(TableEnvironment), None, | |
| object.__new__(TableEnvironment)): | |
| pf.set_table_environment(environment) | |
| self.assertEqual(pf.list_model_providers(), ["chat", "embed"]) | |
| self.assertIs(llm._resolve_provider(), provider) | |
| if __name__ == '__main__': | |
| unittest.main() |